Commit ee928cf2 authored by Nic's avatar Nic
Browse files

add must_not_exist_in spec

parent f262f09c
......@@ -318,12 +318,13 @@ class DataFrameSchematizer:
"uniqueItems": True,
}
for colname, (ext_tabname, ext_colname) in self.xcalling.items():
for colname, (ext_tabname, ext_colname, func) in self.xcalling.items():
if "xcalls" not in schema:
schema["xcalls"] = {}
schema["xcalls"][colname] = {
"tabname": ext_tabname,
"colname": ext_colname,
"func": func,
}
return schema
......@@ -359,6 +360,7 @@ class DataFrameSchematizer:
mandatory=False,
uniqueness_id=None,
must_exist_in=None,
must_not_exist_in=None,
units=None,
**kwargs,
):
......@@ -373,7 +375,9 @@ class DataFrameSchematizer:
if uniqueness_id:
self.uniqueness_sets[uniqueness_id].append(name)
if must_exist_in:
self.xcalling[name] = tuple(must_exist_in.split("::"))
self.xcalling[name] = tuple(must_exist_in.split("::") + ["isin"])
if must_not_exist_in:
self.xcalling[name] = tuple(must_not_exist_in.split("::") + ["~isin"])
# ---------------------------------------------------------
if len(types) > 1:
items = {"anyOf": [{"type": typ} for typ in types]}
......@@ -727,30 +731,45 @@ class ValidatorMixin:
# ---------------------------------------------------------------------
# check cross-references calls
for src_tabname, targets in xcallings.items():
for src_colname, (target_tabname, target_colname) in targets.items():
schema = self._schemas[src_tabname].columns_specs[src_colname]
for src_colname, (target_tabname, target_colname, func) in targets.items():
try:
src = self._data[src_tabname][src_colname]
except KeyError:
# colname is probably missing. This may not be an issue
# if the given column is not mandatory
if schema.get("mandatory"):
if (
self._schemas[src_tabname]
.columns_specs[src_colname]
.get("mandatory")
):
logging.warning(
f"cannot check xcallings for {src_tabname}/{src_colname}."
)
continue
else:
target = self._data[target_tabname][target_colname]
missing = src[
~src.isin(target)
].dropna() # dropna since we may allow empty value
# calculating missing and applying dropna since we may allow empty value
if func.startswith("~"):
# must NOT exist in
overlapping = src[src.isin(target)].dropna()
else:
# must exist in...
missing = src[~src.isin(target)].dropna()
report = ret[src_tabname]
for row_nb, _missing_ref in missing.items():
if (src_colname, row_nb) not in ret[src_tabname]:
ret[src_tabname][(src_colname, row_nb)] = []
ret[src_tabname][(src_colname, row_nb)].append(
f"'value `{_missing_ref}` not found in `{target_tabname}::{target_colname}`"
)
if func.startswith("~"):
for row_nb, _overlapping_ref in overlapping.items():
if (src_colname, row_nb) not in ret[src_tabname]:
ret[src_tabname][(src_colname, row_nb)] = []
ret[src_tabname][(src_colname, row_nb)].append(
f"'value `{_overlapping_ref}` unexpectidely found in `{target_tabname}::{target_colname}`"
)
else:
for row_nb, _missing_ref in missing.items():
if (src_colname, row_nb) not in ret[src_tabname]:
ret[src_tabname][(src_colname, row_nb)] = []
ret[src_tabname][(src_colname, row_nb)].append(
f"'value `{_missing_ref}` not found in `{target_tabname}::{target_colname}`"
)
# ---------------------------------------------------------------------
# apply offset
if row_reporting_offset:
......
......@@ -31,6 +31,7 @@ colors:
types: integer
unique: true
mandatory: true
must_not_exist_in: beam_def::id
name:
types: string
mandatory: true
......
......@@ -813,7 +813,7 @@ def test_IVData_08_multi_cols_uniqueness(datadir):
def test_IVData_09_must_exist_in(datadir):
"""check must exist req"""
"""check 'must exist' and 'must not exist' req"""
indir, outdir = datadir
data = IVData()
data.read_excel(os.path.join(indir, "test_06_must_exist_in.xlsx"))
......@@ -828,6 +828,7 @@ def test_IVData_09_must_exist_in(datadir):
],
},
"colors": {
("id", 2): ["'value `1` unexpectidely found in `beam_def::id`"],
("id", 4): ["'tata' is not of type 'integer'"],
("name", 4): ["5 is not of type 'string'"],
},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment