Commit ff7379d1 authored by Nic's avatar Nic
Browse files

[validation] fix early-report type-checking bug

parent 406e9a22
......@@ -342,12 +342,6 @@ class DataFrameSchematizer:
validator,
check_types=True,
)
# if len(early_report) > 0:
# # we probably have false-positive here, since an empty cell may
# # be filled later on...
# if self._is_units:
# self._is_units = False
# df = types_checking
# =====================================================================
# builds mult-header from dataframes if schemas are units-aware
# =====================================================================
......@@ -404,7 +398,7 @@ class DataFrameSchematizer:
k: schema["properties"][k].get("units", "")
for k in schema["properties"]
}
for col, units in self._target_units.items():
for col, units in self._target_units.copy().items():
if not units:
continue
try:
......@@ -414,7 +408,15 @@ class DataFrameSchematizer:
except:
# maybe a string where a number is expected...
# let's investigate it for a better user feedback
breakpoint()
df[col] = pd.Series(
types_checking[col].values,
dtype="pint[ ]",
index=types_checking.index,
)
self._target_units[col] = "" # .pop(col)
self._source_units[col] = "" # .pop(col)
early_report[(col, "*")].append("unable to convert this column")
df = df.pint.dequantify()
df.columns = [t[0] for t in df.columns.tolist()] # restrict to level 0
# =====================================================================
......@@ -428,10 +430,7 @@ class DataFrameSchematizer:
k: v for k, v in _fillnas.items() if k in df.columns and v is not None
}
if fillnas:
try:
df = df.fillna(value=fillnas, downcast="infer")
except:
breakpoint()
df = df.fillna(value=fillnas, downcast="infer")
# =====================================================================
# second validation
# =====================================================================
......@@ -439,33 +438,34 @@ class DataFrameSchematizer:
report = _validate(
json.loads(json.dumps(df.to_dict(orient="list"), ignore_nan=True)),
validator=validator,
check_types=None,
check_types=False,
# report=early_report,
)
# =====================================================================
# it may happen that first validation has raised false-positive
# example: NaN for an integer-expected column, with defaulting
# occurring in-between
# =====================================================================
for k, errors in early_report.items():
filtered_errors = [e for e in errors if "is not of type" not in e]
if filtered_errors:
report[k].extend(filtered_errors)
report = {**early_report, **report}
return df, len(report) == 0, dict(report)
def _is_typing_error(error):
"""check if error is type-checking error
return True if error is type-checking related
"""
msgs = [error.message] + [e.message for e in error.context]
for msg in msgs:
if "is not of type" in msg:
return True
return False
def _validate(document, validator, check_types=None, report=None):
"""if check_types is:
* None (default) report everythong
* True: will only report typing
* False: will NOT report typing
"""
type_checking_clue = "is not of type"
if report is None:
report = defaultdict(list)
for error in validator.iter_errors(document):
if type_checking_clue in error.message:
if _is_typing_error(error):
# if error reports a type checking issue, we skip in case
# `check_types` is False
if check_types is False:
......
---
_length: &length
units: mm
default: 0
_pressure: &pressure
units: MPa
_mass: &mass
units: kg
_volume: &volume
units: mm^3
_force: &force
units: N
---
mass:
test_mass_kg:
types: number
<<: *mass
test_mass_lb:
types: number
<<: *mass
test_mass_lbs:
types: number
<<: *mass
test_mass_lbm:
types: number
<<: *mass
test_mass_tonne:
types: number
<<: *mass
test_mass_g:
types: number
<<: *mass
mandatory: true
......@@ -89,7 +89,7 @@ def test_DFS_01_validation():
|--------------+---------+-----------+-----------+-------|
| blank or nan | | True | FAIL | da |
| blank or nan | | False | blank nan | db |
| blank or nan | 1 | True | 1 | dc |
| blank or nan | 1 | True | FAIL | dc |
| blank or nan | 1 | False | 1 | dd |
| 2 | | True | 2 | de |
| 2 | | False | 2 | df |
......@@ -143,7 +143,10 @@ dh:
)
df, is_valid, errors = v.validate_dataframe(df)
assert is_valid is False
expected_errors = {("da", 0): ["None is not of type 'integer'"]}
expected_errors = {
("da", 0): ["None is not of type 'integer'"],
("dc", 0): ["None is not of type 'integer'"],
}
assert errors == expected_errors
# check that values have been correctly filled
expected_df = pd.DataFrame(
......@@ -511,3 +514,28 @@ def test_IVData_06_units(datadir):
df_imp = data_imp.convert_to(tabname) # .pint.to_base_units()
df_si = data_SI.convert_to(tabname) # .pint.to_base_units()
assert pd.testing.assert_frame_equal(df_imp, df_si) is None
def test_IVData_07_failing_units(datadir):
"""check units"""
indir, outdir = datadir
data_si = IVData()
data_si.read(os.path.join(indir, "test_05_units_failing.xlsx"))
# -------------------------------------------------------------------------
# read and check schema
data_si.read_schema(
os.path.join(indir, "test_05_SI.yaml"),
os.path.join(indir, "test_05_data.yaml"),
)
rep = data_si.validate()
assert rep == {
"mass": {
"general": ["'test_mass_g' is a required property"],
("test_mass_lb", "*"): ["unable to convert this column"],
("test_mass_lb", 0): [
"'a' is not valid under any of the given " "schemas",
"'a' is not of type 'number'",
"'a' is not of type 'null'",
],
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment