Commit 69b7c6ab authored by Nic's avatar Nic
Browse files

begin working on code robustness

parent 16bac288
......@@ -112,6 +112,29 @@ def load_yaml(*files, clean=True, debug=False):
return specs
def quantify_df(df):
"""preprocess a dataframe assuming units strings are on first line"""
df = df.copy()
df.columns = pd.MultiIndex.from_tuples(zip(df.columns, df.iloc[0]))
df = df.iloc[1:] # drop first line which has been merged previously
# units = dict(df.columns.tolist()) # {'id': '', 'distA': 'm',... }
units = {col: u for col, u in df.columns if isinstance(u, str)}
# split df between numeric / non-numeric columns
df_num = pd.DataFrame(index=df.index)
df_nonnum = pd.DataFrame(index=df.index)
for col, col_units in df.columns:
if isinstance(col_units, str):
df_num[(col, col_units)] = pd.to_numeric(df[(col, col_units)])
else:
df_nonnum[(col, col_units)] = df[(col, col_units)]
df_num.columns = pd.MultiIndex.from_tuples(df_num.columns)
try:
df_nonnum.columns = pd.MultiIndex.from_tuples(df_nonnum.columns)
except:
pass
return df_num, df_nonnum, units
# use short units
pintpandas.PintType.ureg.default_format = "~P"
......@@ -348,65 +371,40 @@ class DataFrameSchematizer:
# builds mult-header from dataframes if schemas are units-aware
# =====================================================================
if self._is_units:
# -----------------------------------------------------------------
# work on a copy:
# so we can revert back to original `df` if something is failing
_df = df.copy()
# -----------------------------------------------------------------
# create a multi-index columns such as pint-pandas can work
_df.columns = pd.MultiIndex.from_tuples(
zip(_df.columns, _df.iloc[0].fillna(""))
)
_df = _df.iloc[1:]
# -----------------------------------------------------------------
# collect defined units for later usage
self._source_units = dict(
_df.columns.tolist()
) # {'id': '', 'distA': 'm',... }
for i in range(len(_df.columns)):
try:
_df.iloc[:, i] = pd.to_numeric(_df.iloc[:, i])
except:
pass # probably a column with strings
_props = schema["properties"]
self._target_units = {k: v for k in _props if (v := _props[k].get("units"))}
df_num, df_nonnum, self._source_units = quantify_df(df)
try:
# insert the correct to workaround this bug:
# https://github.com/hgrecco/pint-pandas/issues/49
_df = _df.pint.quantify(level=-1)
df_num = df_num.pint.quantify(level=-1)
except pint.errors.UndefinedUnitError as exc:
self._is_units = False
colnames = [
colname
for colname, unit in self._source_units.items()
if unit in exc.args
]
for colname in colnames:
early_report[(colname, "*")].append(
early_report[(colname, None)].append(
"undefined units '%s'" % self._source_units[colname]
)
df = df[1:]
except Exception as exc:
self._is_units = False
df = df[1:]
except Exception as exc:
early_report["uncatched unit error"].extend(list(exc.args))
else:
# -------------------------------------------------------------
# everything was fine, commit temporary `_df` to `df`:
df = _df
del _df
self._is_units = False
df = df[1:]
# =====================================================================
# convert read units to schema expected units (if required)
# =====================================================================
if self._is_units:
self._target_units = {
k: schema["properties"][k].get("units", "")
for k in schema["properties"]
}
# at this point, we still have df_num and df_nonnum
for col, units in self._target_units.copy().items():
if not units:
continue
try:
df[col] = df[col].pint.to(units)
df_num[col] = df_num[col].pint.to(units)
except KeyError:
pass
early_report[(col, None)].append("no units specified in source file")
except:
# maybe a string where a number is expected...
# let's investigate it for a better user feedback
......@@ -417,10 +415,13 @@ class DataFrameSchematizer:
)
self._target_units[col] = "" # .pop(col)
self._source_units[col] = "" # .pop(col)
early_report[(col, "*")].append("unable to convert this column")
early_report[(col, None)].append("unable to convert this column")
df = df.pint.dequantify()
df_num = df_num.pint.dequantify()
df = pd.concat((df_num, df_nonnum), axis=1)
# reorder columns as initially
df.columns = [t[0] for t in df.columns.tolist()] # restrict to level 0
df.columns = [c for c in types_checking.columns]
# =====================================================================
# fill empty values as requested by schema
# =====================================================================
......@@ -495,21 +496,20 @@ class ValidatorMixin:
self._schemas = {}
def getunits(self, tabname, key):
""" retrieve from schema the working units. Raise KeyError the the
"""retrieve from schema the working units. Raise KeyError the the
column is not units-aware
"""
return self.get(tabname, 'schemas').build()['properties'][key]['units']
return self.get(tabname, "schemas").build()["properties"][key]["units"]
def getrow(self, tabname, key, search_by=None):
""" retrieve a row from any tabname, eventually sorting by `search_by`
"""
"""retrieve a row from any tabname, eventually sorting by `search_by`"""
df = self.get(tabname).copy()
if search_by:
df.set_index(search_by, inplace=True)
return df.loc[key]
def getvalue(self, tabname, key, column, search_by=None, as_quantity=True):
""" retrive a cell content by intersecting row/column index
"""retrive a cell content by intersecting row/column index
if as_quantity is:
* `True`: return a quantity argument *if possible*
......@@ -615,8 +615,13 @@ class ValidatorMixin:
if not hasattr(self, "_raw_data"):
self._raw_data = {tabname: df.copy() for tabname, df in self._data.items()}
ret = {}
for tabname, df in self._data.items():
for tabname, df in self._data.copy().items():
df, is_ok, report = self._validate_tab(tabname)
if df is None:
# tabname not described in schema; drop it
logging.info('drop tab "%s" which is not described in schema.', tabname)
self._data.pop(tabname)
continue
self._data[tabname] = df # override with filled (fillna) dataframe
if not is_ok:
ret[tabname] = report
......
......@@ -164,6 +164,76 @@ dh:
assert pd.testing.assert_frame_equal(df, expected_df, check_dtype=False) is None
def test_DFS_02_validation():
"""test DataFrameSchematizer class validation.
specifically testing enums with default
"""
v = DataFrameSchematizer()
columns_specs = """\
---
CA:
types: integer
default: 0
units: m
CB:
types: number
default: 1.0
CC:
types: number
default: 1.0
CD:
types: number
default: 0
units: N
CE:
types: string
enum:
- fwd
- aft
default: aft
CF:
types: number
default: 0
units: kg
"""
v.add_columns(columns_specs)
df = pd.DataFrame(
{
"CA": {0: "cm", 1: 5, 2: 15},
"CB": {0: "", 1: 5.3}, # empty units specified, but that's fine
"CC": {1: 5.3}, # no units specified, but that's fine
"CD": {1: 54.3}, # no units specified, that's an issue!
"CE": {1: "fwd"},
"CF": {0: "lbm", 1: 2}, # "lbm" doesn't exist
}
)
df, is_valid, errors = v.validate_dataframe(df)
breakpoint()
assert errors == {}
assert (
pd.testing.assert_frame_equal(
df, pd.DataFrame({"da": {1: 0.05, 2: 0.15}, "db": {1: "aft", 2: "fwd"}})
)
is None
)
df = pd.DataFrame(
{
"da": {0: "cm", 2: 15},
"db": {0: "", 2: "fwd"},
}
)
df, is_valid, errors = v.validate_dataframe(df)
assert errors == {}
breakpoint()
assert (
pd.testing.assert_frame_equal(
df, pd.DataFrame({"da": {1: 0.05, 2: 0.15}, "db": {1: "aft", 2: "fwd"}})
)
is None
)
def test_VData_00():
data = VData()
# ------------------------------------------------------------------------
......@@ -478,9 +548,12 @@ def test_IVData_05_failing_units(datadir):
expected_report = {
"mass": {
("test_mass_g", 2): ["None is not of type 'number'"],
("test_mass_lbm", "*"): ["undefined units 'lbm'"],
("test_mass_lbm", None): ["undefined units 'lbm'"],
},
"volume": {
"general": ["'test_volume_foot3' is a required property"],
("test_volume_foot3", None): ["no units specified in source file"],
},
"volume": {"general": ["'test_volume_foot3' is a required property"]},
}
assert expected_report == ret
......@@ -610,7 +683,7 @@ def test_IVData_07_failing_units(datadir):
assert rep == {
"mass": {
"general": ["'test_mass_g' is a required property"],
("test_mass_lb", "*"): ["unable to convert this column"],
("test_mass_lb", None): ["unable to convert this column"],
("test_mass_lb", 0): [
"'a' is not valid under any of the given " "schemas",
"'a' is not of type 'number'",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment