Commit 5a9bd1ca authored by Nic's avatar Nic
Browse files

allow padding as default value

parent 68f00a2b
......@@ -389,6 +389,7 @@ class DataFrameSchematizer:
3. default values are applied
4. final validation
"""
_df_debug = df.copy()
schema = self.build()
validator = jsonschema.Draft7Validator(schema)
# =====================================================================
......@@ -456,21 +457,25 @@ class DataFrameSchematizer:
k: schema["properties"][k]["items"].get("default")
for k, v in schema["properties"].items()
}
padded_columns = [colname for colname, value in _fillnas.items() if value == "_pad_"]
fillnas = {
k: v for k, v in _fillnas.items() if k in df.columns and v is not None
}
if fillnas:
df = df.fillna(value=fillnas, downcast="infer")
if padded_columns:
df = df.replace({"_pad_": np.nan})
df[padded_columns] = df[padded_columns].fillna(method="pad")
# df = df.fillna(method="pad",
# =====================================================================
# build dummy columns if required by the multiple-columns uniqueness
# =====================================================================
dummies = []
for uniqueness_tag, columns in self.uniqueness_sets.items():
_df = df[columns].astype(str)
_df = df[columns].fillna(method="ffill").astype(str)
dummy_col = UNIQUENESS_SEP.join(columns)
dummies.append(dummy_col) # to delete them later on
df[dummy_col] = _df.apply(UNIQUENESS_SEP.join, axis=1)
# =====================================================================
# second validation
# =====================================================================
......
......@@ -4,23 +4,34 @@ asm: &asm
types: string
unique: false
uniqueness_id: toto
mandatory: true
mandatory: false
default: _pad_
x_loc:
<<: *length
mandatory: true
mandatory: false
uniqueness_id: toto
default: _pad_
block_label:
types: string
unique: false
mandatory: true
mandatory: false
uniqueness_id: toto
default: _pad_
block_ref_point_id:
mandatory: true
another:
unique: true
optional:
types: number
asm_ok:
<<: *asm
asm_failing:
<<: *asm
WithPadding_ok:
<<: *asm
WithPadding_failing:
<<: *asm
[WithPadding_failing]
[[_row_0]]
rail_tag = ''
x_loc = 'mm'
block_label = ''
block_ref_point_id = ''
another = ''
optional = ''
[[_row_1]]
rail_tag = 'RBL11'
x_loc = 3500
block_label = 'RECARO1'
block_ref_point_id = 2.0
another = 1.0
optional = 2.0
[[_row_2]]
rail_tag = 'RBL11'
x_loc = 8500
block_label = 'RECARO1'
block_ref_point_id = 2.0
another = 2.0
optional = 3.0
[[_row_3]]
rail_tag = 'RBL12'
x_loc = 3800
block_label = 's1'
block_ref_point_id = 2.0
another = 3.0
optional = ''
[[_row_4]]
rail_tag = ''
x_loc = ''
block_label = 's1'
block_ref_point_id = 2.0
another = 4.0
optional = 0.0
[[_row_5]]
rail_tag = ''
x_loc = 8500
block_label = 's1'
block_ref_point_id = 2.0
another = 5.0
optional = ''
[[_row_6]]
rail_tag = ''
x_loc = 7900
block_label = 's1'
block_ref_point_id = 2.0
another = 6.0
optional = ''
[asm_ok]
[[_row_0]]
rail_tag = ''
......@@ -42,3 +92,53 @@
block_label = 'RECARO1'
block_ref_point_id = 3.0
another = 2.0
[WithPadding_ok]
[[_row_0]]
rail_tag = ''
x_loc = 'mm'
block_label = ''
block_ref_point_id = ''
another = ''
optional = ''
[[_row_1]]
rail_tag = 'RBL11'
x_loc = 3500
block_label = 'RECARO1'
block_ref_point_id = 2.0
another = 1.0
optional = 2.0
[[_row_2]]
rail_tag = 'RBL11'
x_loc = 8500
block_label = 'RECARO1'
block_ref_point_id = 2.0
another = 2.0
optional = 3.0
[[_row_3]]
rail_tag = 'RBL12'
x_loc = 3800
block_label = 's1'
block_ref_point_id = 2.0
another = 3.0
optional = ''
[[_row_4]]
rail_tag = ''
x_loc = ''
block_label = 's2'
block_ref_point_id = 2.0
another = 4.0
optional = 0.0
[[_row_5]]
rail_tag = ''
x_loc = 8500
block_label = 's1'
block_ref_point_id = 2.0
another = 5.0
optional = ''
[[_row_6]]
rail_tag = ''
x_loc = 7900
block_label = 's1'
block_ref_point_id = 2.0
another = 6.0
optional = ''
......@@ -714,10 +714,70 @@ def test_IVData_08_multi_cols_uniqueness(datadir):
)
rep = data_si.validate()
assert rep == {
"WithPadding_failing": {
("rail_tag & x_loc & block_label", "?"): [
"value " "{'RBL12 & " "3800.0 & " "s1'} is " "not " "unique"
]
},
"asm_failing": {
("another", "?"): ["value {1.0} is not unique"],
("rail_tag & x_loc & block_label", "?"): [
"value {'RBL11 & 3500 & RECARO1'} is not unique"
"value {'RBL11 & " "3500 & RECARO1'} " "is not unique"
],
}
},
}
df = data_si.get("WithPadding_ok")
expected_df = pd.DataFrame(
{
1: {
"another": 1.0,
"block_label": "RECARO1",
"block_ref_point_id": 2.0,
"optional": 2.0,
"rail_tag": "RBL11",
"x_loc": 3500.0,
},
2: {
"another": 2.0,
"block_label": "RECARO1",
"block_ref_point_id": 2.0,
"optional": 3.0,
"rail_tag": "RBL11",
"x_loc": 8500.0,
},
3: {
"another": 3.0,
"block_label": "s1",
"block_ref_point_id": 2.0,
"optional": np.nan,
"rail_tag": "RBL12",
"x_loc": 3800.0,
},
4: {
"another": 4.0,
"block_label": "s2",
"block_ref_point_id": 2.0,
"optional": 0.0,
"rail_tag": "RBL12",
"x_loc": 3800.0,
},
5: {
"another": 5.0,
"block_label": "s1",
"block_ref_point_id": 2.0,
"optional": np.nan,
"rail_tag": "RBL12",
"x_loc": 8500.0,
},
6: {
"another": 6.0,
"block_label": "s1",
"block_ref_point_id": 2.0,
"optional": np.nan,
"rail_tag": "RBL12",
"x_loc": 7900.0,
},
}
).T
expected_df = expected_df[[c for c in df]]
assert pd.testing.assert_frame_equal(df, expected_df, check_dtype=False) is None
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment