Commit ffe29420 authored by Nic's avatar Nic
Browse files

add XLSX template dumping

parent 84d71df9
......@@ -5,6 +5,8 @@ import shlex
import shutil
import subprocess as sp
from pathlib import Path
import string
import re
import numpy as np
import pandas as pd
......@@ -283,6 +285,139 @@ class VDataAutodoc:
fh.write(_write(f" * ``{uniq}``: {columns}"))
# =============================================================================
# dumping XLSX template
# =============================================================================
FMTS = {
"default": {
"bold": True,
"font_color": "#3A81CC",
"align": "center",
"valign": "vcenter",
"border": 1,
"bg_color": "#eeeeec",
},
"mandatory": {"bg_color": "#fcaf3e", "color": "black"},
"padded": {"bg_color": "#8ae234", "color": "black"},
"units": {"bg_color": "#729fcf", "color": "black"},
}
def fmt_dict(attributes=()):
fmt = FMTS["default"].copy()
for attr in attributes:
_d = FMTS[attr].copy()
fmt.update(_d)
return fmt
def dump_xlsx_template(schemas, output, tabs=None, excluded_tabs=None):
"""creates a blank XSLX spreadsheet based on provided schema"""
# -------------------------------------------------------------------------
alphabet = string.ascii_uppercase # "ABC..."
# -------------------------------------------------------------------------
# read shema(s)
data = IVData()
data.read_schema(*schemas)
tabs = list(tabs)
for tabname, schema in data._schemas.items():
for referencing_col, (
referenced_tab,
referenced_col,
) in schema.xcalling.items():
if referenced_tab not in tabs:
logging.warning(f"add referenced {referenced_tab}")
tabs.append(referenced_tab)
# -------------------------------------------------------------------------
# eventually filter with tabs wishlist
if tabs:
data._schemas = {k: v for k, v in data._schemas.items() if k in tabs}
if excluded_tabs:
data._schemas = {
k: v for k, v in data._schemas.items() if k not in excluded_tabs
}
# -------------------------------------------------------------------------
# automatically integrate referenced tabs
# -------------------------------------------------------------------------
# get columns and attributes
cols = data.dump_template()
with pd.ExcelWriter(output) as writer:
workbook = writer.book
# prepare formats
default_format = workbook.add_format(fmt_dict())
mandatory_format = workbook.add_format(fmt_dict(["mandatory"]))
padded_format = workbook.add_format(fmt_dict(["padded"]))
units_format = workbook.add_format(fmt_dict(["units"]))
for tabname, data in cols.items():
df = data["df"]
colspecs = data["schema"]
required = data["required"]
xcallings = data["xcalling"]
df.to_excel(
writer, sheet_name=tabname, startrow=1, header=False, index=False
)
worksheet = workbook.sheetnames[tabname]
# -----------------------------------------------------------------
# format header
# worksheet.set_row(0, height=None, cell_format=mandatory_format)
ws_has_units = {cs.get("units") for _, cs in colspecs.items()} > {None}
for i, colname in enumerate(df.columns):
is_mandatory = colname in required
pattern = colspecs[colname]["items"].get("pattern")
if pattern and not re.match(pattern, ""):
is_mandatory = True
default = colspecs[colname]["items"].get("default")
is_padded = False
if default == "_pad_":
is_mandatory = False
is_padded = True
xcalling = xcallings.get(colname)
if is_mandatory or xcalling:
fmt = mandatory_format
elif is_padded:
fmt = padded_format
else:
fmt = default_format
worksheet.write_string(0, i, colname, cell_format=fmt)
cell_header = "{col}{row}".format(col=alphabet[i], row=1)
# -------------------------------------------------------------
# process comments
types = set((colspecs[colname]["items"].get("type"),))
for typ in colspecs[colname]["items"].get("anyOf", []):
types.add(typ["type"])
types.discard(None)
types_str = ", ".join(types)
comment = f"Type(s): {types_str}"
if colspecs[colname]["uniqueItems"]:
comment += f"\nMust be Unique"
if pattern:
comment += f"\nPattern: {pattern}"
if xcalling:
comment += f"\ncross-ref: {xcalling[0]}/{xcalling[1]}"
if default is not None:
if isinstance(default, str):
comment += f'\ndefault: "{default}"'
else:
comment += f"\ndefault: {default}"
worksheet.write_comment(
cell_header,
comment,
{
"author": "numeric-GmbH",
"color": "#ad7fa8",
"visible": False,
},
)
if ws_has_units:
worksheet.set_row(1, height=None, cell_format=units_format)
worksheet.freeze_panes(2, 1)
else:
worksheet.freeze_panes(1, 1)
# -----------------------------------------------------------------
# autofit
return output
if __name__ == "__main__":
import doctest
......
......@@ -12,11 +12,13 @@ import argparse
import os
import pprint
import sys
import string
import pandas as pd
from gridsource import Data, IOData, ValidData, __version__
from gridsource.io import DEFAULT_EXCEL_ENGINE
from gridsource.autodoc import dump_xlsx_template
def version(args=None):
......@@ -72,21 +74,17 @@ def export(args):
return 0
def dump_xlsx_template(args):
def cli_dump_xlsx_template(args):
"""creates a blank XSLX spreadsheet based on provided schema"""
schema = args.schema
schemas = args.schemas
output = args.output
engine = args.engine
schema_dir, schema_fname = os.path.split(schema)
schema_fname, _ = os.path.splitext(schema_fname)
data = Data()
data.read_schema(args.schema)
breakpoint()
cols = data.dump_template()
with pd.ExcelWriter(output) as writer:
for tabname, df in cols.items():
df.to_excel(writer, sheet_name=tabname, index=False)
return output
tabs = args.tabs
excluded_tabs = args.excluded_tabs
output = dump_xlsx_template(
output=output, tabs=tabs, excluded_tabs=excluded_tabs, schemas=schemas
)
print(f"created {output}")
def parse_args(args):
......@@ -138,7 +136,9 @@ def parse_args(args):
parser_dumper = subparsers.add_parser(
"dump-template", help="dump template based on provided schema"
)
parser_dumper.add_argument("schema", type=str, help="path to schema file")
parser_dumper.add_argument(
"schemas", type=str, metavar="S", nargs="+", help="path to schema file(s)"
)
parser_dumper.add_argument(
"-o",
"--output",
......@@ -147,14 +147,31 @@ def parse_args(args):
help="output filepath",
)
parser_dumper.add_argument(
"-e",
"--engine",
type=str,
default="xlrd",
choices=("xlrd", "openpyxl"),
help="pandas XLSX reader engine",
)
parser_dumper.set_defaults(process=dump_xlsx_template)
parser_dumper.add_argument(
"-t",
"--tabs",
metavar="T",
nargs="+",
default=None,
type=str,
help="tabs whishlist",
)
parser_dumper.add_argument(
"-e",
"--excluded-tabs",
metavar="X",
nargs="+",
default=None,
type=str,
help="tabs blacklist",
)
parser_dumper.set_defaults(process=cli_dump_xlsx_template)
# ========================================================================
# return default func
......
......@@ -716,7 +716,9 @@ class ValidatorMixin:
xcallings[tabname] = xcalling
if df is None:
# tabname not described in schema; drop it
logging.info('drop tab "%s" which is not described in schema.', tabname)
logging.warning(
'drop tab "%s" which is not described in schema.', tabname
)
self._data.pop(tabname)
continue
self._data[tabname] = df # override with filled (fillna) dataframe
......@@ -726,9 +728,22 @@ class ValidatorMixin:
# check cross-references calls
for src_tabname, targets in xcallings.items():
for src_colname, (target_tabname, target_colname) in targets.items():
target = self._data[target_tabname][target_colname]
src = self._data[src_tabname][src_colname]
missing = src[~src.isin(target)]
schema = self._schemas[src_tabname].columns_specs[src_colname]
try:
src = self._data[src_tabname][src_colname]
except KeyError:
# colname is probably missing. This may not be an issue
# if the given column is not mandatory
if schema.get("mandatory"):
logging.warning(
f"cannot check xcallings for {src_tabname}/{src_colname}."
)
continue
else:
target = self._data[target_tabname][target_colname]
missing = src[
~src.isin(target)
].dropna() # dropna since we may allow empty value
report = ret[src_tabname]
for row_nb, _missing_ref in missing.items():
if (src_colname, row_nb) not in ret[src_tabname]:
......@@ -762,9 +777,18 @@ class ValidatorMixin:
def dump_template(self):
"""return list of columns ready to be dumped as XLSX template"""
dic = {}
dic = defaultdict(dict)
for tabname, schema in self._schemas.items():
dic[tabname] = pd.DataFrame({k: [] for k in schema.columns_specs.keys()})
df = pd.DataFrame(
{k: [v.get("units")] for k, v in schema.columns_specs.items()}
)
df.dropna(axis=0, how="all", inplace=True)
dic[tabname]["df"] = df
dic[tabname]["schema"] = schema.columns_specs
dic[tabname]["required"] = schema.required
dic[tabname]["xcalling"] = schema.xcalling
dic[tabname]["uniqueness"] = schema.uniqueness_sets
dic = dict(dic)
return dic
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment