Commit 7980c04b authored by Nic's avatar Nic
Browse files

improve autodoc

parent d3df9300
......@@ -49,15 +49,21 @@ def _indent(txt, indent):
def _comment(txt):
# ensure comment has no blank line
return _directive(content=txt)
def _include(filename, relative_to=False):
def _include(filename, relative_to=False, relative_prefix=None, literal=False):
if isinstance(filename, str):
filename = Path(filename)
if relative_to:
filename = filename.relative_to(relative_to)
return f"\n.. include:: {filename}\n\n"
if relative_prefix:
filename = relative_prefix / filename
include = "include"
if literal:
include = "literal" + include
return f"\n.. {include}:: {filename}\n\n"
def _directive(name="", arg=None, fields=None, content=None, indent=0):
......@@ -71,10 +77,9 @@ def _directive(name="", arg=None, fields=None, content=None, indent=0):
Add a comment with a not-named directive:
>>> print(_directive(content='bla'))
..
<BLANKLINE>
bla
>>> print(_directive(content='bla\\nbla'))
.. bla
blad
"""
o = list()
if name:
......@@ -90,12 +95,17 @@ def _directive(name="", arg=None, fields=None, content=None, indent=0):
o.append(_indent(":" + k + ": " + str(v), indent=1))
if content is not None:
o.append("")
if name:
o.append("")
if isinstance(content, list):
o.extend(_indent(content, 1))
else:
o.append(_indent(content, 1))
if not name:
# merge 1st and second item, since comments must start next to ".."
old_o = o[:]
o = [old_o[0] + old_o[1]] + old_o[2:]
return "\n".join(o)
......@@ -103,6 +113,7 @@ def _new_public_file(
filename,
titles=(),
includes=(),
clue_message=True,
):
"""create a file at `filename` path structured as follows:
......@@ -124,7 +135,11 @@ def _new_public_file(
# write titles
for title, title_level in titles:
fh.write(_write(title, header=title_level))
fh.write(_write(f"Please edit Me (``{filename}``)"))
if clue_message:
fh.write(_write(f"Please edit Me (``{filename}``)"))
else:
fh.write(_comment(f"Please edit Me (``{filename}``)"))
for include_target, relative_to in includes:
fh.write(_include(include_target, relative_to=relative_to))
......@@ -146,12 +161,19 @@ class VDataAutodoc:
"target {target_dir} exists and is not a proper Sphynx folder"
)
else:
logging.warning("Sphynx project does not exist. Please run `.create`")
logging.warning("Sphinx project does not exist. Please run `.create`")
else:
self.src_dir = self.target_dir / "source"
def create(
self, project_name, author, version, lang="en", exist_ok=False, templatedir=None
self,
project_name,
author,
version,
lang="en",
exist_ok=False,
templatedir=None,
**kwargs,
):
if self.target_dir.exists():
if not exist_ok:
......@@ -162,25 +184,25 @@ class VDataAutodoc:
# create project structure
cmd = f"sphinx-quickstart --sep -p {project_name} -a {author} -v {version} -l {lang} {self.target_dir} -q --ext-mathjax"
if templatedir:
cmd += f" --templatedir {templatedir}"
cmd += f" --templatedir {templatedir} "
# =====================================================================
# updating expected mastertoctree with head and tail chapters
# =====================================================================
mastertoctree = ["head.rst"] + kwargs.pop("mastertoctree") + ["tail.rst"]
mastertoctree = _indent("\n".join(mastertoctree), 1)
cmd += f"-d mastertoctree='{mastertoctree}' "
cmd += " ".join([f"-d {k}={v}" for k, v in kwargs.items()])
print(40 * "*")
print(cmd)
print(40 * "*")
args = shlex.split(cmd)
ret = sp.run(args)
index_content = (
f"Welcome to {project_name}'s documentation!\n"
"===========================================\n"
"\n"
".. toctree::\n"
" :maxdepth: 3\n"
" :caption: Contents:\n"
"\n"
" input_data.rst\n"
)
self.src_dir = self.target_dir / "source"
with open(self.src_dir / "index.rst", "w") as fh:
fh.write(index_content)
# ---------------------------------------------------------------------
# create head and tails files
for chapter in ("head.rst", "tail.rst"):
path = self.src_dir / Path(chapter)
_new_public_file(path, titles=(("Change Me", 1),))
def exists(self):
if not self.target_dir.exists():
......@@ -198,49 +220,52 @@ class VDataAutodoc:
order_columns=("column",),
tabs_chapters=((None, "*"),),
):
# =====================================================================
# index.rst call master file `input_data.rst`
# =====================================================================
# Private master file `.input_data.rst` is created from scratch
_master_file = self.src_dir / ".input_data.rst"
with open(_master_file, "w") as fh:
fh.write(_write("Input Data Specifications", header=1))
fh.write(_write("The following sections describe the expected data."))
# ---------------------------------------------------------------------
# Public master file is called "input_data.rst" and is created only
# if it doesn't exist
master_file = self.src_dir / "input_data.rst"
_new_public_file(
master_file,
titles=(("User Input", 1), ("Introduction", 2)),
includes=((_master_file, self.src_dir),),
)
# =====================================================================
# processing tabs
# intended file structure is:
# self.src_dir ("source")
# + root_tabdir ("source/tabs")
# + tabname.rst
# + .tabname.rst
# =====================================================================
root_tabdir = self.src_dir / "tabs"
root_tabdir.mkdir(exist_ok=True)
processed_tabs = set()
for tab_chapter, tabnames in tabs_chapters:
if tab_chapter is None:
tab_header_level = 2
else:
tab_header_level = 3
# update master fiel with chapter
with open(_master_file, "a") as fh:
fh.write(_write(tab_chapter, header=2))
slugged = tab_chapter.lower().replace(" ", "_") + ".rst"
master_file = self.src_dir / f"{slugged}"
_master_file = self.src_dir / f".{slugged}"
_master_file.touch()
# ---------------------------------------------------------------------
# Public master file is called "input_data.rst" and is created only
# if it doesn't exist
_new_public_file(
master_file,
titles=((tab_chapter, 1),),
includes=((_master_file, self.src_dir),),
)
# =====================================================================
# processing tabs
# intended file structure is:
# self.src_dir ("source")
# + root_tabdir ("source/tabs")
# + tabname.rst
# + .tabname.rst
# =====================================================================
root_tabdir = self.src_dir / "tabs"
root_fielddir = root_tabdir / "fields"
root_tabdir.mkdir(exist_ok=True)
root_fielddir.mkdir(exist_ok=True)
processed_tabs = set()
# -----------------------------------------------------------------
tab_header_level = 3
# update master field with chapter
with open(_master_file, "a") as fh:
fh.write(_write("Conventions", header=2))
p = "Each sheet Columns (or Fields) can be either:\n"
p += "* :orange:`<field name>`: Mandatory field\n"
p += "* :green:`<field name>`: Padded field\n"
p += "* <**field name**>: Optional field\n"
fh.write(_write(p))
if tabnames == "*":
# retrieve remaining tabs
tabnames = set(self.schemas.keys()) - processed_tabs
to_process_tabs = set(tabnames) - set(skip_tabs)
for tab_no, tab in enumerate(tabnames):
processed_tabs.add(tab)
if tab in skip_tabs:
if tab not in to_process_tabs:
continue
to_process_tabs.discard(tab)
schema = self.schemas[tab]
# -----------------------------------------------------------------
# create public tab description file `source/tabs/<tab>.rst`
......@@ -251,28 +276,49 @@ class VDataAutodoc:
# update private master file
with open(_master_file, "a") as fh:
fh.write(_write(f'Sheet "``{tab}``"', header=tab_header_level))
# fh.write(_write(f"Specifications", header=tab_header_level + 1))
# include public tab description file `source/tabs/<tab>.rst`
fh.write(_include(filename, relative_to=self.src_dir))
fh.write(
_include(root_tabdir / f".{tab}.rst", relative_to=self.src_dir)
)
if tab_no < len(tabnames) - 1:
fh.write(_write("\n-------------------------\n"))
# -----------------------------------------------------------------
# create private tab description file `source/tabs/.<tab>.rst`
_filename = root_tabdir / f".{tab}.rst"
with open(_filename, "w") as fh:
fh.write(
_write(
f'"``{tab}``" Columns Specifications',
header=tab_header_level + 1,
)
)
# -------------------------------------------------------------
# fh.write(
# _write(
# f'"``{tab}``" Columns Description',
# header=tab_header_level + 1,
# )
# )
# =================================================================
# columns description
# =================================================================
columns = {k: v["items"] for k, v in schema.columns_specs.items()}
for colname, colspecs in columns.items():
col_status = field_status(colname, schema)
if col_status == "padded":
fh.write(_write(f"\n{tab}/:green:`{colname}`"))
elif col_status == "mandatory":
fh.write(_write(f"\n{tab}/:orange:`{colname}`"))
else:
fh.write(_write(f"\n{tab}/**{colname}**"))
if "anyOf" in colspecs:
anyOf = [d["type"] for d in colspecs.pop("anyOf")]
colspecs["type"] = ", ".join(anyOf)
avoid = ("example",)
if "default" in colspecs and colspecs["default"] == "":
colspecs["default"] = '""'
# if colname == "repeat_discard_nth":
# breakpoint()
colspecs = {k: v for k, v in colspecs.items() if k not in avoid}
definition = (
" | ".join([f"{k}: ``{v}``" for k, v in colspecs.items()])
+ "\n"
)
field_public_file = root_fielddir / f"{tab}::{colname}.rst"
_new_public_file(field_public_file, clue_message=False)
definition_include = _include(
field_public_file,
relative_to=self.src_dir,
)
fh.write(_write(definition + definition_include, indent=1))
# ---------------------------------------------------------
# columns description summary
df = pd.DataFrame.from_dict(columns, orient="index")
# -----------------------------------------------------------------
# make anyOf more readable
......@@ -371,6 +417,8 @@ class VDataAutodoc:
msg += _write(table, indent=1)
fh.write(_directive(name="note", content=msg))
if len(to_process_tabs) > 0:
fh.write(_write("\n-------------------------\n"))
# =============================================================================
......@@ -447,22 +495,19 @@ def dump_xlsx_template(schemas, output, tabs=(), excluded_tabs=None):
# worksheet.set_row(0, height=None, cell_format=mandatory_format)
ws_has_units = {cs.get("units") for _, cs in colspecs.items()} > {None}
for i, colname in enumerate(df.columns):
is_mandatory = colname in required
colname_status = field_status(colname, schemas[tabname])
if colname_status == "default":
fmt = default_format
elif colname_status == "padded":
fmt = padded_format
elif colname_status == "mandatory":
fmt = mandatory_format
else:
raise ValueError(f"dont know anythin about {colname_status=}")
# is_mandatory = colname in required
pattern = colspecs[colname]["items"].get("pattern")
if pattern and not re.match(pattern, ""):
is_mandatory = True
default = colspecs[colname]["items"].get("default")
is_padded = False
if default == "_pad_":
is_mandatory = False
is_padded = True
xcalling = xcallings.get(colname)
if is_mandatory or xcalling:
fmt = mandatory_format
elif is_padded:
fmt = padded_format
else:
fmt = default_format
worksheet.write_string(0, i, colname, cell_format=fmt)
cell_header = "{col}{row}".format(col=string.ascii_uppercase[i], row=1)
# -------------------------------------------------------------
......@@ -504,6 +549,27 @@ def dump_xlsx_template(schemas, output, tabs=(), excluded_tabs=None):
return output
def field_status(colname, schema):
"""return one of {'mandatory', 'padded', 'default'"""
is_mandatory = colname in schema.required
pattern = schema.columns_specs[colname]["items"].get("pattern")
if pattern and not re.match(pattern, ""):
is_mandatory = True
default = schema.columns_specs[colname]["items"].get("default")
is_padded = False
if default == "_pad_":
is_mandatory = False
is_padded = True
# xcalling = schema.xcalling.get(colname)
if is_mandatory: # or xcalling:
status = "mandatory"
elif is_padded:
status = "padded"
else:
status = "default"
return status
if __name__ == "__main__":
import doctest
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment