Unverified Commit cce62f53 authored by Jeremy Tuloup's avatar Jeremy Tuloup Committed by GitHub
Browse files

Merge pull request #69 from bollwyvl/add-typedoc

Add TypeScript API documentation with typedoc
parents 8fe50e56 bb122e55
......@@ -10,6 +10,8 @@ dependencies:
- jupyterlab >=3,<4
- jupyterlab-classic
- git
# extra docs tools
- sphinx-autobuild
### DOCS ENV ###
# runtimes
......@@ -24,5 +26,6 @@ dependencies:
# docs
- myst-nb
- pydata-sphinx-theme
- pytest-check-links
- sphinx
### DOCS ENV ###
......@@ -7,7 +7,7 @@ on:
branches: '*'
env:
CACHE_EPOCH: 0
CACHE_EPOCH: 1
jobs:
build:
......@@ -39,17 +39,19 @@ jobs:
${{ env.CACHE_EPOCH }}-yarn-packages-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ env.CACHE_EPOCH }}-yarn-packages-
- name: Setup pip (pip)
run: |
pip install -U pip setuptools wheel
- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: |
${{ env.CACHE_EPOCH }}-${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
${{ env.CACHE_EPOCH }}-${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }}
restore-keys: |
${{ env.CACHE_EPOCH }}-${{ runner.os }}-pip-
- name: Setup
- name: Setup pip (dependecies)
run: |
pip install -U pip setuptools
pip install -r requirements.txt
pip install -r requirements-docs.txt
- name: Install
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: |
......@@ -68,8 +70,15 @@ jobs:
- name: Test
run: |
doit test
- name: Docs
run: |
doit docs
- name: Check Built Artifacts
run: |
doit check
deploy:
if: github.ref == 'refs/heads/main'
needs: [build]
runs-on: ubuntu-latest
steps:
......
......@@ -127,4 +127,6 @@ app/kernels
.doit*
# generated
docs/api/ts
.vercel
......@@ -5,3 +5,5 @@
build
docs/_build
node_modules
docs/api/ts
.pytest_cache
# API
```{toctree}
:caption: TypeScript API
:maxdepth: 2
ts/index.md
```
......@@ -27,6 +27,8 @@ extensions = [
]
autosectionlabel_prefix_document = True
myst_heading_anchors = 3
suppress_warnings = ["autosectionlabel.*"]
# files
templates_path = ["_templates"]
......@@ -69,4 +71,4 @@ html_context = {
if os.environ.get("READTHEDOCS"):
import subprocess
subprocess.check_call(["doit", "build"], cwd=str(ROOT))
subprocess.check_call(["doit", "build", "docs:typedoc:mystify"], cwd=str(ROOT))
......@@ -19,5 +19,6 @@ dependencies:
# docs
- myst-nb
- pydata-sphinx-theme
- pytest-check-links
- sphinx
### DOCS ENV ###
......@@ -8,6 +8,7 @@
:maxdepth: 1
kernels/index
api/index
contributing
changelog
```
import json
import os
import re
import shutil
from pathlib import Path
import doit
from collections import defaultdict
def task_env():
......@@ -21,12 +24,21 @@ def task_setup():
args = ["yarn", "--prefer-offline", "--ignore-optional"]
if C.CI:
# .yarn-integrity will only exist on a full cache hit vs yarn.lock, saves 1min+
if B.YARN_INTEGRITY.exists():
return
args += ["--frozen-lockfile"]
yield dict(
name="js",
doc="install node packages",
file_dep=[P.YARN_LOCK, *P.PACKAGE_JSONS, P.ROOT_PACKAGE_JSON],
file_dep=[
P.YARN_LOCK,
*P.PACKAGE_JSONS,
P.ROOT_PACKAGE_JSON,
P.APP_PACKAGE_JSON,
*P.APP_JSONS,
],
actions=[U.do(*args)],
targets=[B.YARN_INTEGRITY],
)
......@@ -34,6 +46,8 @@ def task_setup():
def task_lint():
"""format and ensure style of code, docs, etc."""
if C.RTD:
return
yield U.ok(
B.OK_PRETTIER,
......@@ -47,7 +61,7 @@ def task_lint():
B.OK_ESLINT,
name="eslint",
doc="format and verify .ts, .js files with eslint",
file_dep=[B.OK_PRETTIER],
file_dep=[B.OK_PRETTIER, *L.ALL_ESLINT],
actions=[U.do("yarn", "eslint:check" if C.CI else "eslint")],
)
......@@ -65,7 +79,12 @@ def task_build():
yield dict(
name="js:lib",
doc="build .ts files into .js files",
file_dep=[*L.ALL_TS, P.ROOT_PACKAGE_JSON, *P.PACKAGE_JSONS, B.YARN_INTEGRITY],
file_dep=[
*L.ALL_ESLINT,
P.ROOT_PACKAGE_JSON,
*P.PACKAGE_JSONS,
B.YARN_INTEGRITY,
],
actions=[
U.do("yarn", "build:lib"),
],
......@@ -124,15 +143,56 @@ def task_build():
def task_docs():
"""build documentation"""
yield dict(
name="typedoc:ensure",
file_dep=[*P.PACKAGE_JSONS],
actions=[U.typedoc_conf],
targets=[P.TYPEDOC_JSON, P.TSCONFIG_TYPEDOC],
)
yield dict(
name="typedoc:build",
doc="build the TS API documentation with typedoc",
file_dep=[B.META_BUILDINFO, *P.TYPEDOC_CONF],
actions=[U.do("yarn", "docs")],
targets=[B.DOCS_RAW_TYPEDOC_README],
)
yield dict(
name="typedoc:mystify",
doc="transform raw typedoc into myst markdown",
file_dep=[B.DOCS_RAW_TYPEDOC_README],
targets=[B.DOCS_TS_MYST_INDEX, *B.DOCS_TS_MODULES],
actions=[U.mystify, U.do("yarn", "prettier")],
)
yield dict(
name="sphinx",
doc="build the documentation site with sphinx",
file_dep=[*P.DOCS_MD, *P.DOCS_PY, B.APP_PACK],
file_dep=[B.DOCS_TS_MYST_INDEX, *P.DOCS_MD, *P.DOCS_PY, B.APP_PACK],
actions=[U.do("sphinx-build", "-b", "html", P.DOCS, B.DOCS)],
targets=[B.DOCS_BUILDINFO],
)
@doit.create_after("docs")
def task_check():
"""perform checks of built artifacts"""
yield dict(
name="docs:links",
doc="check for broken (internal) links",
file_dep=[*B.DOCS.rglob("*.html")],
actions=[
U.do(
"pytest-check-links",
B.DOCS,
"--check-anchors",
"--check-links-ignore",
"^https?://",
)
],
)
def task_watch():
"""watch sources and rebuild on change"""
yield dict(
......@@ -142,13 +202,14 @@ def task_watch():
file_dep=[B.YARN_INTEGRITY],
actions=[U.do("yarn", "watch")],
)
yield dict(
name="docs",
doc="watch .md sources and rebuild the documentation",
uptodate=[lambda: False],
file_dep=[*P.DOCS_MD, *P.DOCS_PY, B.APP_PACK],
actions=[U.do("sphinx-autobuild", P.DOCS, B.DOCS)],
)
if shutil.which("sphinx-autobuild"):
yield dict(
name="docs",
doc="watch .md sources and rebuild the documentation",
uptodate=[lambda: False],
file_dep=[*P.DOCS_MD, *P.DOCS_PY, B.APP_PACK],
actions=[U.do("sphinx-autobuild", P.DOCS, B.DOCS)],
)
def task_test():
......@@ -167,14 +228,16 @@ class C:
APPS = ["classic", "lab"]
ENC = dict(encoding="utf-8")
CI = bool(json.loads(os.environ.get("CI", "0")))
RTD = bool(json.loads(os.environ.get("READTHEDOCS", "False").lower()))
DOCS_ENV_MARKER = "### DOCS ENV ###"
NO_TYPEDOC = ["_metapackage"]
class P:
DODO = Path(__file__)
ROOT = DODO.parent
PACKAGES = ROOT / "packages"
PACKAGE_JSONS = [*PACKAGES.glob("*/package.json")]
PACKAGE_JSONS = sorted(PACKAGES.glob("*/package.json"))
ROOT_PACKAGE_JSON = ROOT / "package.json"
YARN_LOCK = ROOT / "yarn.lock"
......@@ -184,7 +247,7 @@ class P:
APP = ROOT / "app"
APP_PACKAGE_JSON = APP / "package.json"
WEBPACK_CONFIG = APP / "webpack.config.js"
APP_JSONS = [*APP.glob("*/package.json")]
APP_JSONS = sorted(APP.glob("*/package.json"))
APP_NPM_IGNORE = APP / ".npmignore"
# docs
......@@ -192,9 +255,15 @@ class P:
CONTRIBUTING = ROOT / "CONTRIBUTING.md"
CHANGELOG = ROOT / "CHANGELOG.md"
DOCS = ROOT / "docs"
TSCONFIG_TYPEDOC = ROOT / "tsconfig.typedoc.json"
TYPEDOC_JSON = ROOT / "typedoc.json"
TYPEDOC_CONF = [TSCONFIG_TYPEDOC, TYPEDOC_JSON]
DOCS_SRC_MD = sorted(
[p for p in DOCS.rglob("*.md") if "docs/api/ts" not in str(p.as_posix())]
)
DOCS_ENV = DOCS / "environment.yml"
DOCS_PY = [*DOCS.rglob("*.py")]
DOCS_MD = [*DOCS.rglob("*.md"), README, CONTRIBUTING, CHANGELOG]
DOCS_PY = sorted([*DOCS.rglob("*.py")])
DOCS_MD = sorted([*DOCS_SRC_MD, README, CONTRIBUTING, CHANGELOG])
# demo
BINDER = ROOT / ".binder"
......@@ -221,11 +290,16 @@ P.PYOLITE_PACKAGES = [
class L:
# linting
ALL_TS = [*P.PACKAGES.rglob("*/src/**/*.js"), *P.PACKAGES.rglob("*/src/**/*.ts")]
ALL_JSON = [*P.PACKAGE_JSONS, *P.APP_JSONS, P.ROOT_PACKAGE_JSON, *ALL_TS]
ALL_ESLINT = [
*P.PACKAGES.rglob("*/src/**/*.js"),
*P.PACKAGES.rglob("*/src/**/*.ts"),
]
ALL_JSON = set(
[*P.PACKAGE_JSONS, *P.APP_JSONS, P.ROOT_PACKAGE_JSON, *P.ROOT.glob("*.json")]
)
ALL_MD = [*P.CI.rglob("*.md"), *P.DOCS_MD]
ALL_YAML = [*P.ROOT.glob("*.yml"), *P.BINDER.glob("*.yml"), *P.CI.rglob("*.yml")]
ALL_PRETTIER = [*ALL_JSON, *ALL_MD, *ALL_YAML]
ALL_PRETTIER = [*ALL_JSON, *ALL_MD, *ALL_YAML, *ALL_ESLINT]
ALL_BLACK = [
*P.DOCS_PY,
P.DODO,
......@@ -244,8 +318,18 @@ class B:
DIST = P.ROOT / "dist"
APP_PACK = DIST / f"""jupyterlite-app-{D.APP["version"]}.tgz"""
DOCS = P.DOCS / "_build"
DOCS_HTML = DOCS / "html"
DOCS_BUILDINFO = DOCS_HTML / ".buildinfo"
DOCS_BUILDINFO = DOCS / ".buildinfo"
# typedoc
DOCS_RAW_TYPEDOC = BUILD / "typedoc"
DOCS_RAW_TYPEDOC_README = DOCS_RAW_TYPEDOC / "README.md"
DOCS_TS = P.DOCS / "api/ts"
DOCS_TS_MYST_INDEX = DOCS_TS / "index.md"
DOCS_TS_MODULES = [
P.ROOT / "docs/api/ts" / f"{p.parent.name}.md"
for p in P.PACKAGE_JSONS
if p.parent.name not in C.NO_TYPEDOC
]
OK = BUILD / "ok"
OK_PRETTIER = OK / "prettier"
......@@ -258,7 +342,14 @@ class U:
@staticmethod
def do(*args, cwd=P.ROOT, **kwargs):
"""wrap a CmdAction for consistency"""
return doit.tools.CmdAction(list(args), shell=False, cwd=str(Path(cwd)))
cmd = args[0]
cmd = Path(
shutil.which(cmd)
or shutil.which(f"{cmd}.exe")
or shutil.which(f"{cmd}.cmd")
or shutil.which(f"{cmd}.bat")
).resolve()
return doit.tools.CmdAction([cmd, *args[1:]], shell=False, cwd=str(Path(cwd)))
@staticmethod
def ok(ok, **task):
......@@ -280,6 +371,121 @@ class U:
**C.ENC,
)
@staticmethod
def typedoc_conf():
typedoc = json.loads(P.TYPEDOC_JSON.read_text(**C.ENC))
original_entry_points = sorted(typedoc["entryPoints"])
new_entry_points = sorted(
[
str((p.parent / "src/index.ts").relative_to(P.ROOT).as_posix())
for p in P.PACKAGE_JSONS
if p.parent.name not in C.NO_TYPEDOC
]
)
if json.dumps(original_entry_points) != json.dumps(new_entry_points):
typedoc["entryPoints"] = new_entry_points
P.TYPEDOC_JSON.write_text(
json.dumps(typedoc, indent=2, sort_keys=True), **C.ENC
)
tsconfig = json.loads(P.TSCONFIG_TYPEDOC.read_text(**C.ENC))
original_references = tsconfig["references"]
new_references = [
{"path": f"./packages/{p.parent.name}"}
for p in P.PACKAGE_JSONS
if p.parent.name not in C.NO_TYPEDOC
]
if json.dumps(original_references) != json.dumps(new_references):
tsconfig["references"] = new_references
P.TSCONFIG_TYPEDOC.write_text(
json.dumps(tsconfig, indent=2, sort_keys=True), **C.ENC
)
@staticmethod
def mystify():
"""unwrap monorepo docs into per-module docs"""
mods = defaultdict(lambda: defaultdict(list))
if B.DOCS_TS.exists():
shutil.rmtree(B.DOCS_TS)
def mod_md_name(mod):
return mod.replace("@jupyterlite/", "") + ".md"
for doc in sorted(B.DOCS_RAW_TYPEDOC.rglob("*.md")):
if doc.parent == B.DOCS_RAW_TYPEDOC:
continue
if doc.name == "README.md":
continue
doc_text = doc.read_text(**C.ENC)
doc_lines = doc_text.splitlines()
mod_chunks = doc_lines[0].split(" / ")
src = mod_chunks[1]
if src.startswith("["):
src = re.findall(r"\[(.*)/src\]", src)[0]
else:
src = src.replace("/src", "")
pkg = f"""@jupyterlite/{src.replace("/src", "")}"""
mods[pkg][doc.parent.name] += [
str(doc.relative_to(B.DOCS_RAW_TYPEDOC).as_posix())[:-3]
]
# rewrite doc and write back out
out_doc = B.DOCS_TS / doc.relative_to(B.DOCS_RAW_TYPEDOC)
if not out_doc.parent.exists():
out_doc.parent.mkdir(parents=True)
out_text = "\n".join([*doc_lines[1:], ""]).replace("README.md", "index.md")
out_text = re.sub(
r"## Table of contents(.*?)\n## ",
"\n## ",
out_text,
flags=re.M | re.S,
)
out_text = out_text.replace("/src]", "]")
out_text = re.sub("/src$", "", out_text, flags=re.M)
out_text = re.sub(
r"^Defined in: ([^\n]+)$",
"_Defined in:_ `\\1`",
out_text,
flags=re.M | re.S,
)
out_text = re.sub(
r"^((Implementation of|Overrides|Inherited from):)",
"_\\1_",
out_text,
flags=re.M | re.S,
)
out_doc.write_text(out_text, **C.ENC)
for mod, sections in mods.items():
out_doc = B.DOCS_TS / mod_md_name(mod)
mod_lines = [f"""# `{mod.replace("@jupyterlite/", "")}`\n"""]
for label, contents in sections.items():
mod_lines += [
f"## {label.title()}\n",
"```{toctree}",
":maxdepth: 1",
*contents,
"```\n",
]
out_doc.write_text("\n".join(mod_lines))
B.DOCS_TS_MYST_INDEX.write_text(
"\n".join(
[
"# `@jupyterlite`\n",
"```{toctree}",
":maxdepth: 1",
*[mod_md_name(mod) for mod in sorted(mods)],
"```",
]
),
**C.ENC,
)
# environment overloads
os.environ.update(
......
......@@ -40,7 +40,8 @@
"watch:lib": "lerna exec --stream --scope @jupyterlite/metapackage yarn watch",
"watch:app": "lerna exec --stream --stream --parallel --scope \"@jupyterlite/app-classic\" --scope \"@jupyterlite/app-lab\" yarn watch",
"serve": "node scripts/serve.js",
"serve:py": "cd app && python -m http.server -b 127.0.0.1"
"serve:py": "cd app && python -m http.server -b 127.0.0.1",
"docs": "typedoc"
},
"husky": {
"hooks": {
......@@ -68,6 +69,8 @@
"prettier": "^1.19.0",
"rimraf": "^3.0.2",
"shell-quote": "^1.7.2",
"typedoc": "^0.20.36",
"typedoc-plugin-markdown": "^3.7.2",
"typescript": "~4.2.3"
}
}
/**
* Store the kernel and interpreter instances.
*/
// eslint-disable-next-line
// @ts-ignore: breaks typedoc
let kernel: any;
// eslint-disable-next-line
// @ts-ignore: breaks typedoc
let interpreter: any;
/**
......@@ -50,6 +54,8 @@ function formatResult(res: any): any {
return results;
}
// eslint-disable-next-line
// @ts-ignore: breaks typedoc
const pyodideReadyPromise = loadPyodideAndPackages();
self.onmessage = async (event: MessageEvent): Promise<void> => {
......
# minimum needed to build jupyterlite docs... in addition to nodejs and yarn
# see .binder/ and docs/ for full development/docs environments
-r requirements.txt
myst-nb
pydata-sphinx-theme
pytest-check-links
sphinx
{
"$schema": "http://json.schemastore.org/tsconfig",
"compilerOptions": {
"lib": ["ES2019", "WebWorker", "ScriptHost", "DOM", "DOM.Iterable"],
"paths": {
"@jupyterlite/*": ["./packages/*/src"]
}
},
"exclude": ["**/test/**"],
"extends": "./tsconfigbase",
"references": [
{
"path": "./packages/application"
},
{
"path": "./packages/application-extension"
},
{
"path": "./packages/classic-application-extension"
},
{
"path": "./packages/contents"
},
{
"path": "./packages/iframe-extension"
},
{
"path": "./packages/javascript-kernel"
},
{
"path": "./packages/javascript-kernel-extension"
},
{
"path": "./packages/kernel"
},
{
"path": "./packages/p5-kernel"
},
{
"path": "./packages/p5-kernel-extension"
},
{
"path": "./packages/pyolite-kernel"
},
{
"path": "./packages/pyolite-kernel-extension"
},
{
"path": "./packages/server"
},
{
"path": "./packages/server-extension"
},
{
"path": "./packages/session"
},
{
"path": "./packages/settings"
},
{
"path": "./packages/theme"
},
{
"path": "./packages/theme-extension"
}
]
}
{
"entryPoints": [
"packages/application/src/index.ts",
"packages/application-extension/src/index.ts",
"packages/classic-application-extension/src/index.ts",
"packages/contents/src/index.ts",
"packages/iframe-extension/src/index.ts",
"packages/javascript-kernel/src/index.ts",
"packages/javascript-kernel-extension/src/index.ts",
"packages/kernel/src/index.ts",
"packages/p5-kernel/src/index.ts",
"packages/p5-kernel-extension/src/index.ts",
"packages/pyolite-kernel/src/index.ts",
"packages/pyolite-kernel-extension/src/index.ts",
"packages/server/src/index.ts",
"packages/server-extension/src/index.ts",
"packages/session/src/index.ts",
"packages/settings/src/index.ts",
"packages/theme/src/index.ts",
"packages/theme-extension/src/index.ts"
],
"exclude": ["*.spec.ts", "**/node_modules/**", "**/test/**", "**/lib/**"],