diff --git a/.gitignore b/.gitignore index ec7ba825d..caea1ea78 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,8 @@ __pycache__/ # Screenshot injection output (staged from screenshots/captured/ at build time) content/images/generated/ + +# Generated by the merge (scripts/merge_docs.py concat_into): mto's own half of +# the page is content/reference/api.src.md, this is that plus every +# sub-operator's API reference, rebuilt on each merge. +content/reference/api.md diff --git a/.vale.ini b/.vale.ini index 37811a9da..04590d220 100644 --- a/.vale.ini +++ b/.vale.ini @@ -13,11 +13,11 @@ BasedOnStyles = Vale # reach the reader. The name is a filename, so spelling and term rules don't apply. TokenIgnores = (\{\{\s*screenshot:[^}]*\}\}) -# content/reference/api.md is machine-generated from the operator's Go types +# content/reference/api.src.md is machine-generated from the operator's Go types # (via crd-ref-docs). It contains API identifiers and verbatim code comments # (e.g. the `tenantoperator.stakater.com` group, the `LabelSelector` type, # camelCase field names), not prose, so prose style rules don't apply. -[content/reference/api.md] +[content/reference/api.src.md] BasedOnStyles = Vale.Terms = NO Vale.Spelling = NO diff --git a/Makefile b/Makefile index b7f8dc6a3..f6e03a24d 100644 --- a/Makefile +++ b/Makefile @@ -66,9 +66,13 @@ serve: theme ## Full local preview: clones the sub-operator repos, then mkdocs s $(PY) -m mkdocs serve serve-local: theme merge-local ## Same preview from local checkouts, no cloning - $(PY) -m mkdocs serve + $(PY) -m mkdocs serve -a localhost:9000 clean: ## Remove fetched repos and generated artifacts (surgical; never `git clean`) rm -rf .suboperators mkdocs.yml dist site + @python3 -c "import sys;sys.path.insert(0,'scripts');import merge_docs;\ + print('\n'.join(sorted({m['concat_into'] for o in merge_docs.load_config('merge.yaml') \ + for m in o['mappings'] if m.get('concat_into')})))" 2>/dev/null \ + | while read -r f; do [ -n "$$f" ] && rm -f "content/$$f"; done @python3 -c "import sys;sys.path.insert(0,'scripts');import merge_docs;[print(o['slug']) for o in merge_docs.load_config('merge.yaml')]" 2>/dev/null \ | while read -r s; do [ -n "$$s" ] && find content -type d -name "$$s" -prune -exec rm -rf {} + 2>/dev/null || true; done diff --git a/content/reference/api.md b/content/reference/api.src.md similarity index 100% rename from content/reference/api.md rename to content/reference/api.src.md diff --git a/merge.yaml b/merge.yaml index 03abc932c..84f6ea769 100644 --- a/merge.yaml +++ b/merge.yaml @@ -24,7 +24,9 @@ operators: - from: "guides/**" into: "guides" under: "Guides" - - from: "reference/api.md" # concatenated into the single API page + # concatenated onto mto's own reference/api.src.md; the result, + # reference/api.md, is generated and gitignored + - from: "reference/api.md" concat_into: "reference/api.md" under: "Reference" as: "API Reference" diff --git a/screenshots/captured/create-interval.png b/screenshots/captured/create-interval.png index 1b9720b19..e77d5866a 100644 Binary files a/screenshots/captured/create-interval.png and b/screenshots/captured/create-interval.png differ diff --git a/screenshots/captured/cti-yaml.png b/screenshots/captured/cti-yaml.png index 55f810883..20ae59515 100644 Binary files a/screenshots/captured/cti-yaml.png and b/screenshots/captured/cti-yaml.png differ diff --git a/screenshots/captured/integration-config.png b/screenshots/captured/integration-config.png index 1acf7720f..0646cfe6a 100644 Binary files a/screenshots/captured/integration-config.png and b/screenshots/captured/integration-config.png differ diff --git a/screenshots/captured/namespaces.png b/screenshots/captured/namespaces.png index 7b730f376..55f8b2cc1 100644 Binary files a/screenshots/captured/namespaces.png and b/screenshots/captured/namespaces.png differ diff --git a/screenshots/captured/quotas.png b/screenshots/captured/quotas.png index ea407bb79..023fce2d6 100644 Binary files a/screenshots/captured/quotas.png and b/screenshots/captured/quotas.png differ diff --git a/screenshots/captured/showback.png b/screenshots/captured/showback.png index eb8326071..98f9dff1b 100644 Binary files a/screenshots/captured/showback.png and b/screenshots/captured/showback.png differ diff --git a/screenshots/captured/template-instance-yaml-view.png b/screenshots/captured/template-instance-yaml-view.png index 8fb61d318..171fdc826 100644 Binary files a/screenshots/captured/template-instance-yaml-view.png and b/screenshots/captured/template-instance-yaml-view.png differ diff --git a/screenshots/captured/tenant-utilization-namespace-stats.png b/screenshots/captured/tenant-utilization-namespace-stats.png index 29903f530..cfdb4f020 100644 Binary files a/screenshots/captured/tenant-utilization-namespace-stats.png and b/screenshots/captured/tenant-utilization-namespace-stats.png differ diff --git a/screenshots/captured/tenant-utilization-namespaces.png b/screenshots/captured/tenant-utilization-namespaces.png index fb05756d8..81eda1075 100644 Binary files a/screenshots/captured/tenant-utilization-namespaces.png and b/screenshots/captured/tenant-utilization-namespaces.png differ diff --git a/scripts/merge_docs.py b/scripts/merge_docs.py index 89ba467a6..5fd56a046 100644 --- a/scripts/merge_docs.py +++ b/scripts/merge_docs.py @@ -12,6 +12,8 @@ import yaml from pathlib import Path +import reorder_api_reference + _WILDCARD = set("*?[]") @@ -315,28 +317,57 @@ def apply_group(nav, group): section[:] = new_section +def reshape_api_page(text): + """Kinds above their own types, no `Resource Types` wrapper — applied to + every source of a concatenated page, mto-docs' own and each sub-operator's, + so the one page reads the same the whole way down. Sub-operator repos have + no such step of their own, and their published sites keep whatever order + crd-ref-docs gave them; this only shapes the copy merged in here. A page + that is not crd-ref-docs output has no group-version headings and comes back + unchanged.""" + return reorder_api_reference.reorder_text(text) + + +def concat_source(content_dir, into): + """Where mto-docs' own half of a concatenated page lives: `api.md` is built + from `api.src.md`. The two must stay separate — `into` is overwritten on + every merge, so reading mto's own content back out of it would demote and + re-wrap the whole page one level deeper each run.""" + path = Path(content_dir) / into + return path.with_name(f"{path.stem}.src{path.suffix}") + + def apply_concat(nav, targets, content_dir, op_ctx, site_title): - """Concatenate several source pages into one per `into` target. The existing - content page (mto-docs' own) is the first `## site_title` section; each - contributing operator adds a `## heading` section sourced from its clone and + """Concatenate several source pages into one per `into` target. mto-docs' + own `.src.md` is the first `## site_title` section; each contributing + operator adds a `## heading` section sourced from its clone and link-rewritten. Every source's own headings are demoted one level so the - in-page TOC lists the sections. One `{as: into}` leaf is set under `under`.""" + in-page TOC lists the sections. One `{as: into}` leaf is set under `under`. + `into` itself is pure output: regenerated from scratch here, gitignored.""" content_dir = Path(content_dir) for into, spec in targets.items(): blocks, page_title = [], None self_path = content_dir / into - if self_path.is_file(): - page_title, body = split_h1(self_path.read_text(encoding="utf-8")) + src_path = concat_source(content_dir, into) + if src_path.is_file(): + raw = reshape_api_page(src_path.read_text(encoding="utf-8")) + page_title, body = split_h1(raw) blocks.append(f"## {site_title}\n\n{shift_headings(body, 1).strip()}\n") + elif self_path.is_file(): + raise ValueError( + f"concat target {into!r} exists but {src_path.name!r} does not: " + f"mto-docs' own half of the page belongs in {src_path.name!r} " + f"({into} is generated). Rename it, or delete it if it is " + f"output from an earlier merge.") for heading, op_title, frm in spec["sections"]: ctx = op_ctx[op_title] raw = (ctx["docs"] / frm).read_text(encoding="utf-8") raw, _ = rewrite_links(raw, frm, into, ctx["op_map"], ctx["live_url"], ctx["style"]) - body = shift_headings(split_h1(raw)[1], 1).strip() + body = shift_headings(split_h1(reshape_api_page(raw))[1], 1).strip() blocks.append(f"## {heading}\n\n{body}\n") if page_title is None: - page_title = read_h1(self_path) or prettify(Path(into).stem) + page_title = read_h1(src_path) or prettify(Path(into).stem) self_path.parent.mkdir(parents=True, exist_ok=True) self_path.write_text(f"# {page_title}\n\n" + "\n".join(blocks), encoding="utf-8") _set_single_leaf(nav, spec["under"], into, spec.get("as")) diff --git a/scripts/reorder_api_reference.py b/scripts/reorder_api_reference.py new file mode 100644 index 000000000..effd70fe0 --- /dev/null +++ b/scripts/reorder_api_reference.py @@ -0,0 +1,263 @@ +"""Reshape the type sections of a crd-ref-docs markdown page. + +Two things, both of which the generator has no knob for: + +- **Order.** crd-ref-docs renders every type of a group version alphabetically + (`GroupVersionDetails.SortedTypes`), so `Tenant` ends up buried after + `AccessControl`. Instead the types are walked as the tree they are: depth + first from each kind, in field order, so the types a section references sit + directly below it — scroll past `TenantSpec` and you are in the types + `TenantSpec` is made of. Kinds are taken one at a time, so their subtrees do + not interleave, and anything no kind reaches is left alphabetical at the end. +- **Depth.** A kind's heading sits directly under its package; every type that + something else references is one step smaller, however deep in the tree it is, + so a package reads as a list of kinds with their parts under them. + The `Resource Types` heading and its list of kinds are dropped along the way: + it was the only thing between a package and its types, and with the kinds + ordered first it said nothing the page does not. + +Run it after generation, alongside the fixups in +docs/generating-api-reference.md: + + python scripts/reorder_api_reference.py content/reference/api.md + +Section bodies are moved verbatim (only their heading level and trailing blank +lines change), and re-running the script changes nothing. +""" +import argparse +import re +import sys +from pathlib import Path + +# A group-version heading, e.g. "tenantoperator.stakater.com/v1beta3". Type +# names never contain a slash, so this cannot match a type section. +GV_RE = re.compile(r"^[A-Za-z0-9.\-]+/[A-Za-z0-9.\-]+$") +HEADING_RE = re.compile(r"(#{1,6})\s+(.*\S)\s*$") +KIND_BULLET_RE = re.compile(r"^-\s+\[([A-Za-z0-9_]+)\]") +KIND_ROW_RE = re.compile(r"\|\s*`kind`\s+_string_\s*\|") +ANCHOR_LINK_RE = re.compile(r"\]\(#([A-Za-z0-9_-]+)\)") +SUFFIXES = ("Spec", "Status") + + +def anchor(name): + """The in-page anchor crd-ref-docs links a type by.""" + return re.sub(r"[^a-z0-9]", "", name.lower()) + + +def code_flags(lines): + """One bool per line, True inside a ``` / ~~~ fence, so that a `#` in a + YAML sample is never mistaken for a heading. Mirrors merge_docs._md_lines; + kept local so this script can be dropped into a sub-operator docs repo.""" + fence, out = None, [] + for line in lines: + mo = re.match(r"(`{3,}|~{3,})", line.strip()) + marker = mo.group(1)[0] if mo else None + if fence: + out.append(True) + if marker == fence: + fence = None + elif marker: + fence = marker + out.append(True) + else: + out.append(False) + return out + + +def headings(lines, flags): + """(line index, level, title) for every ATX heading outside code fences.""" + out = [] + for i, line in enumerate(lines): + if flags[i]: + continue + mo = HEADING_RE.match(line) + if mo: + out.append((i, len(mo.group(1)), mo.group(2).strip())) + return out + + +def type_order(kinds, blocks, refs=None): + """The order type sections should appear in: one depth-first walk per kind, + following its fields in the order they are declared, so whatever a section + references sits directly below it. `Spec` and `Status` are + walked after the kind as a fallback, in case `ignoreFields` dropped the rows + that would have linked them. Types no kind reaches follow alphabetically. + With no kinds at all the existing order is kept.""" + refs = refs or {} + order, seen = [], set() + + def walk(name): + if name not in blocks or name in seen: + return + seen.add(name) + order.append(name) + for ref in refs.get(name, ()): + walk(ref) + + for kind in kinds: + walk(kind) + for suffix in SUFFIXES: + walk(kind + suffix) + rest = [name for name in blocks if name not in seen] + order.extend(sorted(rest) if kinds else rest) + return order + + +def references(lines, flags, spans): + """Forward references per type, in field order: the other types of this group + version that its field table links to, plus an alias's underlying type. + Only table rows are read, so the `Appears in:` bullet list — which points the + other way — never becomes an edge.""" + known = {anchor(name): name for name in spans} + out = {} + for name, (start, stop) in spans.items(): + refs, seen = [], {name} + for j in range(start, stop): + line = lines[j] + if flags[j]: + continue + if not (line.startswith("|") or "_Underlying type:_" in line): + continue + for target in ANCHOR_LINK_RE.findall(line): + ref = known.get(target) + if ref and ref not in seen: + seen.add(ref) + refs.append(ref) + out[name] = refs + return out + + +def resource_types_span(lines, flags, section, rt_level): + """The half-open line range one group version's `Resource Types` heading and + its list of kinds occupy, so the caller can drop the block.""" + for i, level, title in section: + if level != rt_level or title.lower() != "resource types": + continue + j, seen_bullet = i + 1, False + while j < len(lines): + line = lines[j].strip() + if flags[j] or line.startswith("#"): + break + if not line: + if seen_bullet: + break + j += 1 + continue + if not KIND_BULLET_RE.match(line): + break + seen_bullet = True + j += 1 + return i, j + return None + + +def kind_sections(lines, flags, spans): + """The type names that are kinds, in page order. crd-ref-docs gives a type + with a GVK an `apiVersion` and a `kind` row that nothing else has, so this + does not depend on the `Resource Types` list — which this script deletes, and + which therefore cannot be the thing that identifies them.""" + kinds = [] + for name, (start, stop) in sorted(spans.items(), key=lambda kv: kv[1][0]): + for j in range(start, stop): + if not flags[j] and KIND_ROW_RE.match(lines[j]): + kinds.append(name) + break + return kinds + + +def at_level(lines, flags, start, stop, level): + """One type section with its heading set to `level`, and any heading inside + it moved by the same amount. Kinds sit directly under their package; a type + something else references is a step smaller, whatever its depth in the tree. + Trailing blank lines are normalised to one, so that the section which + happened to sit last in the file does not end up flush against the next + heading once moved.""" + mo = HEADING_RE.match(lines[start]) + shift = level - len(mo.group(1)) if mo else 0 + out = [] + for j in range(start, stop): + line = lines[j] + if not flags[j] and shift: + mo = HEADING_RE.match(line) + if mo: + depth = min(max(len(mo.group(1)) + shift, 1), 6) + line = "#" * depth + line[len(mo.group(1)):] + out.append(line) + while out and not out[-1].strip(): + out.pop() + return out + [""] + + +def reorder_text(text): + lines = text.splitlines() + flags = code_flags(lines) + heads = headings(lines, flags) + + out, pos = [], 0 + for n, (start, level, title) in enumerate(heads): + if not GV_RE.match(title): + continue + end = len(lines) + for i, lvl, _ in heads[n + 1:]: + if lvl <= level: + end = i + break + section = [h for h in heads if start < h[0] < end] + # Every heading just below the package is a type section, at whichever + # level it currently sits: crd-ref-docs puts them all at level + 2 under + # `Resource Types`, and this script leaves kinds at level + 1 and the + # rest at level + 2, so both shapes have to be recognised. + types = [(i, t) for i, lvl, t in section + if level < lvl <= level + 2 and t.lower() != "resource types"] + if not types: + continue + + rt_span = resource_types_span(lines, flags, section, level + 1) + bounds = [i for i, _ in types] + [end] + spans = {t: (i, bounds[k + 1]) for k, (i, t) in enumerate(types)} + refs = references(lines, flags, spans) + kinds = set(kind_sections(lines, flags, spans)) + blocks = {t: at_level(lines, flags, *spans[t], + level + (1 if t in kinds else 2)) + for t in spans} + # the package doc, up to whatever the types used to be introduced by + preamble = lines[pos:rt_span[0] if rt_span else types[0][0]] + while preamble and not preamble[-1].strip(): + preamble.pop() + out.extend(preamble + [""]) + for name in type_order(kind_sections(lines, flags, spans), blocks, refs): + out.extend(blocks[name]) + pos = end + out.extend(lines[pos:]) + + result = "\n".join(out) + return result + "\n" if text.endswith("\n") else result + + +def main(argv=None): + ap = argparse.ArgumentParser( + description="Order kinds before their nested types in a crd-ref-docs " + "page, and drop the Resource Types wrapper.") + ap.add_argument("paths", nargs="+", type=Path, help="markdown files to reorder") + ap.add_argument("--check", action="store_true", + help="report files that would change, write nothing") + args = ap.parse_args(argv) + + changed = [] + for path in args.paths: + text = path.read_text(encoding="utf-8") + new = reorder_text(text) + if new == text: + print(f"{path}: already reshaped") + continue + changed.append(path) + if args.check: + print(f"{path}: would be reshaped") + else: + path.write_text(new, encoding="utf-8") + print(f"{path}: reshaped") + return 1 if args.check and changed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_merge_docs.py b/tests/test_merge_docs.py index 590046bb9..4f6fbfddc 100644 --- a/tests/test_merge_docs.py +++ b/tests/test_merge_docs.py @@ -909,8 +909,8 @@ def test_run_unlabelled_operator_folds_while_labelled_one_stays_grouped(tmp_path def test_apply_concat_builds_page_and_leaf(tmp_path): content = tmp_path / "content" (content / "reference").mkdir(parents=True) - (content / "reference/api.md").write_text( - "# API Reference\n\n## Packages\n\nmto crds\n") # mto's own (self) + (content / "reference/api.src.md").write_text( + "# API Reference\n\n## Packages\n\nmto crds\n") # mto's own half opdocs = tmp_path / "hib" / "content" (opdocs / "reference").mkdir(parents=True) (opdocs / "reference/api.md").write_text( @@ -933,6 +933,107 @@ def test_apply_concat_builds_page_and_leaf(tmp_path): assert "reference/rbac.md" in section # siblings kept +def test_apply_concat_reshapes_every_api_section(tmp_path): + """Both halves of the page go through reorder_api_reference: kinds above + their own types, no `Resource Types` wrapper, whoever generated them.""" + content = tmp_path / "content" + (content / "reference").mkdir(parents=True) + def api_page(group, kind, helper): + """A crd-ref-docs page: alphabetical, types under Resource Types, and + the apiVersion/kind rows that mark which type is the kind.""" + return (f"# API Reference\n\n## {group}\n\ndocs\n\n" + f"### Resource Types\n- [{kind}](#{kind.lower()})\n\n" + f"#### {helper}\n\nhelper\n\n" + f"#### {kind}\n\nkind\n\n" + "| Field | Description | Default | Validation |\n" + "| --- | --- | --- | --- |\n" + f"| `apiVersion` _string_ | `{group}` | | |\n" + f"| `kind` _string_ | `{kind}` | | |\n") + + (content / "reference/api.src.md").write_text( + api_page("mto.io/v1", "Tenant", "AccessControl")) + opdocs = tmp_path / "hib" / "content" + (opdocs / "reference").mkdir(parents=True) + (opdocs / "reference/api.md").write_text( + api_page("hib.io/v1", "Supervisor", "Helper")) + op_ctx = {"Hib": {"docs": opdocs, "op_map": {}, "live_url": None, + "style": "directory"}} + targets = {"reference/api.md": { + "under": "Reference", "as": "API Reference", + "sections": [("Hibernation Operator", "Hib", "reference/api.md")]}} + m.apply_concat([{"Reference": ["reference/api.md"]}], targets, str(content), + op_ctx, "Tenant Operator") + + page = (content / "reference/api.md").read_text() + assert "Resource Types" not in page + assert page.index("#### Tenant") < page.index("##### AccessControl") + assert page.index("#### Supervisor") < page.index("##### Helper") + # the concat demotes by one, so a package is 3, its kinds 4, their types 5 + assert "### mto.io/v1" in page + assert "###### " not in page + + +def test_apply_concat_is_idempotent(tmp_path): + """The bug this split exists to prevent: merging twice used to read the + generated page back in, demoting it and adding another site_title wrapper.""" + content = tmp_path / "content" + (content / "reference").mkdir(parents=True) + (content / "reference/api.src.md").write_text( + "# API Reference\n\n## Packages\n\nmto crds\n") + opdocs = tmp_path / "hib" / "content" + (opdocs / "reference").mkdir(parents=True) + (opdocs / "reference/api.md").write_text("# API Reference\n\n## Widgets\n\nhib\n") + op_ctx = {"Hib": {"docs": opdocs, "op_map": {}, "live_url": None, + "style": "directory"}} + targets = {"reference/api.md": { + "under": "Reference", "as": "API Reference", + "sections": [("Hib", "Hib", "reference/api.md")]}} + + pages = [] + for _ in range(3): + nav = [{"Reference": ["reference/api.md"]}] + m.apply_concat(nav, targets, str(content), op_ctx, "Tenant Operator") + pages.append((content / "reference/api.md").read_text()) + assert pages[0] == pages[1] == pages[2] + assert pages[0].count("## Tenant Operator") == 1 + assert "### Packages" in pages[0] and "#### Packages" not in pages[0] + + +def test_apply_concat_rejects_a_target_with_no_source(tmp_path): + content = tmp_path / "content" + (content / "reference").mkdir(parents=True) + # only the generated page is present: output from an earlier merge, or a + # page that was never split into api.md + api.src.md + (content / "reference/api.md").write_text("# API Reference\n\n## Own\n\nmto\n") + targets = {"reference/api.md": {"under": "Reference", "as": None, "sections": []}} + with pytest.raises(ValueError, match="api.src.md"): + m.apply_concat([{"Reference": ["reference/api.md"]}], targets, + str(content), {}, "Tenant Operator") + + +def test_apply_concat_without_own_source_uses_operator_sections_only(tmp_path): + content = tmp_path / "content" + content.mkdir() + opdocs = tmp_path / "hib" / "content" + (opdocs / "reference").mkdir(parents=True) + (opdocs / "reference/api.md").write_text("# API Reference\n\n## Widgets\n\nhib\n") + op_ctx = {"Hib": {"docs": opdocs, "op_map": {}, "live_url": None, + "style": "directory"}} + targets = {"reference/api.md": { + "under": "Reference", "as": "API Reference", + "sections": [("Hib", "Hib", "reference/api.md")]}} + m.apply_concat([{"Reference": ["reference/api.md"]}], targets, str(content), + op_ctx, "Tenant Operator") + page = (content / "reference/api.md").read_text() + assert page.startswith("# Api") # no source page to take an H1 from + assert "## Tenant Operator" not in page and "## Hib" in page + + +def test_concat_source_path(): + assert m.concat_source("content", "reference/api.md") == \ + Path("content/reference/api.src.md") + + def test_shift_headings_skips_code_fences_and_caps(): text = "# Title\n## Sec\n```yaml\n# not a heading\n```\n### Deep\n###### Max" assert m.shift_headings(text, 1) == ( @@ -958,7 +1059,7 @@ def test_run_product_first_fills_placeholder_and_concats(tmp_path): "# API Reference\n\n## Kinds\n\ntpl crds\n") content = tmp_path / "content" (content / "reference").mkdir(parents=True) - (content / "reference/api.md").write_text( + (content / "reference/api.src.md").write_text( "# API Reference\n\n## Packages\n\nmto crds\n") mkdocs = tmp_path / "mkdocs.yml" mkdocs.write_text( @@ -999,7 +1100,7 @@ def test_run_concat_conflicting_under_raises(tmp_path): (repo / "content/reference/api.md").write_text("# API\n\n## X\n\nbody\n") content = tmp_path / "content"; content.mkdir() (content / "reference").mkdir() - (content / "reference/api.md").write_text("# API\n\n## Own\n\nmto\n") + (content / "reference/api.src.md").write_text("# API\n\n## Own\n\nmto\n") mkdocs = tmp_path / "mkdocs.yml" mkdocs.write_text("site_name: MTO\nnav:\n - Reference:\n - reference/api.md\n") operators = [{ diff --git a/tests/test_reorder_api_reference.py b/tests/test_reorder_api_reference.py new file mode 100644 index 000000000..d72d54ad6 --- /dev/null +++ b/tests/test_reorder_api_reference.py @@ -0,0 +1,314 @@ +from reorder_api_reference import (code_flags, headings, kind_sections, + references, reorder_text, type_order) + + +def sections(text, base=2): + """(level, name) for every type section, in document order.""" + lines = text.splitlines() + flags = code_flags(lines) + return [(lvl, title) for _, lvl, title in headings(lines, flags) + if base < lvl <= base + 2 and title.lower() != "resource types"] + + +def names(text, base=2): + return [name for _, name in sections(text, base)] + + +def levels(text, base=2): + return {name: lvl for lvl, name in sections(text, base)} + + +GV = "tenantoperator.stakater.com/v1beta3" + + +def gv_page(kinds, types, gv_level=2, fields=None, reshaped=False): + """A minimal crd-ref-docs page: one group version; `kinds` get the + apiVersion/kind rows crd-ref-docs gives a type with a GVK; `fields` maps a + type to the types its field table links to. `reshaped` renders the page as + this script leaves it — no Resource Types block, types one level up.""" + gv, rt = "#" * gv_level, "#" * (gv_level + 1) + out = ["# API Reference\n", f"{gv} {GV}\n", "Package v1beta3 docs\n"] + if kinds and not reshaped: + out.append(f"{rt} Resource Types") + out += [f"- [{k}](#{k.lower()})" for k in kinds] + out.append("") + for t in types: + # fresh output has every type at the same level, under Resource Types; + # reshaped has the kinds a level above the types they reference + depth = gv_level + 2 + if reshaped and t in kinds: + depth = gv_level + 1 + body = ["#" * depth + f" {t}", "", f"{t} does a thing.", ""] + rows = [] + if t in kinds: + rows.append("| `apiVersion` _string_ | `group.io/v1` | | |") + rows.append(f"| `kind` _string_ | `{t}` | | |") + for ref in (fields or {}).get(t, []): + rows.append(f"| `f{ref}` _[{ref}](#{ref.lower()})_ | doc | | |") + if rows: + body += ["| Field | Description | Default | Validation |", + "| --- | --- | --- | --- |"] + rows + [""] + out.append("\n".join(body)) + return "\n".join(out) + + +class TestTypeOrder: + def test_depth_first_in_field_order(self): + blocks = ["Tenant", "TenantSpec", "TenantStatus", "AccessControl", + "Namespaces", "Sandboxes"] + refs = {"Tenant": ["TenantSpec", "TenantStatus"], + "TenantSpec": ["AccessControl", "Namespaces"], + "Namespaces": ["Sandboxes"]} + assert type_order(["Tenant"], blocks, refs) == [ + "Tenant", "TenantSpec", "AccessControl", "Namespaces", "Sandboxes", + "TenantStatus"] + + def test_a_types_own_types_sit_directly_below_it(self): + blocks = ["Kind", "KindSpec", "A", "B", "KindStatus"] + refs = {"Kind": ["KindSpec", "KindStatus"], "KindSpec": ["A", "B"]} + order = type_order(["Kind"], blocks, refs) + assert order[order.index("KindSpec") + 1:order.index("KindStatus")] == \ + ["A", "B"] + + def test_spec_and_status_still_follow_a_kind_with_no_linking_rows(self): + # ignoreFields can drop the rows that would have linked them + blocks = ["Tenant", "TenantSpec", "TenantStatus", "Helper"] + assert type_order(["Tenant"], blocks, {}) == [ + "Tenant", "TenantSpec", "TenantStatus", "Helper"] + + def test_kinds_do_not_interleave(self): + blocks = ["A", "ASpec", "AHelper", "B", "BSpec", "BHelper"] + refs = {"A": ["ASpec"], "ASpec": ["AHelper"], + "B": ["BSpec"], "BSpec": ["BHelper"]} + assert type_order(["A", "B"], blocks, refs) == [ + "A", "ASpec", "AHelper", "B", "BSpec", "BHelper"] + + def test_types_no_kind_reaches_follow_alphabetically(self): + blocks = ["Tenant", "TenantSpec", "Zebra", "Orphan"] + refs = {"Tenant": ["TenantSpec"]} + assert type_order(["Tenant"], blocks, refs) == [ + "Tenant", "TenantSpec", "Orphan", "Zebra"] + + def test_a_reference_cycle_terminates(self): + blocks = ["Kind", "A", "B"] + refs = {"Kind": ["A"], "A": ["B"], "B": ["A", "Kind"]} + assert type_order(["Kind"], blocks, refs) == ["Kind", "A", "B"] + + def test_a_reference_to_a_type_with_no_section_is_skipped(self): + blocks = ["Kind", "KindSpec"] + refs = {"Kind": ["KindSpec"], "KindSpec": ["ObjectMeta"]} + assert type_order(["Kind"], blocks, refs) == ["Kind", "KindSpec"] + + def test_no_kinds_keeps_the_existing_order(self): + assert type_order([], ["Beta", "Alpha"], {}) == ["Beta", "Alpha"] + + +class TestReferences: + def build(self, page): + lines = page.splitlines() + flags = code_flags(lines) + from reorder_api_reference import headings + heads = headings(lines, flags) + types = [(i, t) for i, lvl, t in heads if lvl == 4] + bounds = [i for i, _ in types] + [len(lines)] + spans = {t: (i, bounds[k + 1]) for k, (i, t) in enumerate(types)} + return lines, flags, spans + + def test_field_table_links_become_references(self): + page = gv_page(["Kind"], ["Kind", "KindSpec"], + fields={"Kind": ["KindSpec"]}) + lines, flags, spans = self.build(page) + assert references(lines, flags, spans)["Kind"] == ["KindSpec"] + + def test_appears_in_backreference_is_not_a_reference(self): + page = "\n".join([ + "# API Reference\n", "## group.io/v1\n", + "#### Kind\n", "docs\n", + "| Field | Description | Default | Validation |", + "| --- | --- | --- | --- |", + "| `spec` _[KindSpec](#kindspec)_ | doc | | |\n", + "#### KindSpec\n", + "_Appears in:_", + "- [Kind](#kind)\n", + ]) + lines, flags, spans = self.build(page) + refs = references(lines, flags, spans) + assert refs["Kind"] == ["KindSpec"] + assert refs["KindSpec"] == [] # the back-edge is not followed + + def test_an_alias_references_its_underlying_type(self): + page = "\n".join([ + "# API Reference\n", "## group.io/v1\n", + "#### Alias\n", + "_Underlying type:_ _[Real](#real)_\n", + "#### Real\n", "docs\n", + ]) + lines, flags, spans = self.build(page) + assert references(lines, flags, spans)["Alias"] == ["Real"] + + def test_kinds_are_found_by_their_own_table_rows(self): + page = gv_page(["Beta", "Alpha"], ["Alpha", "AlphaSpec", "Beta"]) + lines, flags, spans = self.build(page) + # page order, not the order they were listed in + assert kind_sections(lines, flags, spans) == ["Alpha", "Beta"] + + +class TestReorderText: + def test_walks_the_tree_from_each_kind(self): + page = gv_page(["Tenant"], + ["AccessControl", "Namespaces", "Tenant", "TenantSpec", + "TenantStatus"], + fields={"Tenant": ["TenantSpec", "TenantStatus"], + "TenantSpec": ["AccessControl", "Namespaces"]}) + assert names(reorder_text(page)) == [ + "Tenant", "TenantSpec", "AccessControl", "Namespaces", + "TenantStatus"] + + def test_drops_the_resource_types_heading_and_its_list(self): + page = gv_page(["Tenant"], ["AccessControl", "Tenant"]) + out = reorder_text(page) + assert "Resource Types" not in out + assert "- [Tenant](#tenant)" not in out + + def test_kinds_sit_directly_under_the_package(self): + page = gv_page(["Tenant"], ["AccessControl", "Tenant"]) + out = reorder_text(page) + assert "### Tenant\n" in out and "#### Tenant\n" not in out + assert "## " + GV + "\n\nPackage v1beta3 docs\n\n### Tenant\n" in out + + def test_a_referenced_type_is_one_step_smaller_at_any_depth(self): + page = gv_page(["Tenant"], + ["AccessControl", "Namespaces", "Sandboxes", "Tenant", + "TenantSpec"], + fields={"Tenant": ["TenantSpec"], + "TenantSpec": ["AccessControl", "Namespaces"], + "Namespaces": ["Sandboxes"]}) + # Sandboxes is three references deep and still only one level smaller + assert levels(reorder_text(page)) == { + "Tenant": 3, "TenantSpec": 4, "AccessControl": 4, + "Namespaces": 4, "Sandboxes": 4} + + def test_a_type_no_kind_reaches_is_a_step_smaller_too(self): + page = gv_page(["Tenant"], ["Orphan", "Tenant"]) + assert levels(reorder_text(page)) == {"Tenant": 3, "Orphan": 4} + + def test_body_of_each_section_travels_with_its_heading(self): + page = gv_page(["Tenant"], ["AccessControl", "Tenant"]) + out = reorder_text(page) + assert "### Tenant\n\nTenant does a thing." in out + assert "#### AccessControl\n\nAccessControl does a thing." in out + + def test_every_section_is_separated_by_a_blank_line(self): + # the section that happened to sit last in the file must not end up + # flush against the next heading once it is moved + page = gv_page(["Tenant"], ["AccessControl", "Tenant", "TenantSpec"]) + assert "does a thing.\n###" not in reorder_text(page) + + def test_reorders_a_page_it_has_already_reshaped(self): + # kinds come from each type's own rows, so the deleted Resource Types + # list is not needed to walk the tree a second time + page = gv_page(["Tenant"], ["AccessControl", "Tenant", "TenantSpec"], + fields={"Tenant": ["TenantSpec"], + "TenantSpec": ["AccessControl"]}, + reshaped=True) + out = reorder_text(page) + assert names(out) == ["Tenant", "TenantSpec", "AccessControl"] + assert levels(out) == {"Tenant": 3, "TenantSpec": 4, + "AccessControl": 4} + assert out == reorder_text(out) + + def test_group_versions_are_reordered_independently(self): + page = "\n".join([ + "# API Reference\n", + "## group.io/v1\n", + "### Resource Types", + "- [Alpha](#alpha)", + "", + "#### AHelper\n", + "#### Alpha\n", + "| Field | Description | Default | Validation |", + "| --- | --- | --- | --- |", + "| `kind` _string_ | `Alpha` | | |\n", + "## group.io/v2\n", + "### Resource Types", + "- [Beta](#beta)", + "", + "#### BHelper\n", + "#### Beta\n", + "| Field | Description | Default | Validation |", + "| --- | --- | --- | --- |", + "| `kind` _string_ | `Beta` | | |\n", + ]) + out = reorder_text(page) + first, _, second = out.partition("## group.io/v2") + assert names(first) == ["Alpha", "AHelper"] + assert names(second) == ["Beta", "BHelper"] + + def test_works_on_merged_heading_levels(self): + # merge_docs demotes every heading by one, so types arrive at level 5 + page = gv_page(["Tenant"], ["AccessControl", "Tenant", "TenantSpec"], + gv_level=3, fields={"Tenant": ["TenantSpec"]}) + out = reorder_text(page) + assert names(out, base=3) == ["Tenant", "TenantSpec", "AccessControl"] + assert levels(out, base=3) == {"Tenant": 4, "TenantSpec": 5, + "AccessControl": 5} + + def test_headings_inside_code_fences_are_not_sections(self): + page = "\n".join([ + "# API Reference\n", + "## group.io/v1\n", + "### Resource Types", + "- [Tenant](#tenant)", + "", + "#### AccessControl\n", + "```yaml", + "#### NotAType", + "```\n", + "#### Tenant\n", + "| Field | Description | Default | Validation |", + "| --- | --- | --- | --- |", + "| `kind` _string_ | `Tenant` | | |\n", + ]) + out = reorder_text(page) + assert names(out) == ["Tenant", "AccessControl"] + assert "#### NotAType" in out # untouched inside the fence + assert out.index("#### AccessControl") < out.index("#### NotAType") + + def test_content_outside_any_group_version_is_preserved(self): + page = "\n".join([ + "# API Reference\n", + "\n", + "## Packages", + "- [group.io/v1](#groupiov1)\n", + "## group.io/v1\n", + "### Resource Types", + "- [Tenant](#tenant)", + "", + "#### AccessControl\n", + "#### Tenant\n", + "## Trailing prose\n", + "Not a group version.\n", + ]) + out = reorder_text(page) + assert "" in out + assert "## Packages" in out + assert "- [group.io/v1](#groupiov1)" in out # the package list stays + assert out.rstrip().endswith("Not a group version.") + + def test_is_idempotent(self): + page = gv_page(["Tenant", "Quota"], + ["AccessControl", "Quota", "QuotaSpec", "Tenant", + "TenantSpec", "TenantStatus", "Zebra"], + fields={"Tenant": ["TenantSpec", "TenantStatus"], + "TenantSpec": ["AccessControl"], + "Quota": ["QuotaSpec"]}) + once = reorder_text(page) + assert reorder_text(once) == once + + def test_trailing_newline_is_preserved(self): + page = gv_page(["Tenant"], ["AccessControl", "Tenant"]) + assert reorder_text(page + "\n").endswith("\n") + + def test_page_without_group_versions_is_unchanged(self): + page = "# API Reference\n\nNothing generated yet.\n" + assert reorder_text(page) == page diff --git a/theme_override/mkdocs.yml b/theme_override/mkdocs.yml index 2a4bd16f5..4a994b610 100644 --- a/theme_override/mkdocs.yml +++ b/theme_override/mkdocs.yml @@ -10,6 +10,17 @@ theme: # which unions lists, so stylesheets/extra.css is preserved. extra_css: - stylesheets/nav-labels.css + - stylesheets/api-toc.css + - stylesheets/api-reference.css + +extra_javascript: + - javascripts/api-toc.js + +# mto-docs' own half of the concatenated API reference (see merge.yaml +# concat_into). The merge builds reference/api.md out of it, so it must not be +# rendered as a page of its own. +exclude_docs: | + reference/api.src.md strict: true validation: @@ -20,6 +31,11 @@ extra: version: provider: mike default: latest + # Pages whose "On this page" sections render collapsed, read by the + # partials/toc-item.html fork. The API reference is one page carrying every + # type of every operator, so its TOC is a few hundred entries long. + collapsible_toc_pages: + - reference/api.md # Resolves {{ screenshot: }} directives to the images captured by # screenshots/capture.sh. Our own module, loaded by mkdocs' built-in hooks: key -- diff --git a/theme_override/resources/javascripts/api-toc.js b/theme_override/resources/javascripts/api-toc.js new file mode 100644 index 000000000..b1b02c2e1 --- /dev/null +++ b/theme_override/resources/javascripts/api-toc.js @@ -0,0 +1,79 @@ +/* Collapsible "On this page" sections (see partials/toc-item.html and + stylesheets/api-toc.css). Collapsing itself is pure CSS; script is only + needed for deep links, which have to reveal the section holding the anchor, + and for keyboard use, since a