feat: arrange types on reference page in DFS sequence

.gitignore

@@ -33,3 +33,8 @@ __pycache__/
 
 # Screenshot injection output (staged from screenshots/captured/ at build time)
 content/images/generated/
+
+# Generated by the merge (scripts/merge_docs.py concat_into): mto's own half of
+# the page is content/reference/api.src.md, this is that plus every
+# sub-operator's API reference, rebuilt on each merge.
+content/reference/api.md

.vale.ini

@@ -13,11 +13,11 @@ BasedOnStyles = Vale
 # reach the reader. The name is a filename, so spelling and term rules don't apply.
 TokenIgnores = (\{\{\s*screenshot:[^}]*\}\})
 
-# content/reference/api.md is machine-generated from the operator's Go types
+# content/reference/api.src.md is machine-generated from the operator's Go types
 # (via crd-ref-docs). It contains API identifiers and verbatim code comments
 # (e.g. the `tenantoperator.stakater.com` group, the `LabelSelector` type,
 # camelCase field names), not prose, so prose style rules don't apply.
-[content/reference/api.md]
+[content/reference/api.src.md]
 BasedOnStyles =
 Vale.Terms = NO
 Vale.Spelling = NO

Makefile

@@ -66,9 +66,13 @@ serve: theme ## Full local preview: clones the sub-operator repos, then mkdocs s
 	$(PY) -m mkdocs serve
 
 serve-local: theme merge-local ## Same preview from local checkouts, no cloning
-	$(PY) -m mkdocs serve
+	$(PY) -m mkdocs serve -a localhost:9000
 
 clean: ## Remove fetched repos and generated artifacts (surgical; never `git clean`)
 	rm -rf .suboperators mkdocs.yml dist site
+	@python3 -c "import sys;sys.path.insert(0,'scripts');import merge_docs;\
+	  print('\n'.join(sorted({m['concat_into'] for o in merge_docs.load_config('merge.yaml') \
+	  for m in o['mappings'] if m.get('concat_into')})))" 2>/dev/null \
+	  | while read -r f; do [ -n "$$f" ] && rm -f "content/$$f"; done
 	@python3 -c "import sys;sys.path.insert(0,'scripts');import merge_docs;[print(o['slug']) for o in merge_docs.load_config('merge.yaml')]" 2>/dev/null \
 	  | while read -r s; do [ -n "$$s" ] && find content -type d -name "$$s" -prune -exec rm -rf {} + 2>/dev/null || true; done

merge.yaml

@@ -24,7 +24,9 @@ operators:
       - from: "guides/**"
         into: "guides"
         under: "Guides"
-      - from: "reference/api.md"          # concatenated into the single API page
+      # concatenated onto mto's own reference/api.src.md; the result,
+      # reference/api.md, is generated and gitignored
+      - from: "reference/api.md"
         concat_into: "reference/api.md"
         under: "Reference"
         as: "API Reference"

scripts/merge_docs.py

@@ -12,6 +12,8 @@
 import yaml
 from pathlib import Path
 
+import reorder_api_reference
+
 _WILDCARD = set("*?[]")
 
 
@@ -315,28 +317,57 @@ def apply_group(nav, group):
     section[:] = new_section
 
 
+def reshape_api_page(text):
+    """Kinds above their own types, no `Resource Types` wrapper — applied to
+    every source of a concatenated page, mto-docs' own and each sub-operator's,
+    so the one page reads the same the whole way down. Sub-operator repos have
+    no such step of their own, and their published sites keep whatever order
+    crd-ref-docs gave them; this only shapes the copy merged in here. A page
+    that is not crd-ref-docs output has no group-version headings and comes back
+    unchanged."""
+    return reorder_api_reference.reorder_text(text)
+
+
+def concat_source(content_dir, into):
+    """Where mto-docs' own half of a concatenated page lives: `api.md` is built
+    from `api.src.md`. The two must stay separate — `into` is overwritten on
+    every merge, so reading mto's own content back out of it would demote and
+    re-wrap the whole page one level deeper each run."""
+    path = Path(content_dir) / into
+    return path.with_name(f"{path.stem}.src{path.suffix}")
+
+
 def apply_concat(nav, targets, content_dir, op_ctx, site_title):
-    """Concatenate several source pages into one per `into` target. The existing
-    content page (mto-docs' own) is the first `## site_title` section; each
-    contributing operator adds a `## heading` section sourced from its clone and
+    """Concatenate several source pages into one per `into` target. mto-docs'
+    own `<into>.src.md` is the first `## site_title` section; each contributing
+    operator adds a `## heading` section sourced from its clone and
     link-rewritten. Every source's own headings are demoted one level so the
-    in-page TOC lists the sections. One `{as: into}` leaf is set under `under`."""
+    in-page TOC lists the sections. One `{as: into}` leaf is set under `under`.
+    `into` itself is pure output: regenerated from scratch here, gitignored."""
     content_dir = Path(content_dir)
     for into, spec in targets.items():
         blocks, page_title = [], None
         self_path = content_dir / into
-        if self_path.is_file():
-            page_title, body = split_h1(self_path.read_text(encoding="utf-8"))
+        src_path = concat_source(content_dir, into)
+        if src_path.is_file():
+            raw = reshape_api_page(src_path.read_text(encoding="utf-8"))
+            page_title, body = split_h1(raw)
             blocks.append(f"## {site_title}\n\n{shift_headings(body, 1).strip()}\n")
+        elif self_path.is_file():
+            raise ValueError(
+                f"concat target {into!r} exists but {src_path.name!r} does not: "
+                f"mto-docs' own half of the page belongs in {src_path.name!r} "
+                f"({into} is generated). Rename it, or delete it if it is "
+                f"output from an earlier merge.")
         for heading, op_title, frm in spec["sections"]:
             ctx = op_ctx[op_title]
             raw = (ctx["docs"] / frm).read_text(encoding="utf-8")
             raw, _ = rewrite_links(raw, frm, into, ctx["op_map"],
                                    ctx["live_url"], ctx["style"])
-            body = shift_headings(split_h1(raw)[1], 1).strip()
+            body = shift_headings(split_h1(reshape_api_page(raw))[1], 1).strip()
             blocks.append(f"## {heading}\n\n{body}\n")
         if page_title is None:
-            page_title = read_h1(self_path) or prettify(Path(into).stem)
+            page_title = read_h1(src_path) or prettify(Path(into).stem)
         self_path.parent.mkdir(parents=True, exist_ok=True)
         self_path.write_text(f"# {page_title}\n\n" + "\n".join(blocks), encoding="utf-8")
         _set_single_leaf(nav, spec["under"], into, spec.get("as"))

scripts/reorder_api_reference.py

@@ -0,0 +1,263 @@
+"""Reshape the type sections of a crd-ref-docs markdown page.
+
+Two things, both of which the generator has no knob for:
+
+- **Order.** crd-ref-docs renders every type of a group version alphabetically
+  (`GroupVersionDetails.SortedTypes`), so `Tenant` ends up buried after
+  `AccessControl`. Instead the types are walked as the tree they are: depth
+  first from each kind, in field order, so the types a section references sit
+  directly below it — scroll past `TenantSpec` and you are in the types
+  `TenantSpec` is made of. Kinds are taken one at a time, so their subtrees do
+  not interleave, and anything no kind reaches is left alphabetical at the end.
+- **Depth.** A kind's heading sits directly under its package; every type that
+  something else references is one step smaller, however deep in the tree it is,
+  so a package reads as a list of kinds with their parts under them.
+  The `Resource Types` heading and its list of kinds are dropped along the way:
+  it was the only thing between a package and its types, and with the kinds
+  ordered first it said nothing the page does not.
+
+Run it after generation, alongside the fixups in
+docs/generating-api-reference.md:
+
+    python scripts/reorder_api_reference.py content/reference/api.md
+
+Section bodies are moved verbatim (only their heading level and trailing blank
+lines change), and re-running the script changes nothing.
+"""
+import argparse
+import re
+import sys
+from pathlib import Path
+
+# A group-version heading, e.g. "tenantoperator.stakater.com/v1beta3". Type
+# names never contain a slash, so this cannot match a type section.
+GV_RE = re.compile(r"^[A-Za-z0-9.\-]+/[A-Za-z0-9.\-]+$")
+HEADING_RE = re.compile(r"(#{1,6})\s+(.*\S)\s*$")
+KIND_BULLET_RE = re.compile(r"^-\s+\[([A-Za-z0-9_]+)\]")
+KIND_ROW_RE = re.compile(r"\|\s*`kind`\s+_string_\s*\|")
+ANCHOR_LINK_RE = re.compile(r"\]\(#([A-Za-z0-9_-]+)\)")
+SUFFIXES = ("Spec", "Status")
+
+
+def anchor(name):
+    """The in-page anchor crd-ref-docs links a type by."""
+    return re.sub(r"[^a-z0-9]", "", name.lower())
+
+
+def code_flags(lines):
+    """One bool per line, True inside a ``` / ~~~ fence, so that a `#` in a
+    YAML sample is never mistaken for a heading. Mirrors merge_docs._md_lines;
+    kept local so this script can be dropped into a sub-operator docs repo."""
+    fence, out = None, []
+    for line in lines:
+        mo = re.match(r"(`{3,}|~{3,})", line.strip())
+        marker = mo.group(1)[0] if mo else None
+        if fence:
+            out.append(True)
+            if marker == fence:
+                fence = None
+        elif marker:
+            fence = marker
+            out.append(True)
+        else:
+            out.append(False)
+    return out
+
+
+def headings(lines, flags):
+    """(line index, level, title) for every ATX heading outside code fences."""
+    out = []
+    for i, line in enumerate(lines):
+        if flags[i]:
+            continue
+        mo = HEADING_RE.match(line)
+        if mo:
+            out.append((i, len(mo.group(1)), mo.group(2).strip()))
+    return out
+
+
+def type_order(kinds, blocks, refs=None):
+    """The order type sections should appear in: one depth-first walk per kind,
+    following its fields in the order they are declared, so whatever a section
+    references sits directly below it. `<Kind>Spec` and `<Kind>Status` are
+    walked after the kind as a fallback, in case `ignoreFields` dropped the rows
+    that would have linked them. Types no kind reaches follow alphabetically.
+    With no kinds at all the existing order is kept."""
+    refs = refs or {}
+    order, seen = [], set()
+
+    def walk(name):
+        if name not in blocks or name in seen:
+            return
+        seen.add(name)
+        order.append(name)
+        for ref in refs.get(name, ()):
+            walk(ref)
+
+    for kind in kinds:
+        walk(kind)
+        for suffix in SUFFIXES:
+            walk(kind + suffix)
+    rest = [name for name in blocks if name not in seen]
+    order.extend(sorted(rest) if kinds else rest)
+    return order
+
+
+def references(lines, flags, spans):
+    """Forward references per type, in field order: the other types of this group
+    version that its field table links to, plus an alias's underlying type.
+    Only table rows are read, so the `Appears in:` bullet list — which points the
+    other way — never becomes an edge."""
+    known = {anchor(name): name for name in spans}
+    out = {}
+    for name, (start, stop) in spans.items():
+        refs, seen = [], {name}
+        for j in range(start, stop):
+            line = lines[j]
+            if flags[j]:
+                continue
+            if not (line.startswith("|") or "_Underlying type:_" in line):
+                continue
+            for target in ANCHOR_LINK_RE.findall(line):
+                ref = known.get(target)
+                if ref and ref not in seen:
+                    seen.add(ref)
+                    refs.append(ref)
+        out[name] = refs
+    return out
+
+
+def resource_types_span(lines, flags, section, rt_level):
+    """The half-open line range one group version's `Resource Types` heading and
+    its list of kinds occupy, so the caller can drop the block."""
+    for i, level, title in section:
+        if level != rt_level or title.lower() != "resource types":
+            continue
+        j, seen_bullet = i + 1, False
+        while j < len(lines):
+            line = lines[j].strip()
+            if flags[j] or line.startswith("#"):
+                break
+            if not line:
+                if seen_bullet:
+                    break
+                j += 1
+                continue
+            if not KIND_BULLET_RE.match(line):
+                break
+            seen_bullet = True
+            j += 1
+        return i, j
+    return None
+
+
+def kind_sections(lines, flags, spans):
+    """The type names that are kinds, in page order. crd-ref-docs gives a type
+    with a GVK an `apiVersion` and a `kind` row that nothing else has, so this
+    does not depend on the `Resource Types` list — which this script deletes, and
+    which therefore cannot be the thing that identifies them."""
+    kinds = []
+    for name, (start, stop) in sorted(spans.items(), key=lambda kv: kv[1][0]):
+        for j in range(start, stop):
+            if not flags[j] and KIND_ROW_RE.match(lines[j]):
+                kinds.append(name)
+                break
+    return kinds
+
+
+def at_level(lines, flags, start, stop, level):
+    """One type section with its heading set to `level`, and any heading inside
+    it moved by the same amount. Kinds sit directly under their package; a type
+    something else references is a step smaller, whatever its depth in the tree.
+    Trailing blank lines are normalised to one, so that the section which
+    happened to sit last in the file does not end up flush against the next
+    heading once moved."""
+    mo = HEADING_RE.match(lines[start])
+    shift = level - len(mo.group(1)) if mo else 0
+    out = []
+    for j in range(start, stop):
+        line = lines[j]
+        if not flags[j] and shift:
+            mo = HEADING_RE.match(line)
+            if mo:
+                depth = min(max(len(mo.group(1)) + shift, 1), 6)
+                line = "#" * depth + line[len(mo.group(1)):]
+        out.append(line)
+    while out and not out[-1].strip():
+        out.pop()
+    return out + [""]
+
+
+def reorder_text(text):
+    lines = text.splitlines()
+    flags = code_flags(lines)
+    heads = headings(lines, flags)
+
+    out, pos = [], 0
+    for n, (start, level, title) in enumerate(heads):
+        if not GV_RE.match(title):
+            continue
+        end = len(lines)
+        for i, lvl, _ in heads[n + 1:]:
+            if lvl <= level:
+                end = i
+                break
+        section = [h for h in heads if start < h[0] < end]
+        # Every heading just below the package is a type section, at whichever
+        # level it currently sits: crd-ref-docs puts them all at level + 2 under
+        # `Resource Types`, and this script leaves kinds at level + 1 and the
+        # rest at level + 2, so both shapes have to be recognised.
+        types = [(i, t) for i, lvl, t in section
+                 if level < lvl <= level + 2 and t.lower() != "resource types"]
+        if not types:
+            continue
+
+        rt_span = resource_types_span(lines, flags, section, level + 1)
+        bounds = [i for i, _ in types] + [end]
+        spans = {t: (i, bounds[k + 1]) for k, (i, t) in enumerate(types)}
+        refs = references(lines, flags, spans)
+        kinds = set(kind_sections(lines, flags, spans))
+        blocks = {t: at_level(lines, flags, *spans[t],
+                              level + (1 if t in kinds else 2))
+                  for t in spans}
+        # the package doc, up to whatever the types used to be introduced by
+        preamble = lines[pos:rt_span[0] if rt_span else types[0][0]]
+        while preamble and not preamble[-1].strip():
+            preamble.pop()
+        out.extend(preamble + [""])
+        for name in type_order(kind_sections(lines, flags, spans), blocks, refs):
+            out.extend(blocks[name])
+        pos = end
+    out.extend(lines[pos:])
+
+    result = "\n".join(out)
+    return result + "\n" if text.endswith("\n") else result
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(
+        description="Order kinds before their nested types in a crd-ref-docs "
+                    "page, and drop the Resource Types wrapper.")
+    ap.add_argument("paths", nargs="+", type=Path, help="markdown files to reorder")
+    ap.add_argument("--check", action="store_true",
+                    help="report files that would change, write nothing")
+    args = ap.parse_args(argv)
+
+    changed = []
+    for path in args.paths:
+        text = path.read_text(encoding="utf-8")
+        new = reorder_text(text)
+        if new == text:
+            print(f"{path}: already reshaped")
+            continue
+        changed.append(path)
+        if args.check:
+            print(f"{path}: would be reshaped")
+        else:
+            path.write_text(new, encoding="utf-8")
+            print(f"{path}: reshaped")
+    return 1 if args.check and changed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())