#!/usr/bin/env python3 """Generate a deterministic, source-addressable LinuxCNC document index. The index is intentionally a source index, not a coverage claim. Every heading in the Master Documentation include graph and every heading in the project's migration documents becomes a planned capability candidate. Files that cannot be classified are retained with ``needs_review`` rather than silently discarded. """ from __future__ import annotations import argparse import hashlib import json import re from dataclasses import dataclass from pathlib import Path from typing import Iterable MASTER = Path("linuxcnc/docs/src/Master_Documentation.adoc") PROJECT_DOCS = ( Path("docs/linuxcnc-web-wasm-source-map.md"), Path("docs/linuxcnc-web-wasm-simulation-coverage.md"), ) INCLUDE_RE = re.compile(r"^\s*include::([^\[]+)\[.*\]\s*$") ASCIIDOC_ANCHOR_RE = re.compile(r"^\s*\[\[([^,\]]+)(?:,[^\]]*)?\]\]\s*$") ASCIIDOC_BLOCK_ID_RE = re.compile(r"^\s*\[#([^\]]+)\]\s*$") ASCIIDOC_HEADING_RE = re.compile(r"^(={1,6})\s+(.+?)\s*$") MARKDOWN_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$") SLUG_RE = re.compile(r"[^a-z0-9]+") DOMAIN_RULES = ( ("getting-started", "getting_started"), ("user", "user"), ("config", "configuration"), ("hal", "hal"), ("ladder", "classicladder"), ("motion", "motion"), ("gcode", "gcode"), ("remap", "remap"), ("gui", "gui"), ("plasma", "plasma"), ("tooldatabase", "tooldata"), ("drivers", "real_hardware"), ("examples", "examples"), ("integrator", "integrator"), ("install", "installation"), ("source-highlight", "documentation"), ) @dataclass(frozen=True) class Heading: level: int title: str anchor: str | None line: int def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def slugify(title: str) -> str: value = title.lower().replace("&", " and ") value = SLUG_RE.sub("-", value).strip("-") return value or "section" def parse_headings(path: Path) -> list[Heading]: lines = path.read_text(encoding="utf-8", errors="strict").splitlines() headings: list[Heading] = [] pending_anchor: str | None = None for line_number, line in enumerate(lines, 1): anchor_match = ASCIIDOC_ANCHOR_RE.match(line) or ASCIIDOC_BLOCK_ID_RE.match(line) if anchor_match: pending_anchor = anchor_match.group(1).strip() continue match = ASCIIDOC_HEADING_RE.match(line) or MARKDOWN_HEADING_RE.match(line) if not match: continue headings.append(Heading(len(match.group(1)), match.group(2).strip(), pending_anchor, line_number)) pending_anchor = None return headings def include_graph(repo_root: Path, master: Path = MASTER) -> tuple[list[Path], list[dict[str, str]]]: ordered: list[Path] = [] seen: set[Path] = set() missing: list[dict[str, str]] = [] def visit(path: Path) -> None: path = path.resolve() if path in seen: return if not path.is_file(): raise FileNotFoundError(f"included documentation file is missing: {path}") seen.add(path) ordered.append(path) for line in path.read_text(encoding="utf-8", errors="strict").splitlines(): match = INCLUDE_RE.match(line) if match: include_path = (path.parent / match.group(1)).resolve() if not include_path.is_file(): missing.append({ "document": relative_path(repo_root, path), "include": match.group(1), "resolved_path": relative_path(repo_root, include_path), }) continue visit(include_path) visit((repo_root / master).resolve()) return ordered, missing def classify(path: Path, title: str) -> tuple[str, str]: lowered = "/".join(path.parts).lower() for token, domain in DOMAIN_RULES: if token in lowered: return domain, "mapped" title_lower = title.lower() if any(token in title_lower for token in ("g-code", "m-code", "o-code", "rs274")): return "gcode", "mapped" if any(token in title_lower for token in ("hardware", "driver", "mesa", "gpio")): return "real_hardware", "needs_review" if path.suffix.lower() in {".md", ".adoc"} and path.as_posix().startswith("docs/"): return "web_wasm_governance", "mapped" return "unclassified", "needs_review" def relative_path(repo_root: Path, path: Path) -> str: return path.resolve().relative_to(repo_root.resolve()).as_posix() def make_entries(repo_root: Path) -> tuple[list[dict[str, object]], dict[str, object]]: paths, missing_includes = include_graph(repo_root) for project_doc in PROJECT_DOCS: path = (repo_root / project_doc).resolve() if not path.is_file(): raise FileNotFoundError(f"project migration document is missing: {project_doc}") if path not in paths: paths.append(path) entries: list[dict[str, object]] = [] report: dict[str, object] = { "schema": "document-capabilities-report.v1", "documents": len(paths), "headings": 0, "unclassified": [], "duplicate_ids": [], "empty_documents": [], "missing_includes": missing_includes, } used_ids: set[str] = set() for path in paths: relative = relative_path(repo_root, path) document_sha = sha256(path) headings = parse_headings(path) if not headings: report["empty_documents"].append(relative) headings = [Heading(1, path.stem, None, 1)] anchor_counts: dict[str, int] = {} parent_stack: list[tuple[int, str]] = [] for heading in headings: base_anchor = heading.anchor or slugify(heading.title) anchor_counts[base_anchor] = anchor_counts.get(base_anchor, 0) + 1 anchor = base_anchor if anchor_counts[base_anchor] == 1 else f"{base_anchor}-{anchor_counts[base_anchor]}" capability_id = f"doc:{relative}#{anchor}" if capability_id in used_ids: report["duplicate_ids"].append(capability_id) continue used_ids.add(capability_id) while parent_stack and parent_stack[-1][0] >= heading.level: parent_stack.pop() parent_anchor = parent_stack[-1][1] if parent_stack else None parent_stack.append((heading.level, anchor)) domain, mapping_status = classify(path, heading.title) entry = { "capability_source_id": capability_id, "document": relative, "anchor": anchor, "title": heading.title, "domain": domain, "source_sha": document_sha, "line": heading.line, "level": heading.level, "parent_anchor": parent_anchor, "mapping_status": mapping_status, "sim_status": "planned", "web_status": "planned", } entries.append(entry) report["headings"] += 1 if mapping_status != "mapped": report["unclassified"].append({"capability_source_id": capability_id, "domain": domain}) entries.sort(key=lambda item: str(item["capability_source_id"])) return entries, report def write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8") def write_jsonl(path: Path, entries: Iterable[dict[str, object]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8", newline="\n") as handle: for entry in entries: handle.write(json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) handle.write("\n") def generate(repo_root: Path, output: Path, report_path: Path) -> tuple[int, str]: entries, report = make_entries(repo_root) write_jsonl(output, entries) report["output_sha256"] = sha256(output) write_json(report_path, report) return len(entries), str(report["output_sha256"]) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1]) parser.add_argument("--output", type=Path, default=Path("coverage/document-capabilities.jsonl")) parser.add_argument("--report", type=Path, default=Path("coverage/document-capabilities.report.json")) args = parser.parse_args() root = args.repo_root.resolve() count, digest = generate(root, root / args.output, root / args.report) print(f"document_capabilities={count} sha256={digest}") return 0 if __name__ == "__main__": raise SystemExit(main())