Initial LinuxCNC WebAssembly workspace snapshot
This commit is contained in:
241
scripts/generate_document_capabilities.py
Normal file
241
scripts/generate_document_capabilities.py
Normal file
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a deterministic, source-addressable LinuxCNC document index.
|
||||
|
||||
The index is intentionally a source index, not a coverage claim. Every
|
||||
heading in the Master Documentation include graph and every heading in the
|
||||
project's migration documents becomes a planned capability candidate. Files
|
||||
that cannot be classified are retained with ``needs_review`` rather than
|
||||
silently discarded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
MASTER = Path("linuxcnc/docs/src/Master_Documentation.adoc")
|
||||
PROJECT_DOCS = (
|
||||
Path("docs/linuxcnc-web-wasm-source-map.md"),
|
||||
Path("docs/linuxcnc-web-wasm-simulation-coverage.md"),
|
||||
)
|
||||
INCLUDE_RE = re.compile(r"^\s*include::([^\[]+)\[.*\]\s*$")
|
||||
ASCIIDOC_ANCHOR_RE = re.compile(r"^\s*\[\[([^,\]]+)(?:,[^\]]*)?\]\]\s*$")
|
||||
ASCIIDOC_BLOCK_ID_RE = re.compile(r"^\s*\[#([^\]]+)\]\s*$")
|
||||
ASCIIDOC_HEADING_RE = re.compile(r"^(={1,6})\s+(.+?)\s*$")
|
||||
MARKDOWN_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$")
|
||||
SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
DOMAIN_RULES = (
|
||||
("getting-started", "getting_started"),
|
||||
("user", "user"),
|
||||
("config", "configuration"),
|
||||
("hal", "hal"),
|
||||
("ladder", "classicladder"),
|
||||
("motion", "motion"),
|
||||
("gcode", "gcode"),
|
||||
("remap", "remap"),
|
||||
("gui", "gui"),
|
||||
("plasma", "plasma"),
|
||||
("tooldatabase", "tooldata"),
|
||||
("drivers", "real_hardware"),
|
||||
("examples", "examples"),
|
||||
("integrator", "integrator"),
|
||||
("install", "installation"),
|
||||
("source-highlight", "documentation"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Heading:
|
||||
level: int
|
||||
title: str
|
||||
anchor: str | None
|
||||
line: int
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def slugify(title: str) -> str:
|
||||
value = title.lower().replace("&", " and ")
|
||||
value = SLUG_RE.sub("-", value).strip("-")
|
||||
return value or "section"
|
||||
|
||||
|
||||
def parse_headings(path: Path) -> list[Heading]:
|
||||
lines = path.read_text(encoding="utf-8", errors="strict").splitlines()
|
||||
headings: list[Heading] = []
|
||||
pending_anchor: str | None = None
|
||||
for line_number, line in enumerate(lines, 1):
|
||||
anchor_match = ASCIIDOC_ANCHOR_RE.match(line) or ASCIIDOC_BLOCK_ID_RE.match(line)
|
||||
if anchor_match:
|
||||
pending_anchor = anchor_match.group(1).strip()
|
||||
continue
|
||||
match = ASCIIDOC_HEADING_RE.match(line) or MARKDOWN_HEADING_RE.match(line)
|
||||
if not match:
|
||||
continue
|
||||
headings.append(Heading(len(match.group(1)), match.group(2).strip(), pending_anchor, line_number))
|
||||
pending_anchor = None
|
||||
return headings
|
||||
|
||||
|
||||
def include_graph(repo_root: Path, master: Path = MASTER) -> tuple[list[Path], list[dict[str, str]]]:
|
||||
ordered: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
missing: list[dict[str, str]] = []
|
||||
|
||||
def visit(path: Path) -> None:
|
||||
path = path.resolve()
|
||||
if path in seen:
|
||||
return
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"included documentation file is missing: {path}")
|
||||
seen.add(path)
|
||||
ordered.append(path)
|
||||
for line in path.read_text(encoding="utf-8", errors="strict").splitlines():
|
||||
match = INCLUDE_RE.match(line)
|
||||
if match:
|
||||
include_path = (path.parent / match.group(1)).resolve()
|
||||
if not include_path.is_file():
|
||||
missing.append({
|
||||
"document": relative_path(repo_root, path),
|
||||
"include": match.group(1),
|
||||
"resolved_path": relative_path(repo_root, include_path),
|
||||
})
|
||||
continue
|
||||
visit(include_path)
|
||||
|
||||
visit((repo_root / master).resolve())
|
||||
return ordered, missing
|
||||
|
||||
|
||||
def classify(path: Path, title: str) -> tuple[str, str]:
|
||||
lowered = "/".join(path.parts).lower()
|
||||
for token, domain in DOMAIN_RULES:
|
||||
if token in lowered:
|
||||
return domain, "mapped"
|
||||
title_lower = title.lower()
|
||||
if any(token in title_lower for token in ("g-code", "m-code", "o-code", "rs274")):
|
||||
return "gcode", "mapped"
|
||||
if any(token in title_lower for token in ("hardware", "driver", "mesa", "gpio")):
|
||||
return "real_hardware", "needs_review"
|
||||
if path.suffix.lower() in {".md", ".adoc"} and path.as_posix().startswith("docs/"):
|
||||
return "web_wasm_governance", "mapped"
|
||||
return "unclassified", "needs_review"
|
||||
|
||||
|
||||
def relative_path(repo_root: Path, path: Path) -> str:
|
||||
return path.resolve().relative_to(repo_root.resolve()).as_posix()
|
||||
|
||||
|
||||
def make_entries(repo_root: Path) -> tuple[list[dict[str, object]], dict[str, object]]:
|
||||
paths, missing_includes = include_graph(repo_root)
|
||||
for project_doc in PROJECT_DOCS:
|
||||
path = (repo_root / project_doc).resolve()
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"project migration document is missing: {project_doc}")
|
||||
if path not in paths:
|
||||
paths.append(path)
|
||||
|
||||
entries: list[dict[str, object]] = []
|
||||
report: dict[str, object] = {
|
||||
"schema": "document-capabilities-report.v1",
|
||||
"documents": len(paths),
|
||||
"headings": 0,
|
||||
"unclassified": [],
|
||||
"duplicate_ids": [],
|
||||
"empty_documents": [],
|
||||
"missing_includes": missing_includes,
|
||||
}
|
||||
used_ids: set[str] = set()
|
||||
for path in paths:
|
||||
relative = relative_path(repo_root, path)
|
||||
document_sha = sha256(path)
|
||||
headings = parse_headings(path)
|
||||
if not headings:
|
||||
report["empty_documents"].append(relative)
|
||||
headings = [Heading(1, path.stem, None, 1)]
|
||||
anchor_counts: dict[str, int] = {}
|
||||
parent_stack: list[tuple[int, str]] = []
|
||||
for heading in headings:
|
||||
base_anchor = heading.anchor or slugify(heading.title)
|
||||
anchor_counts[base_anchor] = anchor_counts.get(base_anchor, 0) + 1
|
||||
anchor = base_anchor if anchor_counts[base_anchor] == 1 else f"{base_anchor}-{anchor_counts[base_anchor]}"
|
||||
capability_id = f"doc:{relative}#{anchor}"
|
||||
if capability_id in used_ids:
|
||||
report["duplicate_ids"].append(capability_id)
|
||||
continue
|
||||
used_ids.add(capability_id)
|
||||
while parent_stack and parent_stack[-1][0] >= heading.level:
|
||||
parent_stack.pop()
|
||||
parent_anchor = parent_stack[-1][1] if parent_stack else None
|
||||
parent_stack.append((heading.level, anchor))
|
||||
domain, mapping_status = classify(path, heading.title)
|
||||
entry = {
|
||||
"capability_source_id": capability_id,
|
||||
"document": relative,
|
||||
"anchor": anchor,
|
||||
"title": heading.title,
|
||||
"domain": domain,
|
||||
"source_sha": document_sha,
|
||||
"line": heading.line,
|
||||
"level": heading.level,
|
||||
"parent_anchor": parent_anchor,
|
||||
"mapping_status": mapping_status,
|
||||
"sim_status": "planned",
|
||||
"web_status": "planned",
|
||||
}
|
||||
entries.append(entry)
|
||||
report["headings"] += 1
|
||||
if mapping_status != "mapped":
|
||||
report["unclassified"].append({"capability_source_id": capability_id, "domain": domain})
|
||||
entries.sort(key=lambda item: str(item["capability_source_id"]))
|
||||
return entries, report
|
||||
|
||||
|
||||
def write_json(path: Path, value: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def write_jsonl(path: Path, entries: Iterable[dict[str, object]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
for entry in entries:
|
||||
handle.write(json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(",", ":")))
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def generate(repo_root: Path, output: Path, report_path: Path) -> tuple[int, str]:
|
||||
entries, report = make_entries(repo_root)
|
||||
write_jsonl(output, entries)
|
||||
report["output_sha256"] = sha256(output)
|
||||
write_json(report_path, report)
|
||||
return len(entries), str(report["output_sha256"])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
|
||||
parser.add_argument("--output", type=Path, default=Path("coverage/document-capabilities.jsonl"))
|
||||
parser.add_argument("--report", type=Path, default=Path("coverage/document-capabilities.report.json"))
|
||||
args = parser.parse_args()
|
||||
root = args.repo_root.resolve()
|
||||
count, digest = generate(root, root / args.output, root / args.report)
|
||||
print(f"document_capabilities={count} sha256={digest}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user