242 lines
8.9 KiB
Python
242 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate a deterministic, source-addressable LinuxCNC document index.
|
|
|
|
The index is intentionally a source index, not a coverage claim. Every
|
|
heading in the Master Documentation include graph and every heading in the
|
|
project's migration documents becomes a planned capability candidate. Files
|
|
that cannot be classified are retained with ``needs_review`` rather than
|
|
silently discarded.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
|
|
MASTER = Path("linuxcnc/docs/src/Master_Documentation.adoc")
|
|
PROJECT_DOCS = (
|
|
Path("docs/linuxcnc-web-wasm-source-map.md"),
|
|
Path("docs/linuxcnc-web-wasm-simulation-coverage.md"),
|
|
)
|
|
INCLUDE_RE = re.compile(r"^\s*include::([^\[]+)\[.*\]\s*$")
|
|
ASCIIDOC_ANCHOR_RE = re.compile(r"^\s*\[\[([^,\]]+)(?:,[^\]]*)?\]\]\s*$")
|
|
ASCIIDOC_BLOCK_ID_RE = re.compile(r"^\s*\[#([^\]]+)\]\s*$")
|
|
ASCIIDOC_HEADING_RE = re.compile(r"^(={1,6})\s+(.+?)\s*$")
|
|
MARKDOWN_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$")
|
|
SLUG_RE = re.compile(r"[^a-z0-9]+")
|
|
|
|
DOMAIN_RULES = (
|
|
("getting-started", "getting_started"),
|
|
("user", "user"),
|
|
("config", "configuration"),
|
|
("hal", "hal"),
|
|
("ladder", "classicladder"),
|
|
("motion", "motion"),
|
|
("gcode", "gcode"),
|
|
("remap", "remap"),
|
|
("gui", "gui"),
|
|
("plasma", "plasma"),
|
|
("tooldatabase", "tooldata"),
|
|
("drivers", "real_hardware"),
|
|
("examples", "examples"),
|
|
("integrator", "integrator"),
|
|
("install", "installation"),
|
|
("source-highlight", "documentation"),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Heading:
|
|
level: int
|
|
title: str
|
|
anchor: str | None
|
|
line: int
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def slugify(title: str) -> str:
|
|
value = title.lower().replace("&", " and ")
|
|
value = SLUG_RE.sub("-", value).strip("-")
|
|
return value or "section"
|
|
|
|
|
|
def parse_headings(path: Path) -> list[Heading]:
|
|
lines = path.read_text(encoding="utf-8", errors="strict").splitlines()
|
|
headings: list[Heading] = []
|
|
pending_anchor: str | None = None
|
|
for line_number, line in enumerate(lines, 1):
|
|
anchor_match = ASCIIDOC_ANCHOR_RE.match(line) or ASCIIDOC_BLOCK_ID_RE.match(line)
|
|
if anchor_match:
|
|
pending_anchor = anchor_match.group(1).strip()
|
|
continue
|
|
match = ASCIIDOC_HEADING_RE.match(line) or MARKDOWN_HEADING_RE.match(line)
|
|
if not match:
|
|
continue
|
|
headings.append(Heading(len(match.group(1)), match.group(2).strip(), pending_anchor, line_number))
|
|
pending_anchor = None
|
|
return headings
|
|
|
|
|
|
def include_graph(repo_root: Path, master: Path = MASTER) -> tuple[list[Path], list[dict[str, str]]]:
|
|
ordered: list[Path] = []
|
|
seen: set[Path] = set()
|
|
missing: list[dict[str, str]] = []
|
|
|
|
def visit(path: Path) -> None:
|
|
path = path.resolve()
|
|
if path in seen:
|
|
return
|
|
if not path.is_file():
|
|
raise FileNotFoundError(f"included documentation file is missing: {path}")
|
|
seen.add(path)
|
|
ordered.append(path)
|
|
for line in path.read_text(encoding="utf-8", errors="strict").splitlines():
|
|
match = INCLUDE_RE.match(line)
|
|
if match:
|
|
include_path = (path.parent / match.group(1)).resolve()
|
|
if not include_path.is_file():
|
|
missing.append({
|
|
"document": relative_path(repo_root, path),
|
|
"include": match.group(1),
|
|
"resolved_path": relative_path(repo_root, include_path),
|
|
})
|
|
continue
|
|
visit(include_path)
|
|
|
|
visit((repo_root / master).resolve())
|
|
return ordered, missing
|
|
|
|
|
|
def classify(path: Path, title: str) -> tuple[str, str]:
|
|
lowered = "/".join(path.parts).lower()
|
|
for token, domain in DOMAIN_RULES:
|
|
if token in lowered:
|
|
return domain, "mapped"
|
|
title_lower = title.lower()
|
|
if any(token in title_lower for token in ("g-code", "m-code", "o-code", "rs274")):
|
|
return "gcode", "mapped"
|
|
if any(token in title_lower for token in ("hardware", "driver", "mesa", "gpio")):
|
|
return "real_hardware", "needs_review"
|
|
if path.suffix.lower() in {".md", ".adoc"} and path.as_posix().startswith("docs/"):
|
|
return "web_wasm_governance", "mapped"
|
|
return "unclassified", "needs_review"
|
|
|
|
|
|
def relative_path(repo_root: Path, path: Path) -> str:
|
|
return path.resolve().relative_to(repo_root.resolve()).as_posix()
|
|
|
|
|
|
def make_entries(repo_root: Path) -> tuple[list[dict[str, object]], dict[str, object]]:
|
|
paths, missing_includes = include_graph(repo_root)
|
|
for project_doc in PROJECT_DOCS:
|
|
path = (repo_root / project_doc).resolve()
|
|
if not path.is_file():
|
|
raise FileNotFoundError(f"project migration document is missing: {project_doc}")
|
|
if path not in paths:
|
|
paths.append(path)
|
|
|
|
entries: list[dict[str, object]] = []
|
|
report: dict[str, object] = {
|
|
"schema": "document-capabilities-report.v1",
|
|
"documents": len(paths),
|
|
"headings": 0,
|
|
"unclassified": [],
|
|
"duplicate_ids": [],
|
|
"empty_documents": [],
|
|
"missing_includes": missing_includes,
|
|
}
|
|
used_ids: set[str] = set()
|
|
for path in paths:
|
|
relative = relative_path(repo_root, path)
|
|
document_sha = sha256(path)
|
|
headings = parse_headings(path)
|
|
if not headings:
|
|
report["empty_documents"].append(relative)
|
|
headings = [Heading(1, path.stem, None, 1)]
|
|
anchor_counts: dict[str, int] = {}
|
|
parent_stack: list[tuple[int, str]] = []
|
|
for heading in headings:
|
|
base_anchor = heading.anchor or slugify(heading.title)
|
|
anchor_counts[base_anchor] = anchor_counts.get(base_anchor, 0) + 1
|
|
anchor = base_anchor if anchor_counts[base_anchor] == 1 else f"{base_anchor}-{anchor_counts[base_anchor]}"
|
|
capability_id = f"doc:{relative}#{anchor}"
|
|
if capability_id in used_ids:
|
|
report["duplicate_ids"].append(capability_id)
|
|
continue
|
|
used_ids.add(capability_id)
|
|
while parent_stack and parent_stack[-1][0] >= heading.level:
|
|
parent_stack.pop()
|
|
parent_anchor = parent_stack[-1][1] if parent_stack else None
|
|
parent_stack.append((heading.level, anchor))
|
|
domain, mapping_status = classify(path, heading.title)
|
|
entry = {
|
|
"capability_source_id": capability_id,
|
|
"document": relative,
|
|
"anchor": anchor,
|
|
"title": heading.title,
|
|
"domain": domain,
|
|
"source_sha": document_sha,
|
|
"line": heading.line,
|
|
"level": heading.level,
|
|
"parent_anchor": parent_anchor,
|
|
"mapping_status": mapping_status,
|
|
"sim_status": "planned",
|
|
"web_status": "planned",
|
|
}
|
|
entries.append(entry)
|
|
report["headings"] += 1
|
|
if mapping_status != "mapped":
|
|
report["unclassified"].append({"capability_source_id": capability_id, "domain": domain})
|
|
entries.sort(key=lambda item: str(item["capability_source_id"]))
|
|
return entries, report
|
|
|
|
|
|
def write_json(path: Path, value: object) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def write_jsonl(path: Path, entries: Iterable[dict[str, object]]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
|
for entry in entries:
|
|
handle.write(json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(",", ":")))
|
|
handle.write("\n")
|
|
|
|
|
|
def generate(repo_root: Path, output: Path, report_path: Path) -> tuple[int, str]:
|
|
entries, report = make_entries(repo_root)
|
|
write_jsonl(output, entries)
|
|
report["output_sha256"] = sha256(output)
|
|
write_json(report_path, report)
|
|
return len(entries), str(report["output_sha256"])
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument("--output", type=Path, default=Path("coverage/document-capabilities.jsonl"))
|
|
parser.add_argument("--report", type=Path, default=Path("coverage/document-capabilities.report.json"))
|
|
args = parser.parse_args()
|
|
root = args.repo_root.resolve()
|
|
count, digest = generate(root, root / args.output, root / args.report)
|
|
print(f"document_capabilities={count} sha256={digest}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|