Initial LinuxCNC WebAssembly workspace snapshot

scripts/generate_source_symbols.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""Generate deterministic source symbol and HAL component entry indexes."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+from pathlib import Path
+from typing import Iterable
+
+
+ROOTS = (
+    Path("linuxcnc/src/emc"),
+    Path("linuxcnc/src/hal"),
+    Path("linuxcnc/lib/python"),
+)
+EXTENSIONS = {
+    ".c": "c", ".h": "c", ".cc": "cpp", ".hh": "cpp", ".cpp": "cpp",
+    ".hpp": "cpp", ".cxx": "cpp", ".hxx": "cpp", ".py": "python", ".comp": "hal_comp",
+}
+IDENT = r"[A-Za-z_][A-Za-z0-9_]*"
+KEYWORDS = {"if", "for", "while", "switch", "catch", "sizeof", "return", "do"}
+DEFINE_RE = re.compile(rf"^\s*#\s*define\s+({IDENT})\b")
+C_FUNCTION_RE = re.compile(
+    rf"^\s*(?:(?:static|inline|extern|const|volatile|unsigned|signed|long|short|struct|class|enum|[A-Za-z_][\w:*&<>]*)\s+)+({IDENT})\s*\([^;{{}}]*\)\s*(?:const\s*)?(?:\{{.*|;)?\s*$"
+)
+C_TYPE_RE = re.compile(rf"^\s*(?:typedef\s+)?(?:struct|class|enum|union)\s+({IDENT})\b")
+PY_DEF_RE = re.compile(rf"^\s*(async\s+)?def\s+({IDENT})\s*\(")
+PY_CLASS_RE = re.compile(rf"^\s*class\s+({IDENT})\b")
+COMP_PATTERNS = (
+    ("component", re.compile(rf"^\s*component\s+({IDENT})\b")),
+    ("function", re.compile(rf"^\s*function\s+({IDENT})\b")),
+    ("option", re.compile(rf"^\s*option\s+({IDENT})\b")),
+    ("pin", re.compile(rf"^\s*pin\s+(?:in|out|io)\s+{IDENT}\s+({IDENT})\b")),
+    ("param", re.compile(rf"^\s*param\s+(?:r|rw)\s+{IDENT}\s+({IDENT})\b")),
+    ("variable", re.compile(rf"^\s*variable\s+{IDENT}\s+({IDENT})\b")),
+)
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def files(repo_root: Path) -> Iterable[Path]:
+    for root in ROOTS:
+        absolute = repo_root / root
+        if not absolute.is_dir():
+            raise FileNotFoundError(f"required symbol root is missing: {root}")
+        for path in sorted(path for path in absolute.rglob("*") if path.is_file() and path.suffix.lower() in EXTENSIONS):
+            yield path
+
+
+def relative(repo_root: Path, path: Path) -> str:
+    return path.resolve().relative_to(repo_root.resolve()).as_posix()
+
+
+def add_symbol(rows: list[dict[str, object]], repo_root: Path, path: Path, language: str, symbol: str,
+              symbol_kind: str, line: int, source_sha: str, **extra: object) -> None:
+    rows.append({
+        "symbol_source_id": f"symbol:{relative(repo_root, path)}:{line}:{symbol_kind}:{symbol}",
+        "path": relative(repo_root, path),
+        "language": language,
+        "symbol": symbol,
+        "symbol_kind": symbol_kind,
+        "line": line,
+        "source_sha": source_sha,
+        "parse_status": "parsed",
+        **extra,
+    })
+
+
+def parse_file(repo_root: Path, path: Path) -> tuple[list[dict[str, object]], dict[str, object]]:
+    language = EXTENSIONS[path.suffix.lower()]
+    source_sha = sha256(path)
+    rows: list[dict[str, object]] = []
+    report: dict[str, object] = {"path": relative(repo_root, path), "language": language, "symbols": 0, "parse_status": "parsed"}
+    try:
+        lines = path.read_text(encoding="utf-8", errors="strict").splitlines()
+    except UnicodeDecodeError as exc:
+        report.update({"parse_status": "decode_error", "error": str(exc)})
+        return rows, report
+    for line_number, line in enumerate(lines, 1):
+        match = DEFINE_RE.match(line) if language in {"c", "cpp"} else None
+        if match:
+            add_symbol(rows, repo_root, path, language, match.group(1), "macro", line_number, source_sha)
+            continue
+        if language == "python":
+            match = PY_DEF_RE.match(line) or PY_CLASS_RE.match(line)
+            if match:
+                symbol = match.group(2) if match.re is PY_DEF_RE else match.group(1)
+                kind = "async_function" if match.re is PY_DEF_RE and match.group(1) else ("function" if match.re is PY_DEF_RE else "class")
+                add_symbol(rows, repo_root, path, language, symbol, kind, line_number, source_sha)
+            continue
+        if language == "hal_comp":
+            for kind, pattern in COMP_PATTERNS:
+                match = pattern.match(line)
+                if match:
+                    add_symbol(rows, repo_root, path, language, match.group(1), kind, line_number, source_sha)
+                    break
+            continue
+        match = C_TYPE_RE.match(line)
+        if match:
+            add_symbol(rows, repo_root, path, language, match.group(1), "type", line_number, source_sha)
+            continue
+        match = C_FUNCTION_RE.match(line)
+        if match:
+            symbol = match.group(1)
+            if symbol not in KEYWORDS:
+                add_symbol(rows, repo_root, path, language, symbol, "function", line_number, source_sha)
+    report["symbols"] = len(rows)
+    return rows, report
+
+
+def generate(repo_root: Path, output: Path, report_path: Path) -> tuple[int, str]:
+    rows: list[dict[str, object]] = []
+    file_reports: list[dict[str, object]] = []
+    for path in files(repo_root):
+        parsed, report = parse_file(repo_root, path)
+        rows.extend(parsed)
+        file_reports.append(report)
+    rows.sort(key=lambda row: str(row["symbol_source_id"]))
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with output.open("w", encoding="utf-8", newline="\n") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n")
+    digest = sha256(output)
+    report = {
+        "schema": "source-symbols-report.v1",
+        "files": len(file_reports),
+        "symbols": len(rows),
+        "files_without_symbols": sorted(report["path"] for report in file_reports if report["symbols"] == 0),
+        "decode_errors": [report for report in file_reports if report["parse_status"] != "parsed"],
+        "output_sha256": digest,
+        "file_reports": file_reports,
+    }
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    report_path.write_text(json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8")
+    return len(rows), digest
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
+    parser.add_argument("--output", type=Path, default=Path("coverage/source-symbols.jsonl"))
+    parser.add_argument("--report", type=Path, default=Path("coverage/source-symbols.report.json"))
+    args = parser.parse_args()
+    count, digest = generate(args.repo_root.resolve(), args.repo_root.resolve() / args.output, args.repo_root.resolve() / args.report)
+    print(f"source_symbols={count} sha256={digest}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())