#!/usr/bin/env python3 """Generate deterministic source symbol and HAL component entry indexes.""" from __future__ import annotations import argparse import hashlib import json import re from pathlib import Path from typing import Iterable ROOTS = ( Path("linuxcnc/src/emc"), Path("linuxcnc/src/hal"), Path("linuxcnc/lib/python"), ) EXTENSIONS = { ".c": "c", ".h": "c", ".cc": "cpp", ".hh": "cpp", ".cpp": "cpp", ".hpp": "cpp", ".cxx": "cpp", ".hxx": "cpp", ".py": "python", ".comp": "hal_comp", } IDENT = r"[A-Za-z_][A-Za-z0-9_]*" KEYWORDS = {"if", "for", "while", "switch", "catch", "sizeof", "return", "do"} DEFINE_RE = re.compile(rf"^\s*#\s*define\s+({IDENT})\b") C_FUNCTION_RE = re.compile( rf"^\s*(?:(?:static|inline|extern|const|volatile|unsigned|signed|long|short|struct|class|enum|[A-Za-z_][\w:*&<>]*)\s+)+({IDENT})\s*\([^;{{}}]*\)\s*(?:const\s*)?(?:\{{.*|;)?\s*$" ) C_TYPE_RE = re.compile(rf"^\s*(?:typedef\s+)?(?:struct|class|enum|union)\s+({IDENT})\b") PY_DEF_RE = re.compile(rf"^\s*(async\s+)?def\s+({IDENT})\s*\(") PY_CLASS_RE = re.compile(rf"^\s*class\s+({IDENT})\b") COMP_PATTERNS = ( ("component", re.compile(rf"^\s*component\s+({IDENT})\b")), ("function", re.compile(rf"^\s*function\s+({IDENT})\b")), ("option", re.compile(rf"^\s*option\s+({IDENT})\b")), ("pin", re.compile(rf"^\s*pin\s+(?:in|out|io)\s+{IDENT}\s+({IDENT})\b")), ("param", re.compile(rf"^\s*param\s+(?:r|rw)\s+{IDENT}\s+({IDENT})\b")), ("variable", re.compile(rf"^\s*variable\s+{IDENT}\s+({IDENT})\b")), ) def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def files(repo_root: Path) -> Iterable[Path]: for root in ROOTS: absolute = repo_root / root if not absolute.is_dir(): raise FileNotFoundError(f"required symbol root is missing: {root}") for path in sorted(path for path in absolute.rglob("*") if path.is_file() and path.suffix.lower() in EXTENSIONS): yield path def relative(repo_root: Path, path: Path) -> str: return path.resolve().relative_to(repo_root.resolve()).as_posix() def add_symbol(rows: list[dict[str, object]], repo_root: Path, path: Path, language: str, symbol: str, symbol_kind: str, line: int, source_sha: str, **extra: object) -> None: rows.append({ "symbol_source_id": f"symbol:{relative(repo_root, path)}:{line}:{symbol_kind}:{symbol}", "path": relative(repo_root, path), "language": language, "symbol": symbol, "symbol_kind": symbol_kind, "line": line, "source_sha": source_sha, "parse_status": "parsed", **extra, }) def parse_file(repo_root: Path, path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: language = EXTENSIONS[path.suffix.lower()] source_sha = sha256(path) rows: list[dict[str, object]] = [] report: dict[str, object] = {"path": relative(repo_root, path), "language": language, "symbols": 0, "parse_status": "parsed"} try: lines = path.read_text(encoding="utf-8", errors="strict").splitlines() except UnicodeDecodeError as exc: report.update({"parse_status": "decode_error", "error": str(exc)}) return rows, report for line_number, line in enumerate(lines, 1): match = DEFINE_RE.match(line) if language in {"c", "cpp"} else None if match: add_symbol(rows, repo_root, path, language, match.group(1), "macro", line_number, source_sha) continue if language == "python": match = PY_DEF_RE.match(line) or PY_CLASS_RE.match(line) if match: symbol = match.group(2) if match.re is PY_DEF_RE else match.group(1) kind = "async_function" if match.re is PY_DEF_RE and match.group(1) else ("function" if match.re is PY_DEF_RE else "class") add_symbol(rows, repo_root, path, language, symbol, kind, line_number, source_sha) continue if language == "hal_comp": for kind, pattern in COMP_PATTERNS: match = pattern.match(line) if match: add_symbol(rows, repo_root, path, language, match.group(1), kind, line_number, source_sha) break continue match = C_TYPE_RE.match(line) if match: add_symbol(rows, repo_root, path, language, match.group(1), "type", line_number, source_sha) continue match = C_FUNCTION_RE.match(line) if match: symbol = match.group(1) if symbol not in KEYWORDS: add_symbol(rows, repo_root, path, language, symbol, "function", line_number, source_sha) report["symbols"] = len(rows) return rows, report def generate(repo_root: Path, output: Path, report_path: Path) -> tuple[int, str]: rows: list[dict[str, object]] = [] file_reports: list[dict[str, object]] = [] for path in files(repo_root): parsed, report = parse_file(repo_root, path) rows.extend(parsed) file_reports.append(report) rows.sort(key=lambda row: str(row["symbol_source_id"])) output.parent.mkdir(parents=True, exist_ok=True) with output.open("w", encoding="utf-8", newline="\n") as handle: for row in rows: handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n") digest = sha256(output) report = { "schema": "source-symbols-report.v1", "files": len(file_reports), "symbols": len(rows), "files_without_symbols": sorted(report["path"] for report in file_reports if report["symbols"] == 0), "decode_errors": [report for report in file_reports if report["parse_status"] != "parsed"], "output_sha256": digest, "file_reports": file_reports, } report_path.parent.mkdir(parents=True, exist_ok=True) report_path.write_text(json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8") return len(rows), digest def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1]) parser.add_argument("--output", type=Path, default=Path("coverage/source-symbols.jsonl")) parser.add_argument("--report", type=Path, default=Path("coverage/source-symbols.report.json")) args = parser.parse_args() count, digest = generate(args.repo_root.resolve(), args.repo_root.resolve() / args.output, args.repo_root.resolve() / args.report) print(f"source_symbols={count} sha256={digest}") return 0 if __name__ == "__main__": raise SystemExit(main())