160 lines
6.7 KiB
Python
160 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate deterministic source symbol and HAL component entry indexes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
|
|
ROOTS = (
|
|
Path("linuxcnc/src/emc"),
|
|
Path("linuxcnc/src/hal"),
|
|
Path("linuxcnc/lib/python"),
|
|
)
|
|
EXTENSIONS = {
|
|
".c": "c", ".h": "c", ".cc": "cpp", ".hh": "cpp", ".cpp": "cpp",
|
|
".hpp": "cpp", ".cxx": "cpp", ".hxx": "cpp", ".py": "python", ".comp": "hal_comp",
|
|
}
|
|
IDENT = r"[A-Za-z_][A-Za-z0-9_]*"
|
|
KEYWORDS = {"if", "for", "while", "switch", "catch", "sizeof", "return", "do"}
|
|
DEFINE_RE = re.compile(rf"^\s*#\s*define\s+({IDENT})\b")
|
|
C_FUNCTION_RE = re.compile(
|
|
rf"^\s*(?:(?:static|inline|extern|const|volatile|unsigned|signed|long|short|struct|class|enum|[A-Za-z_][\w:*&<>]*)\s+)+({IDENT})\s*\([^;{{}}]*\)\s*(?:const\s*)?(?:\{{.*|;)?\s*$"
|
|
)
|
|
C_TYPE_RE = re.compile(rf"^\s*(?:typedef\s+)?(?:struct|class|enum|union)\s+({IDENT})\b")
|
|
PY_DEF_RE = re.compile(rf"^\s*(async\s+)?def\s+({IDENT})\s*\(")
|
|
PY_CLASS_RE = re.compile(rf"^\s*class\s+({IDENT})\b")
|
|
COMP_PATTERNS = (
|
|
("component", re.compile(rf"^\s*component\s+({IDENT})\b")),
|
|
("function", re.compile(rf"^\s*function\s+({IDENT})\b")),
|
|
("option", re.compile(rf"^\s*option\s+({IDENT})\b")),
|
|
("pin", re.compile(rf"^\s*pin\s+(?:in|out|io)\s+{IDENT}\s+({IDENT})\b")),
|
|
("param", re.compile(rf"^\s*param\s+(?:r|rw)\s+{IDENT}\s+({IDENT})\b")),
|
|
("variable", re.compile(rf"^\s*variable\s+{IDENT}\s+({IDENT})\b")),
|
|
)
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def files(repo_root: Path) -> Iterable[Path]:
|
|
for root in ROOTS:
|
|
absolute = repo_root / root
|
|
if not absolute.is_dir():
|
|
raise FileNotFoundError(f"required symbol root is missing: {root}")
|
|
for path in sorted(path for path in absolute.rglob("*") if path.is_file() and path.suffix.lower() in EXTENSIONS):
|
|
yield path
|
|
|
|
|
|
def relative(repo_root: Path, path: Path) -> str:
|
|
return path.resolve().relative_to(repo_root.resolve()).as_posix()
|
|
|
|
|
|
def add_symbol(rows: list[dict[str, object]], repo_root: Path, path: Path, language: str, symbol: str,
|
|
symbol_kind: str, line: int, source_sha: str, **extra: object) -> None:
|
|
rows.append({
|
|
"symbol_source_id": f"symbol:{relative(repo_root, path)}:{line}:{symbol_kind}:{symbol}",
|
|
"path": relative(repo_root, path),
|
|
"language": language,
|
|
"symbol": symbol,
|
|
"symbol_kind": symbol_kind,
|
|
"line": line,
|
|
"source_sha": source_sha,
|
|
"parse_status": "parsed",
|
|
**extra,
|
|
})
|
|
|
|
|
|
def parse_file(repo_root: Path, path: Path) -> tuple[list[dict[str, object]], dict[str, object]]:
|
|
language = EXTENSIONS[path.suffix.lower()]
|
|
source_sha = sha256(path)
|
|
rows: list[dict[str, object]] = []
|
|
report: dict[str, object] = {"path": relative(repo_root, path), "language": language, "symbols": 0, "parse_status": "parsed"}
|
|
try:
|
|
lines = path.read_text(encoding="utf-8", errors="strict").splitlines()
|
|
except UnicodeDecodeError as exc:
|
|
report.update({"parse_status": "decode_error", "error": str(exc)})
|
|
return rows, report
|
|
for line_number, line in enumerate(lines, 1):
|
|
match = DEFINE_RE.match(line) if language in {"c", "cpp"} else None
|
|
if match:
|
|
add_symbol(rows, repo_root, path, language, match.group(1), "macro", line_number, source_sha)
|
|
continue
|
|
if language == "python":
|
|
match = PY_DEF_RE.match(line) or PY_CLASS_RE.match(line)
|
|
if match:
|
|
symbol = match.group(2) if match.re is PY_DEF_RE else match.group(1)
|
|
kind = "async_function" if match.re is PY_DEF_RE and match.group(1) else ("function" if match.re is PY_DEF_RE else "class")
|
|
add_symbol(rows, repo_root, path, language, symbol, kind, line_number, source_sha)
|
|
continue
|
|
if language == "hal_comp":
|
|
for kind, pattern in COMP_PATTERNS:
|
|
match = pattern.match(line)
|
|
if match:
|
|
add_symbol(rows, repo_root, path, language, match.group(1), kind, line_number, source_sha)
|
|
break
|
|
continue
|
|
match = C_TYPE_RE.match(line)
|
|
if match:
|
|
add_symbol(rows, repo_root, path, language, match.group(1), "type", line_number, source_sha)
|
|
continue
|
|
match = C_FUNCTION_RE.match(line)
|
|
if match:
|
|
symbol = match.group(1)
|
|
if symbol not in KEYWORDS:
|
|
add_symbol(rows, repo_root, path, language, symbol, "function", line_number, source_sha)
|
|
report["symbols"] = len(rows)
|
|
return rows, report
|
|
|
|
|
|
def generate(repo_root: Path, output: Path, report_path: Path) -> tuple[int, str]:
|
|
rows: list[dict[str, object]] = []
|
|
file_reports: list[dict[str, object]] = []
|
|
for path in files(repo_root):
|
|
parsed, report = parse_file(repo_root, path)
|
|
rows.extend(parsed)
|
|
file_reports.append(report)
|
|
rows.sort(key=lambda row: str(row["symbol_source_id"]))
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
with output.open("w", encoding="utf-8", newline="\n") as handle:
|
|
for row in rows:
|
|
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n")
|
|
digest = sha256(output)
|
|
report = {
|
|
"schema": "source-symbols-report.v1",
|
|
"files": len(file_reports),
|
|
"symbols": len(rows),
|
|
"files_without_symbols": sorted(report["path"] for report in file_reports if report["symbols"] == 0),
|
|
"decode_errors": [report for report in file_reports if report["parse_status"] != "parsed"],
|
|
"output_sha256": digest,
|
|
"file_reports": file_reports,
|
|
}
|
|
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
report_path.write_text(json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8")
|
|
return len(rows), digest
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument("--output", type=Path, default=Path("coverage/source-symbols.jsonl"))
|
|
parser.add_argument("--report", type=Path, default=Path("coverage/source-symbols.report.json"))
|
|
args = parser.parse_args()
|
|
count, digest = generate(args.repo_root.resolve(), args.repo_root.resolve() / args.output, args.repo_root.resolve() / args.report)
|
|
print(f"source_symbols={count} sha256={digest}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|