Initial LinuxCNC WebAssembly workspace snapshot
This commit is contained in:
240
scripts/validate_baseline.py
Executable file
240
scripts/validate_baseline.py
Executable file
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the M0 baseline and fail closed on unassigned coverage entries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
|
||||
REQUIRED_MANIFEST_FIELDS = {
|
||||
"id", "source", "source_sha", "domain", "kind", "dependencies", "inputs",
|
||||
"outputs", "errors", "sim_status", "web_status", "test_id", "oracle", "golden",
|
||||
"difference", "coverage_dimensions", "context_id", "exclusion_reason",
|
||||
}
|
||||
ALLOWED_STATUS = {"planned", "in_progress", "implemented", "verified", "accepted", "blocked", "REAL-EXCLUDED"}
|
||||
|
||||
|
||||
def read_jsonl(path: Path) -> list[dict[str, object]]:
|
||||
rows: list[dict[str, object]] = []
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line_number, line in enumerate(handle, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
value = json.loads(line)
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{path}:{line_number}: expected object")
|
||||
rows.append(value)
|
||||
return rows
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def validate(repo_root: Path, lock_path: Path, source_path: Path, manifest_path: Path) -> list[str]:
|
||||
errors: list[str] = []
|
||||
lock = json.loads(lock_path.read_text(encoding="utf-8"))
|
||||
for field in ("context_id", "linuxcnc_version", "linuxcnc_source_sha", "emscripten_version", "schema_versions"):
|
||||
if not lock.get(field):
|
||||
errors.append(f"context lock missing {field}")
|
||||
sources = read_jsonl(source_path)
|
||||
manifests = read_jsonl(manifest_path)
|
||||
if lock.get("source_index_sha256") != sha256(source_path):
|
||||
errors.append("source index hash does not match context lock")
|
||||
if lock.get("manifest_seed_sha256") != sha256(manifest_path):
|
||||
errors.append("manifest seed hash does not match context lock")
|
||||
document_index_path = repo_root / "coverage/document-capabilities.jsonl"
|
||||
document_index_hash = lock.get("document_capabilities_sha256")
|
||||
if document_index_hash:
|
||||
if not document_index_path.is_file():
|
||||
errors.append("document capabilities index is missing")
|
||||
elif document_index_hash != sha256(document_index_path):
|
||||
errors.append("document capabilities index hash does not match context lock")
|
||||
version_path = repo_root / "linuxcnc/VERSION"
|
||||
if lock.get("linuxcnc_version") != version_path.read_text(encoding="utf-8").strip():
|
||||
errors.append("linuxcnc version does not match context lock")
|
||||
try:
|
||||
source_sha = subprocess.check_output(
|
||||
["git", "-C", str(repo_root / "linuxcnc"), "rev-parse", "HEAD"],
|
||||
text=True,
|
||||
).strip()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
source_sha = None
|
||||
if source_sha and lock.get("linuxcnc_source_sha") != source_sha:
|
||||
errors.append("linuxcnc source commit does not match context lock")
|
||||
inventory_count = lock.get("inventory_counts", {})
|
||||
if isinstance(inventory_count, dict):
|
||||
if inventory_count.get("total") != len(sources):
|
||||
errors.append("source index count does not match context lock")
|
||||
if inventory_count.get("source_index") != len(sources):
|
||||
errors.append("source_index count does not match context lock")
|
||||
if inventory_count.get("manifest_seed") != len(manifests):
|
||||
errors.append("manifest_seed count does not match context lock")
|
||||
if document_index_hash and inventory_count.get("document_capabilities") is not None:
|
||||
if inventory_count.get("document_capabilities") != sum(
|
||||
1 for line in document_index_path.read_text(encoding="utf-8").splitlines() if line.strip()
|
||||
):
|
||||
errors.append("document_capabilities count does not match context lock")
|
||||
symbols_path = repo_root / "coverage/source-symbols.jsonl"
|
||||
symbols_hash = lock.get("source_symbols_sha256")
|
||||
if symbols_hash:
|
||||
if not symbols_path.is_file():
|
||||
errors.append("source symbols index is missing")
|
||||
elif symbols_hash != sha256(symbols_path):
|
||||
errors.append("source symbols index hash does not match context lock")
|
||||
if isinstance(inventory_count, dict) and symbols_hash and inventory_count.get("source_symbols") is not None:
|
||||
if inventory_count.get("source_symbols") != sum(1 for line in symbols_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
||||
errors.append("source_symbols count does not match context lock")
|
||||
resources_path = repo_root / "coverage/config-resources.jsonl"
|
||||
resources_hash = lock.get("config_resources_sha256")
|
||||
if resources_hash:
|
||||
if not resources_path.is_file():
|
||||
errors.append("config resources index is missing")
|
||||
elif resources_hash != sha256(resources_path):
|
||||
errors.append("config resources index hash does not match context lock")
|
||||
if isinstance(inventory_count, dict) and resources_hash and inventory_count.get("config_resources") is not None:
|
||||
if inventory_count.get("config_resources") != sum(1 for line in resources_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
||||
errors.append("config_resources count does not match context lock")
|
||||
test_entries_path = repo_root / "coverage/test-entries.jsonl"
|
||||
test_entries_hash = lock.get("test_entries_sha256")
|
||||
if test_entries_hash:
|
||||
if not test_entries_path.is_file():
|
||||
errors.append("test entries index is missing")
|
||||
elif test_entries_hash != sha256(test_entries_path):
|
||||
errors.append("test entries index hash does not match context lock")
|
||||
if isinstance(inventory_count, dict) and test_entries_hash and inventory_count.get("test_entries") is not None:
|
||||
if inventory_count.get("test_entries") != sum(1 for line in test_entries_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
||||
errors.append("test_entries count does not match context lock")
|
||||
normalized_path = repo_root / "coverage/normalized-sources.jsonl"
|
||||
normalized_hash = lock.get("normalized_sources_sha256")
|
||||
if normalized_hash:
|
||||
if not normalized_path.is_file():
|
||||
errors.append("normalized sources index is missing")
|
||||
elif normalized_hash != sha256(normalized_path):
|
||||
errors.append("normalized sources index hash does not match context lock")
|
||||
if isinstance(inventory_count, dict) and normalized_hash and inventory_count.get("normalized_sources") is not None:
|
||||
if inventory_count.get("normalized_sources") != sum(1 for line in normalized_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
||||
errors.append("normalized_sources count does not match context lock")
|
||||
closure_path = repo_root / "coverage/dependency-closure.jsonl"
|
||||
closure_hash = lock.get("dependency_closure_sha256")
|
||||
if closure_hash:
|
||||
if not closure_path.is_file():
|
||||
errors.append("dependency closure index is missing")
|
||||
elif closure_hash != sha256(closure_path):
|
||||
errors.append("dependency closure index hash does not match context lock")
|
||||
if isinstance(inventory_count, dict) and closure_hash and inventory_count.get("dependency_edges") is not None:
|
||||
if inventory_count.get("dependency_edges") != sum(1 for line in closure_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
||||
errors.append("dependency_edges count does not match context lock")
|
||||
golden_schema_path = repo_root / "coverage/golden-schema.json"
|
||||
golden_schema_hash = lock.get("golden_schema_sha256")
|
||||
if golden_schema_hash:
|
||||
if not golden_schema_path.is_file():
|
||||
errors.append("golden schema is missing")
|
||||
elif golden_schema_hash != sha256(golden_schema_path):
|
||||
errors.append("golden schema hash does not match context lock")
|
||||
runner_report_path = repo_root / "coverage/native-runner.report.json"
|
||||
runner_report_hash = lock.get("native_runner_report_sha256")
|
||||
if runner_report_hash:
|
||||
if not runner_report_path.is_file():
|
||||
errors.append("native runner report is missing")
|
||||
elif runner_report_hash != sha256(runner_report_path):
|
||||
errors.append("native runner report hash does not match context lock")
|
||||
recovery_report_path = repo_root / "coverage/workspace-recovery.report.json"
|
||||
recovery_report_hash = lock.get("workspace_recovery_report_sha256")
|
||||
if recovery_report_hash:
|
||||
if not recovery_report_path.is_file():
|
||||
errors.append("workspace recovery report is missing")
|
||||
elif recovery_report_hash != sha256(recovery_report_path):
|
||||
errors.append("workspace recovery report hash does not match context lock")
|
||||
runtime_report_path = repo_root / "coverage/runtime-mapping.report.json"
|
||||
runtime_report_hash = lock.get("runtime_mapping_report_sha256")
|
||||
if runtime_report_hash:
|
||||
if not runtime_report_path.is_file():
|
||||
errors.append("runtime mapping report is missing")
|
||||
elif runtime_report_hash != sha256(runtime_report_path):
|
||||
errors.append("runtime mapping report hash does not match context lock")
|
||||
python_manifest_path = repo_root / "python-runtime/manifest.json"
|
||||
python_manifest_hash = lock.get("python_runtime_manifest_sha256")
|
||||
if python_manifest_hash:
|
||||
if not python_manifest_path.is_file():
|
||||
errors.append("python runtime manifest is missing")
|
||||
elif python_manifest_hash != sha256(python_manifest_path):
|
||||
errors.append("python runtime manifest hash does not match context lock")
|
||||
python_report_path = repo_root / "coverage/python-runtime.report.json"
|
||||
python_report_hash = lock.get("python_runtime_report_sha256")
|
||||
if python_report_hash:
|
||||
if not python_report_path.is_file():
|
||||
errors.append("python runtime report is missing")
|
||||
elif python_report_hash != sha256(python_report_path):
|
||||
errors.append("python runtime report hash does not match context lock")
|
||||
hal_report_path = repo_root / "coverage/hal-runtime.report.json"
|
||||
hal_report_hash = lock.get("hal_runtime_report_sha256")
|
||||
if hal_report_hash:
|
||||
if not hal_report_path.is_file():
|
||||
errors.append("hal runtime report is missing")
|
||||
elif hal_report_hash != sha256(hal_report_path):
|
||||
errors.append("hal runtime report hash does not match context lock")
|
||||
posemath_report_path = repo_root / "coverage/posemath-runtime.report.json"
|
||||
posemath_report_hash = lock.get("posemath_runtime_report_sha256")
|
||||
if posemath_report_hash:
|
||||
if not posemath_report_path.is_file():
|
||||
errors.append("posemath runtime report is missing")
|
||||
elif posemath_report_hash != sha256(posemath_report_path):
|
||||
errors.append("posemath runtime report hash does not match context lock")
|
||||
source_ids = [str(row.get("source_id", "")) for row in sources]
|
||||
if len(source_ids) != len(set(source_ids)):
|
||||
errors.append("source index contains duplicate source_id")
|
||||
manifest_ids = [str(row.get("id", "")) for row in manifests]
|
||||
if len(manifest_ids) != len(set(manifest_ids)):
|
||||
errors.append("manifest contains duplicate id")
|
||||
source_sha_by_id = {str(row["source_id"]): str(row["sha256"]) for row in sources}
|
||||
manifest_sources = {str(row.get("source", "")) for row in manifests}
|
||||
missing = sorted(set(source_ids) - manifest_sources)
|
||||
if missing:
|
||||
errors.append(f"manifest has {len(missing)} orphan sources; first={missing[0]}")
|
||||
extra = sorted(manifest_sources - set(source_ids))
|
||||
if extra:
|
||||
errors.append(f"manifest has {len(extra)} unknown sources; first={extra[0]}")
|
||||
for row in manifests:
|
||||
missing_fields = REQUIRED_MANIFEST_FIELDS - row.keys()
|
||||
if missing_fields:
|
||||
errors.append(f"{row.get('id')}: missing fields {sorted(missing_fields)}")
|
||||
source_id = str(row.get("source", ""))
|
||||
if source_id in source_sha_by_id and row.get("source_sha") != source_sha_by_id[source_id]:
|
||||
errors.append(f"{row.get('id')}: source_sha does not match source index")
|
||||
for field in ("sim_status", "web_status"):
|
||||
if row.get(field) not in ALLOWED_STATUS:
|
||||
errors.append(f"{row.get('id')}: invalid {field}={row.get(field)!r}")
|
||||
if row.get("sim_status") == "accepted" or row.get("web_status") == "accepted":
|
||||
if not row.get("feature_id") or not row.get("test_id") or not row.get("oracle") or not row.get("context_id"):
|
||||
errors.append(f"{row.get('id')}: accepted entry lacks feature/test/oracle/context evidence")
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
|
||||
parser.add_argument("--lock", type=Path, default=Path("context-lock.json"))
|
||||
parser.add_argument("--source-index", type=Path, default=Path("coverage/source-index.jsonl"))
|
||||
parser.add_argument("--manifest", type=Path, default=Path("coverage/manifest.jsonl"))
|
||||
args = parser.parse_args()
|
||||
root = args.repo_root.resolve()
|
||||
errors = validate(root, root / args.lock, root / args.source_index, root / args.manifest)
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}")
|
||||
return 1
|
||||
print("baseline=valid")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user