241 lines
13 KiB
Python
Executable File
241 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Validate the M0 baseline and fail closed on unassigned coverage entries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
|
|
REQUIRED_MANIFEST_FIELDS = {
|
|
"id", "source", "source_sha", "domain", "kind", "dependencies", "inputs",
|
|
"outputs", "errors", "sim_status", "web_status", "test_id", "oracle", "golden",
|
|
"difference", "coverage_dimensions", "context_id", "exclusion_reason",
|
|
}
|
|
ALLOWED_STATUS = {"planned", "in_progress", "implemented", "verified", "accepted", "blocked", "REAL-EXCLUDED"}
|
|
|
|
|
|
def read_jsonl(path: Path) -> list[dict[str, object]]:
|
|
rows: list[dict[str, object]] = []
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, 1):
|
|
if not line.strip():
|
|
continue
|
|
value = json.loads(line)
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"{path}:{line_number}: expected object")
|
|
rows.append(value)
|
|
return rows
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def validate(repo_root: Path, lock_path: Path, source_path: Path, manifest_path: Path) -> list[str]:
|
|
errors: list[str] = []
|
|
lock = json.loads(lock_path.read_text(encoding="utf-8"))
|
|
for field in ("context_id", "linuxcnc_version", "linuxcnc_source_sha", "emscripten_version", "schema_versions"):
|
|
if not lock.get(field):
|
|
errors.append(f"context lock missing {field}")
|
|
sources = read_jsonl(source_path)
|
|
manifests = read_jsonl(manifest_path)
|
|
if lock.get("source_index_sha256") != sha256(source_path):
|
|
errors.append("source index hash does not match context lock")
|
|
if lock.get("manifest_seed_sha256") != sha256(manifest_path):
|
|
errors.append("manifest seed hash does not match context lock")
|
|
document_index_path = repo_root / "coverage/document-capabilities.jsonl"
|
|
document_index_hash = lock.get("document_capabilities_sha256")
|
|
if document_index_hash:
|
|
if not document_index_path.is_file():
|
|
errors.append("document capabilities index is missing")
|
|
elif document_index_hash != sha256(document_index_path):
|
|
errors.append("document capabilities index hash does not match context lock")
|
|
version_path = repo_root / "linuxcnc/VERSION"
|
|
if lock.get("linuxcnc_version") != version_path.read_text(encoding="utf-8").strip():
|
|
errors.append("linuxcnc version does not match context lock")
|
|
try:
|
|
source_sha = subprocess.check_output(
|
|
["git", "-C", str(repo_root / "linuxcnc"), "rev-parse", "HEAD"],
|
|
text=True,
|
|
).strip()
|
|
except (OSError, subprocess.CalledProcessError):
|
|
source_sha = None
|
|
if source_sha and lock.get("linuxcnc_source_sha") != source_sha:
|
|
errors.append("linuxcnc source commit does not match context lock")
|
|
inventory_count = lock.get("inventory_counts", {})
|
|
if isinstance(inventory_count, dict):
|
|
if inventory_count.get("total") != len(sources):
|
|
errors.append("source index count does not match context lock")
|
|
if inventory_count.get("source_index") != len(sources):
|
|
errors.append("source_index count does not match context lock")
|
|
if inventory_count.get("manifest_seed") != len(manifests):
|
|
errors.append("manifest_seed count does not match context lock")
|
|
if document_index_hash and inventory_count.get("document_capabilities") is not None:
|
|
if inventory_count.get("document_capabilities") != sum(
|
|
1 for line in document_index_path.read_text(encoding="utf-8").splitlines() if line.strip()
|
|
):
|
|
errors.append("document_capabilities count does not match context lock")
|
|
symbols_path = repo_root / "coverage/source-symbols.jsonl"
|
|
symbols_hash = lock.get("source_symbols_sha256")
|
|
if symbols_hash:
|
|
if not symbols_path.is_file():
|
|
errors.append("source symbols index is missing")
|
|
elif symbols_hash != sha256(symbols_path):
|
|
errors.append("source symbols index hash does not match context lock")
|
|
if isinstance(inventory_count, dict) and symbols_hash and inventory_count.get("source_symbols") is not None:
|
|
if inventory_count.get("source_symbols") != sum(1 for line in symbols_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
|
errors.append("source_symbols count does not match context lock")
|
|
resources_path = repo_root / "coverage/config-resources.jsonl"
|
|
resources_hash = lock.get("config_resources_sha256")
|
|
if resources_hash:
|
|
if not resources_path.is_file():
|
|
errors.append("config resources index is missing")
|
|
elif resources_hash != sha256(resources_path):
|
|
errors.append("config resources index hash does not match context lock")
|
|
if isinstance(inventory_count, dict) and resources_hash and inventory_count.get("config_resources") is not None:
|
|
if inventory_count.get("config_resources") != sum(1 for line in resources_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
|
errors.append("config_resources count does not match context lock")
|
|
test_entries_path = repo_root / "coverage/test-entries.jsonl"
|
|
test_entries_hash = lock.get("test_entries_sha256")
|
|
if test_entries_hash:
|
|
if not test_entries_path.is_file():
|
|
errors.append("test entries index is missing")
|
|
elif test_entries_hash != sha256(test_entries_path):
|
|
errors.append("test entries index hash does not match context lock")
|
|
if isinstance(inventory_count, dict) and test_entries_hash and inventory_count.get("test_entries") is not None:
|
|
if inventory_count.get("test_entries") != sum(1 for line in test_entries_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
|
errors.append("test_entries count does not match context lock")
|
|
normalized_path = repo_root / "coverage/normalized-sources.jsonl"
|
|
normalized_hash = lock.get("normalized_sources_sha256")
|
|
if normalized_hash:
|
|
if not normalized_path.is_file():
|
|
errors.append("normalized sources index is missing")
|
|
elif normalized_hash != sha256(normalized_path):
|
|
errors.append("normalized sources index hash does not match context lock")
|
|
if isinstance(inventory_count, dict) and normalized_hash and inventory_count.get("normalized_sources") is not None:
|
|
if inventory_count.get("normalized_sources") != sum(1 for line in normalized_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
|
errors.append("normalized_sources count does not match context lock")
|
|
closure_path = repo_root / "coverage/dependency-closure.jsonl"
|
|
closure_hash = lock.get("dependency_closure_sha256")
|
|
if closure_hash:
|
|
if not closure_path.is_file():
|
|
errors.append("dependency closure index is missing")
|
|
elif closure_hash != sha256(closure_path):
|
|
errors.append("dependency closure index hash does not match context lock")
|
|
if isinstance(inventory_count, dict) and closure_hash and inventory_count.get("dependency_edges") is not None:
|
|
if inventory_count.get("dependency_edges") != sum(1 for line in closure_path.read_text(encoding="utf-8").splitlines() if line.strip()):
|
|
errors.append("dependency_edges count does not match context lock")
|
|
golden_schema_path = repo_root / "coverage/golden-schema.json"
|
|
golden_schema_hash = lock.get("golden_schema_sha256")
|
|
if golden_schema_hash:
|
|
if not golden_schema_path.is_file():
|
|
errors.append("golden schema is missing")
|
|
elif golden_schema_hash != sha256(golden_schema_path):
|
|
errors.append("golden schema hash does not match context lock")
|
|
runner_report_path = repo_root / "coverage/native-runner.report.json"
|
|
runner_report_hash = lock.get("native_runner_report_sha256")
|
|
if runner_report_hash:
|
|
if not runner_report_path.is_file():
|
|
errors.append("native runner report is missing")
|
|
elif runner_report_hash != sha256(runner_report_path):
|
|
errors.append("native runner report hash does not match context lock")
|
|
recovery_report_path = repo_root / "coverage/workspace-recovery.report.json"
|
|
recovery_report_hash = lock.get("workspace_recovery_report_sha256")
|
|
if recovery_report_hash:
|
|
if not recovery_report_path.is_file():
|
|
errors.append("workspace recovery report is missing")
|
|
elif recovery_report_hash != sha256(recovery_report_path):
|
|
errors.append("workspace recovery report hash does not match context lock")
|
|
runtime_report_path = repo_root / "coverage/runtime-mapping.report.json"
|
|
runtime_report_hash = lock.get("runtime_mapping_report_sha256")
|
|
if runtime_report_hash:
|
|
if not runtime_report_path.is_file():
|
|
errors.append("runtime mapping report is missing")
|
|
elif runtime_report_hash != sha256(runtime_report_path):
|
|
errors.append("runtime mapping report hash does not match context lock")
|
|
python_manifest_path = repo_root / "python-runtime/manifest.json"
|
|
python_manifest_hash = lock.get("python_runtime_manifest_sha256")
|
|
if python_manifest_hash:
|
|
if not python_manifest_path.is_file():
|
|
errors.append("python runtime manifest is missing")
|
|
elif python_manifest_hash != sha256(python_manifest_path):
|
|
errors.append("python runtime manifest hash does not match context lock")
|
|
python_report_path = repo_root / "coverage/python-runtime.report.json"
|
|
python_report_hash = lock.get("python_runtime_report_sha256")
|
|
if python_report_hash:
|
|
if not python_report_path.is_file():
|
|
errors.append("python runtime report is missing")
|
|
elif python_report_hash != sha256(python_report_path):
|
|
errors.append("python runtime report hash does not match context lock")
|
|
hal_report_path = repo_root / "coverage/hal-runtime.report.json"
|
|
hal_report_hash = lock.get("hal_runtime_report_sha256")
|
|
if hal_report_hash:
|
|
if not hal_report_path.is_file():
|
|
errors.append("hal runtime report is missing")
|
|
elif hal_report_hash != sha256(hal_report_path):
|
|
errors.append("hal runtime report hash does not match context lock")
|
|
posemath_report_path = repo_root / "coverage/posemath-runtime.report.json"
|
|
posemath_report_hash = lock.get("posemath_runtime_report_sha256")
|
|
if posemath_report_hash:
|
|
if not posemath_report_path.is_file():
|
|
errors.append("posemath runtime report is missing")
|
|
elif posemath_report_hash != sha256(posemath_report_path):
|
|
errors.append("posemath runtime report hash does not match context lock")
|
|
source_ids = [str(row.get("source_id", "")) for row in sources]
|
|
if len(source_ids) != len(set(source_ids)):
|
|
errors.append("source index contains duplicate source_id")
|
|
manifest_ids = [str(row.get("id", "")) for row in manifests]
|
|
if len(manifest_ids) != len(set(manifest_ids)):
|
|
errors.append("manifest contains duplicate id")
|
|
source_sha_by_id = {str(row["source_id"]): str(row["sha256"]) for row in sources}
|
|
manifest_sources = {str(row.get("source", "")) for row in manifests}
|
|
missing = sorted(set(source_ids) - manifest_sources)
|
|
if missing:
|
|
errors.append(f"manifest has {len(missing)} orphan sources; first={missing[0]}")
|
|
extra = sorted(manifest_sources - set(source_ids))
|
|
if extra:
|
|
errors.append(f"manifest has {len(extra)} unknown sources; first={extra[0]}")
|
|
for row in manifests:
|
|
missing_fields = REQUIRED_MANIFEST_FIELDS - row.keys()
|
|
if missing_fields:
|
|
errors.append(f"{row.get('id')}: missing fields {sorted(missing_fields)}")
|
|
source_id = str(row.get("source", ""))
|
|
if source_id in source_sha_by_id and row.get("source_sha") != source_sha_by_id[source_id]:
|
|
errors.append(f"{row.get('id')}: source_sha does not match source index")
|
|
for field in ("sim_status", "web_status"):
|
|
if row.get(field) not in ALLOWED_STATUS:
|
|
errors.append(f"{row.get('id')}: invalid {field}={row.get(field)!r}")
|
|
if row.get("sim_status") == "accepted" or row.get("web_status") == "accepted":
|
|
if not row.get("feature_id") or not row.get("test_id") or not row.get("oracle") or not row.get("context_id"):
|
|
errors.append(f"{row.get('id')}: accepted entry lacks feature/test/oracle/context evidence")
|
|
return errors
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument("--lock", type=Path, default=Path("context-lock.json"))
|
|
parser.add_argument("--source-index", type=Path, default=Path("coverage/source-index.jsonl"))
|
|
parser.add_argument("--manifest", type=Path, default=Path("coverage/manifest.jsonl"))
|
|
args = parser.parse_args()
|
|
root = args.repo_root.resolve()
|
|
errors = validate(root, root / args.lock, root / args.source_index, root / args.manifest)
|
|
if errors:
|
|
for error in errors:
|
|
print(f"ERROR: {error}")
|
|
return 1
|
|
print("baseline=valid")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|