186 lines
11 KiB
Python
186 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Aggregate the final S8-10 implementation and acceptance evidence.
|
|
|
|
The implementation regression can be verified while acceptance remains blocked.
|
|
The distinction is intentional: a passing Worker/App test set is not a native
|
|
LinuxCNC oracle, a complete 235-case run, or a supported-browser release gate.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REPORT = Path("coverage/full-case-acceptance-s8-10.report.json")
|
|
CONTEXT_ID = "linuxcnc-2.10.0-pre0-wasm-m1-py-c1"
|
|
|
|
DIRECT_INPUTS = (
|
|
("context-lock.json", "context_lock"),
|
|
("coverage/sim-case-matrix-s8-06.report.json", "case_matrix"),
|
|
("coverage/sim-display-families-s8-07.report.json", "display_families"),
|
|
("coverage/machine-families-s8-08.report.json", "machine_families"),
|
|
("coverage/runtime-extensions-s8-09.report.json", "runtime_extensions"),
|
|
("coverage/native-linuxcnc-case-conditions.report.json", "real_native_regression"),
|
|
("coverage/native-oracle-closure.report.json", "official_oracle_boundary"),
|
|
("coverage/browser-matrix-remediation.report.json", "browser_matrix"),
|
|
("coverage/release-candidate-gate.report.json", "release_gate"),
|
|
)
|
|
|
|
|
|
def canonical(value: Any) -> str:
|
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
|
|
|
|
def sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def artifact(path: str, role: str) -> dict[str, Any]:
|
|
source = ROOT / path
|
|
if not source.is_file():
|
|
return {"path": path, "role": role, "status": "missing"}
|
|
record: dict[str, Any] = {
|
|
"path": path,
|
|
"role": role,
|
|
"status": "present",
|
|
"bytes": source.stat().st_size,
|
|
"sha256": sha256_bytes(source.read_bytes()),
|
|
}
|
|
if source.suffix == ".json":
|
|
try:
|
|
value = json.loads(source.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError):
|
|
value = None
|
|
if isinstance(value, dict):
|
|
for key in ("schema", "task_id", "status", "implementation_status"):
|
|
if key in value:
|
|
record[key] = value[key]
|
|
return record
|
|
|
|
|
|
def read_json(path: str) -> dict[str, Any]:
|
|
value = json.loads((ROOT / path).read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"{path} must contain an object")
|
|
return value
|
|
|
|
|
|
def run_command(command: Iterable[str], label: str) -> dict[str, Any]:
|
|
argv = list(command)
|
|
result = subprocess.run(argv, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
|
|
stdout = result.stdout.decode("utf-8", errors="replace")
|
|
stderr = result.stderr.decode("utf-8", errors="replace")
|
|
return {
|
|
"label": label,
|
|
"command": argv,
|
|
"returncode": result.returncode,
|
|
"status": "passed" if result.returncode == 0 else "failed",
|
|
"stdout_sha256": sha256_bytes(result.stdout),
|
|
"stderr_sha256": sha256_bytes(result.stderr),
|
|
"stdout_tail": stdout[-800:],
|
|
"stderr_tail": stderr[-800:],
|
|
}
|
|
|
|
|
|
def technical_regression() -> dict[str, Any]:
|
|
records = [run_command(["bash", "scripts/build-wasm.sh"], "wasm_build_and_native_ctest")]
|
|
records.append(run_command(["ctest", "--test-dir", "build-native", "--output-on-failure"], "native_ctest"))
|
|
worker_tests = sorted(path.relative_to(ROOT).as_posix() for path in (ROOT / "web/worker").glob("test_*.mjs"))
|
|
app_tests = sorted(path.relative_to(ROOT).as_posix() for path in (ROOT / "web/app").glob("test_*.mjs"))
|
|
for path in worker_tests:
|
|
records.append(run_command(["node", path], path))
|
|
for path in app_tests:
|
|
records.append(run_command(["node", path], path))
|
|
passed = sum(row["status"] == "passed" for row in records)
|
|
return {
|
|
"status": "verified" if passed == len(records) else "blocked",
|
|
"commands": records,
|
|
"counts": {"total": len(records), "passed": passed, "failed": len(records) - passed, "worker_tests": len(worker_tests), "app_tests": len(app_tests)},
|
|
"policy": "build, native CTest, Worker and App regressions are implementation evidence only",
|
|
}
|
|
|
|
|
|
def generate(output: Path, *, run_regressions: bool = True) -> dict[str, Any]:
|
|
lock = read_json("context-lock.json")
|
|
matrix = read_json("coverage/sim-case-matrix-s8-06.report.json")
|
|
display = read_json("coverage/sim-display-families-s8-07.report.json")
|
|
machine = read_json("coverage/machine-families-s8-08.report.json")
|
|
extensions = read_json("coverage/runtime-extensions-s8-09.report.json")
|
|
native = read_json("coverage/native-linuxcnc-case-conditions.report.json")
|
|
oracle = read_json("coverage/native-oracle-closure.report.json")
|
|
browser = read_json("coverage/browser-matrix-remediation.report.json")
|
|
gate = read_json("coverage/release-candidate-gate.report.json")
|
|
|
|
counts = matrix.get("counts", {})
|
|
case_count = sum(int(value) for value in counts.values() if isinstance(value, int))
|
|
ready_count = int(counts.get("TASK_READY", 0))
|
|
blocked_count = case_count - ready_count
|
|
context_ids = [matrix.get("context_id"), extensions.get("context_id"), native.get("context_id"), oracle.get("context_id"), browser.get("context_id"), lock.get("context_id")]
|
|
context_consistent = all(value == CONTEXT_ID for value in context_ids)
|
|
direct_verified = all(value.get("status") == "verified" and value.get("implementation_status", "verified") in {"verified", "passed"} for value in (matrix, display, machine, extensions, native))
|
|
regression = technical_regression() if run_regressions else {"status": "not_run", "commands": [], "counts": {"total": 0, "passed": 0, "failed": 0, "worker_tests": 0, "app_tests": 0}, "policy": "not run; use the default generator invocation for acceptance evidence"}
|
|
implementation_status = "verified" if direct_verified and context_consistent and regression["status"] == "verified" else "blocked"
|
|
|
|
official_oracle_blocked = oracle.get("oracle_boundary", {}).get("status") == "blocked"
|
|
minimums = lock.get("browser_matrix", {}).get("minimum", {})
|
|
browser_matrix_blocked = lock.get("browser_matrix", {}).get("status") != "verified" or any(value in (None, "", "TBD") for value in minimums.values()) or browser.get("remediation_status") != "verified"
|
|
blockers = [
|
|
{"id": "remaining_case_blockers", "count": blocked_count, "status": "blocked", "reason": "235-case startup matrix retains resource/HAL/INI/STARTUP blockers; blocked rows are not promoted"},
|
|
{"id": "official_oracle", "count": 1 if official_oracle_blocked else 0, "status": "blocked" if official_oracle_blocked else "verified", "reason": "official full LinuxCNC oracle inputs remain source-gated"},
|
|
{"id": "browser_matrix", "count": 1 if browser_matrix_blocked else 0, "status": "blocked" if browser_matrix_blocked else "verified", "reason": "locked Chromium/Firefox/Safari minimums are not pinned and Safari is unavailable in this environment"},
|
|
]
|
|
acceptance_status = "verified" if not any(row["status"] == "blocked" for row in blockers) else "blocked"
|
|
report: dict[str, Any] = {
|
|
"schema": "full-case-acceptance-s8-10-report.v1",
|
|
"task_id": "S8-10",
|
|
"context_id": CONTEXT_ID,
|
|
"status": "verified" if implementation_status == "verified" and acceptance_status == "verified" else "blocked",
|
|
"implementation_status": implementation_status,
|
|
"acceptance_status": acceptance_status,
|
|
"coverage": {
|
|
"case_matrix": {"case_count": case_count, "ready": ready_count, "blocked": blocked_count, "counts": counts, "status": "verified" if matrix.get("status") == "verified" else "blocked"},
|
|
"display_families": {"case_count": display.get("case_count"), "counts": display.get("counts", {}), "status": display.get("status")},
|
|
"machine_families": {"status": machine.get("status"), "implementation_status": machine.get("implementation_status")},
|
|
"runtime_extensions": {"status": extensions.get("status"), "runner_counts": extensions.get("counts", {})},
|
|
},
|
|
"real_regression": {"native_case_conditions": native.get("status"), "technical": regression},
|
|
"checks": {
|
|
"direct_inputs": {"status": "verified" if direct_verified else "blocked"},
|
|
"context_consistency": {"status": "verified" if context_consistent else "blocked", "context_ids": context_ids},
|
|
"implementation_regression": {"status": regression["status"]},
|
|
"case_matrix_aggregated": {"status": "verified" if case_count == 235 and blocked_count > 0 else "blocked", "cases": case_count, "ready": ready_count, "blocked": blocked_count},
|
|
"acceptance_blockers_retained": {"status": "verified" if acceptance_status == "blocked" else "verified", "blockers": blockers},
|
|
"release_stop_preserved": {"status": "verified" if gate.get("release_action") == "STOP_RELEASE" and gate.get("candidate_eligible") is False else "blocked", "release_action": gate.get("release_action"), "candidate_eligible": gate.get("candidate_eligible")},
|
|
},
|
|
"blockers": blockers,
|
|
"release_action": "STOP_RELEASE" if acceptance_status == "blocked" else "NONE",
|
|
"candidate_eligible": acceptance_status == "verified" and implementation_status == "verified",
|
|
"inputs": [artifact(path, role) for path, role in DIRECT_INPUTS],
|
|
"boundaries": ["implementation coverage is not a LinuxCNC full-feature release claim", "blocked case rows retain their first diagnostics and are not reclassified", "official oracle and browser matrix blockers are independent of module regression status"],
|
|
}
|
|
report["report_sha256"] = sha256_bytes(canonical(report).encode("utf-8"))
|
|
destination = output if output.is_absolute() else ROOT / output
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--report", type=Path, default=REPORT)
|
|
parser.add_argument("--skip-regressions", action="store_true", help="only for local report-shape iteration")
|
|
args = parser.parse_args()
|
|
report = generate(args.report, run_regressions=not args.skip_regressions)
|
|
print(f"full_case_acceptance_s8_10={report['status']} implementation={report['implementation_status']} acceptance={report['acceptance_status']} blockers={sum(row['status'] == 'blocked' for row in report['blockers'])} report_sha256={report['report_sha256']}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|