Implement AXIS parity UI and execution baseline
This commit is contained in:
185
scripts/generate_full_case_acceptance_s8_10.py
Normal file
185
scripts/generate_full_case_acceptance_s8_10.py
Normal file
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Aggregate the final S8-10 implementation and acceptance evidence.
|
||||
|
||||
The implementation regression can be verified while acceptance remains blocked.
|
||||
The distinction is intentional: a passing Worker/App test set is not a native
|
||||
LinuxCNC oracle, a complete 235-case run, or a supported-browser release gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPORT = Path("coverage/full-case-acceptance-s8-10.report.json")
|
||||
CONTEXT_ID = "linuxcnc-2.10.0-pre0-wasm-m1-py-c1"
|
||||
|
||||
DIRECT_INPUTS = (
|
||||
("context-lock.json", "context_lock"),
|
||||
("coverage/sim-case-matrix-s8-06.report.json", "case_matrix"),
|
||||
("coverage/sim-display-families-s8-07.report.json", "display_families"),
|
||||
("coverage/machine-families-s8-08.report.json", "machine_families"),
|
||||
("coverage/runtime-extensions-s8-09.report.json", "runtime_extensions"),
|
||||
("coverage/native-linuxcnc-case-conditions.report.json", "real_native_regression"),
|
||||
("coverage/native-oracle-closure.report.json", "official_oracle_boundary"),
|
||||
("coverage/browser-matrix-remediation.report.json", "browser_matrix"),
|
||||
("coverage/release-candidate-gate.report.json", "release_gate"),
|
||||
)
|
||||
|
||||
|
||||
def canonical(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def sha256_bytes(value: bytes) -> str:
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def artifact(path: str, role: str) -> dict[str, Any]:
|
||||
source = ROOT / path
|
||||
if not source.is_file():
|
||||
return {"path": path, "role": role, "status": "missing"}
|
||||
record: dict[str, Any] = {
|
||||
"path": path,
|
||||
"role": role,
|
||||
"status": "present",
|
||||
"bytes": source.stat().st_size,
|
||||
"sha256": sha256_bytes(source.read_bytes()),
|
||||
}
|
||||
if source.suffix == ".json":
|
||||
try:
|
||||
value = json.loads(source.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
value = None
|
||||
if isinstance(value, dict):
|
||||
for key in ("schema", "task_id", "status", "implementation_status"):
|
||||
if key in value:
|
||||
record[key] = value[key]
|
||||
return record
|
||||
|
||||
|
||||
def read_json(path: str) -> dict[str, Any]:
|
||||
value = json.loads((ROOT / path).read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{path} must contain an object")
|
||||
return value
|
||||
|
||||
|
||||
def run_command(command: Iterable[str], label: str) -> dict[str, Any]:
|
||||
argv = list(command)
|
||||
result = subprocess.run(argv, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
|
||||
stdout = result.stdout.decode("utf-8", errors="replace")
|
||||
stderr = result.stderr.decode("utf-8", errors="replace")
|
||||
return {
|
||||
"label": label,
|
||||
"command": argv,
|
||||
"returncode": result.returncode,
|
||||
"status": "passed" if result.returncode == 0 else "failed",
|
||||
"stdout_sha256": sha256_bytes(result.stdout),
|
||||
"stderr_sha256": sha256_bytes(result.stderr),
|
||||
"stdout_tail": stdout[-800:],
|
||||
"stderr_tail": stderr[-800:],
|
||||
}
|
||||
|
||||
|
||||
def technical_regression() -> dict[str, Any]:
|
||||
records = [run_command(["bash", "scripts/build-wasm.sh"], "wasm_build_and_native_ctest")]
|
||||
records.append(run_command(["ctest", "--test-dir", "build-native", "--output-on-failure"], "native_ctest"))
|
||||
worker_tests = sorted(path.relative_to(ROOT).as_posix() for path in (ROOT / "web/worker").glob("test_*.mjs"))
|
||||
app_tests = sorted(path.relative_to(ROOT).as_posix() for path in (ROOT / "web/app").glob("test_*.mjs"))
|
||||
for path in worker_tests:
|
||||
records.append(run_command(["node", path], path))
|
||||
for path in app_tests:
|
||||
records.append(run_command(["node", path], path))
|
||||
passed = sum(row["status"] == "passed" for row in records)
|
||||
return {
|
||||
"status": "verified" if passed == len(records) else "blocked",
|
||||
"commands": records,
|
||||
"counts": {"total": len(records), "passed": passed, "failed": len(records) - passed, "worker_tests": len(worker_tests), "app_tests": len(app_tests)},
|
||||
"policy": "build, native CTest, Worker and App regressions are implementation evidence only",
|
||||
}
|
||||
|
||||
|
||||
def generate(output: Path, *, run_regressions: bool = True) -> dict[str, Any]:
|
||||
lock = read_json("context-lock.json")
|
||||
matrix = read_json("coverage/sim-case-matrix-s8-06.report.json")
|
||||
display = read_json("coverage/sim-display-families-s8-07.report.json")
|
||||
machine = read_json("coverage/machine-families-s8-08.report.json")
|
||||
extensions = read_json("coverage/runtime-extensions-s8-09.report.json")
|
||||
native = read_json("coverage/native-linuxcnc-case-conditions.report.json")
|
||||
oracle = read_json("coverage/native-oracle-closure.report.json")
|
||||
browser = read_json("coverage/browser-matrix-remediation.report.json")
|
||||
gate = read_json("coverage/release-candidate-gate.report.json")
|
||||
|
||||
counts = matrix.get("counts", {})
|
||||
case_count = sum(int(value) for value in counts.values() if isinstance(value, int))
|
||||
ready_count = int(counts.get("TASK_READY", 0))
|
||||
blocked_count = case_count - ready_count
|
||||
context_ids = [matrix.get("context_id"), extensions.get("context_id"), native.get("context_id"), oracle.get("context_id"), browser.get("context_id"), lock.get("context_id")]
|
||||
context_consistent = all(value == CONTEXT_ID for value in context_ids)
|
||||
direct_verified = all(value.get("status") == "verified" and value.get("implementation_status", "verified") in {"verified", "passed"} for value in (matrix, display, machine, extensions, native))
|
||||
regression = technical_regression() if run_regressions else {"status": "not_run", "commands": [], "counts": {"total": 0, "passed": 0, "failed": 0, "worker_tests": 0, "app_tests": 0}, "policy": "not run; use the default generator invocation for acceptance evidence"}
|
||||
implementation_status = "verified" if direct_verified and context_consistent and regression["status"] == "verified" else "blocked"
|
||||
|
||||
official_oracle_blocked = oracle.get("oracle_boundary", {}).get("status") == "blocked"
|
||||
minimums = lock.get("browser_matrix", {}).get("minimum", {})
|
||||
browser_matrix_blocked = lock.get("browser_matrix", {}).get("status") != "verified" or any(value in (None, "", "TBD") for value in minimums.values()) or browser.get("remediation_status") != "verified"
|
||||
blockers = [
|
||||
{"id": "remaining_case_blockers", "count": blocked_count, "status": "blocked", "reason": "235-case startup matrix retains resource/HAL/INI/STARTUP blockers; blocked rows are not promoted"},
|
||||
{"id": "official_oracle", "count": 1 if official_oracle_blocked else 0, "status": "blocked" if official_oracle_blocked else "verified", "reason": "official full LinuxCNC oracle inputs remain source-gated"},
|
||||
{"id": "browser_matrix", "count": 1 if browser_matrix_blocked else 0, "status": "blocked" if browser_matrix_blocked else "verified", "reason": "locked Chromium/Firefox/Safari minimums are not pinned and Safari is unavailable in this environment"},
|
||||
]
|
||||
acceptance_status = "verified" if not any(row["status"] == "blocked" for row in blockers) else "blocked"
|
||||
report: dict[str, Any] = {
|
||||
"schema": "full-case-acceptance-s8-10-report.v1",
|
||||
"task_id": "S8-10",
|
||||
"context_id": CONTEXT_ID,
|
||||
"status": "verified" if implementation_status == "verified" and acceptance_status == "verified" else "blocked",
|
||||
"implementation_status": implementation_status,
|
||||
"acceptance_status": acceptance_status,
|
||||
"coverage": {
|
||||
"case_matrix": {"case_count": case_count, "ready": ready_count, "blocked": blocked_count, "counts": counts, "status": "verified" if matrix.get("status") == "verified" else "blocked"},
|
||||
"display_families": {"case_count": display.get("case_count"), "counts": display.get("counts", {}), "status": display.get("status")},
|
||||
"machine_families": {"status": machine.get("status"), "implementation_status": machine.get("implementation_status")},
|
||||
"runtime_extensions": {"status": extensions.get("status"), "runner_counts": extensions.get("counts", {})},
|
||||
},
|
||||
"real_regression": {"native_case_conditions": native.get("status"), "technical": regression},
|
||||
"checks": {
|
||||
"direct_inputs": {"status": "verified" if direct_verified else "blocked"},
|
||||
"context_consistency": {"status": "verified" if context_consistent else "blocked", "context_ids": context_ids},
|
||||
"implementation_regression": {"status": regression["status"]},
|
||||
"case_matrix_aggregated": {"status": "verified" if case_count == 235 and blocked_count > 0 else "blocked", "cases": case_count, "ready": ready_count, "blocked": blocked_count},
|
||||
"acceptance_blockers_retained": {"status": "verified" if acceptance_status == "blocked" else "verified", "blockers": blockers},
|
||||
"release_stop_preserved": {"status": "verified" if gate.get("release_action") == "STOP_RELEASE" and gate.get("candidate_eligible") is False else "blocked", "release_action": gate.get("release_action"), "candidate_eligible": gate.get("candidate_eligible")},
|
||||
},
|
||||
"blockers": blockers,
|
||||
"release_action": "STOP_RELEASE" if acceptance_status == "blocked" else "NONE",
|
||||
"candidate_eligible": acceptance_status == "verified" and implementation_status == "verified",
|
||||
"inputs": [artifact(path, role) for path, role in DIRECT_INPUTS],
|
||||
"boundaries": ["implementation coverage is not a LinuxCNC full-feature release claim", "blocked case rows retain their first diagnostics and are not reclassified", "official oracle and browser matrix blockers are independent of module regression status"],
|
||||
}
|
||||
report["report_sha256"] = sha256_bytes(canonical(report).encode("utf-8"))
|
||||
destination = output if output.is_absolute() else ROOT / output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8")
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--report", type=Path, default=REPORT)
|
||||
parser.add_argument("--skip-regressions", action="store_true", help="only for local report-shape iteration")
|
||||
args = parser.parse_args()
|
||||
report = generate(args.report, run_regressions=not args.skip_regressions)
|
||||
print(f"full_case_acceptance_s8_10={report['status']} implementation={report['implementation_status']} acceptance={report['acceptance_status']} blockers={sum(row['status'] == 'blocked' for row in report['blockers'])} report_sha256={report['report_sha256']}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user