317 lines
20 KiB
Python
317 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Run one module's declared business cases and retain attributable evidence."""
|
|
import json
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
from business_acceptance import ROOT, LEDGER, digest, fingerprint, module_inputs, read, requirements, successful_result, accepted_ids
|
|
from failure_reconciliation import SOURCE as FAILURE_SOURCE, entries as failure_entries, occurrence, adopt_business_evidence
|
|
from record_runtime_issue import ACTIVE_STATUSES, observation
|
|
from implementation_first import user_validation_deferred
|
|
|
|
|
|
def close_tested_issues(report: dict, evidence, before: dict, passed: set[str], *, refresh_resolved: bool = True) -> int:
|
|
path = ROOT / "docs/current/runtime-issue-ledger.json"
|
|
ledger = read(path)
|
|
declarations = {issue_id: result["capabilityId"] for result in report.get("results", [])
|
|
if result.get("capabilityId") in passed and successful_result(result)
|
|
for issue_id in result.get("resolvedIssueIds", [])}
|
|
closed = 0
|
|
changed = False
|
|
for issue in ledger.get("issues", []):
|
|
if (issue.get("status", "open") in ACTIVE_STATUSES | ({"resolved"} if refresh_resolved else set())
|
|
and declarations.get(issue.get("id")) == issue.get("capabilityId")
|
|
and before.get(issue.get("id")) == observation(issue)):
|
|
newly_closed = issue.get("status") != "resolved"
|
|
issue.update(status="resolved", resolvedAt=report["executedAt"],
|
|
businessEvidencePath=str(evidence.relative_to(ROOT)), businessEvidenceSha256=digest(evidence),
|
|
resolvedObservation=before[issue["id"]])
|
|
closed += int(newly_closed)
|
|
changed = True
|
|
if changed:
|
|
ledger["updatedAt"] = datetime.now(timezone.utc).isoformat()
|
|
temporary = path.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(path)
|
|
return closed
|
|
|
|
|
|
def record_failures(record_ids: set[str], reason: str) -> None:
|
|
ledger = read(LEDGER)
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
ledger.update(schema="freecad-web-business-acceptance/v1", updatedAt=now)
|
|
entries = ledger.setdefault("records", {})
|
|
for record_id in record_ids:
|
|
entry = entries.setdefault(record_id, {})
|
|
entry.update(status="failed", lastFailure=reason, updatedAt=now)
|
|
LEDGER.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = LEDGER.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(LEDGER)
|
|
|
|
|
|
def case_inputs(contract_path, command: list, dependencies: list) -> dict:
|
|
if not isinstance(dependencies, list) or any(not isinstance(value, str) for value in dependencies):
|
|
raise ValueError("case dependencies must be an array of paths")
|
|
paths = {contract_path, ROOT / command[1], ROOT / "tools/wasm-probe-runtime.mjs"}
|
|
for relative in dependencies:
|
|
path = (ROOT / relative).resolve()
|
|
if not path.is_relative_to(ROOT):
|
|
raise ValueError("case dependencies must remain inside the project")
|
|
if path.is_dir():
|
|
paths.update(candidate for candidate in path.rglob("*") if candidate.is_file() and "__pycache__" not in candidate.parts)
|
|
else:
|
|
paths.add(path)
|
|
files = {str(path.relative_to(ROOT)): digest(path) for path in sorted(paths)}
|
|
# These two contract fields describe progress; they are not consumed by
|
|
# execution or build selection. Preserve every other (including unknown)
|
|
# field, so changing a real case/configuration still requires fresh proof.
|
|
contract = read(contract_path)
|
|
execution_contract = {key: value for key, value in contract.items() if key not in {"scope", "remainingBoundaries"}}
|
|
files[str(contract_path.relative_to(ROOT))] = hashlib.sha256(
|
|
json.dumps(execution_contract, sort_keys=True, ensure_ascii=False).encode("utf-8")).hexdigest()
|
|
return {"files": files,
|
|
"buildOverrides": {key: value for key, value in os.environ.items() if key.startswith("FREECAD_") and key.endswith("_BUSINESS_BUILD")}}
|
|
|
|
|
|
def reusable_reports(selected: list[dict], command: list, inputs: dict, failure_fingerprints: dict, issue_fingerprints: dict):
|
|
ids = {row["id"] for row in selected}
|
|
if accepted_ids(selected) != ids:
|
|
return None
|
|
entries = read(LEDGER).get("records", {})
|
|
groups = {}
|
|
for record_id in sorted(ids):
|
|
name = entries[record_id].get("evidencePath")
|
|
if not isinstance(name, str):
|
|
return None
|
|
groups.setdefault(name, set()).add(record_id)
|
|
reports = []
|
|
for name, covered in groups.items():
|
|
path = (ROOT / name).resolve()
|
|
report = read(path)
|
|
if report.get("command") != command or report.get("caseInputs") != inputs:
|
|
return None
|
|
for result in report.get("results", []):
|
|
if result.get("capabilityId") not in covered:
|
|
continue
|
|
if any(report.get("failureFingerprints", {}).get(key) != failure_fingerprints.get(key) for key in result.get("resolvedFailureKeys", [])):
|
|
return None
|
|
if any(report.get("issueFingerprints", {}).get(key) != issue_fingerprints.get(key) for key in result.get("resolvedIssueIds", [])):
|
|
return None
|
|
reports.append((path, report, covered))
|
|
return reports
|
|
|
|
|
|
def execute_case(command: list, environment: dict):
|
|
# Regular-file stdout also protects existing Node cases that explicitly
|
|
# exit after one write: a pipe can accept only a prefix of a large report.
|
|
with tempfile.TemporaryFile(mode="w+", encoding="utf-8", errors="replace") as output:
|
|
try:
|
|
run = subprocess.run(command, cwd=ROOT, env=environment, stdout=output,
|
|
stderr=subprocess.PIPE, text=True, timeout=180)
|
|
except subprocess.TimeoutExpired as error:
|
|
output.seek(0)
|
|
error.output = output.read()
|
|
raise
|
|
output.seek(0)
|
|
run.stdout = output.read()
|
|
return run
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
if user_validation_deferred():
|
|
print(json.dumps({"status": "validation_deferred_to_user", "executedCount": 0,
|
|
"reason": "Current user policy disables automated business validation; historical evidence is unchanged."}))
|
|
return 0
|
|
started = time.perf_counter()
|
|
digest.cache_clear()
|
|
module_inputs.cache_clear()
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--module")
|
|
parser.add_argument("--ids", nargs="+")
|
|
parser.add_argument("--require-issue-closure", action="store_true")
|
|
parser.add_argument("--reuse", action="store_true", help="reuse current verified evidence without rerunning identical cases")
|
|
options = parser.parse_args(argv or [])
|
|
batch = read(ROOT / "docs/current/module-batch-manifest.json")
|
|
if options.module or options.ids:
|
|
if not options.module or (not options.ids and not options.reuse):
|
|
parser.error("--module and --ids must be supplied together")
|
|
if options.module and not options.ids and options.reuse:
|
|
contract = read(ROOT / "docs/business-acceptance" / f"{options.module}.json")
|
|
options.ids = contract.get("coveredIds") or contract.get("selectedIds")
|
|
if not isinstance(options.ids, list) or not options.ids:
|
|
parser.error("module contract has no covered business IDs")
|
|
identity = "|".join([options.module, "business-acceptance", *sorted(set(options.ids))])
|
|
batch = {"batchId": hashlib.sha256(identity.encode()).hexdigest()[:24],
|
|
"selected": {"module": options.module}, "records": [[record_id] for record_id in sorted(set(options.ids))]}
|
|
module = batch.get("selected", {}).get("module", "")
|
|
rows = {row["id"]: row for row in requirements()}
|
|
selected_ids = {row[0] for row in batch.get("records", [])}
|
|
if not selected_ids or any(record_id not in rows or rows[record_id].get("module") != module for record_id in selected_ids):
|
|
print(json.dumps({"status": "failed", "reason": "acceptance IDs must identify existing business records in the selected module"}))
|
|
return 2
|
|
contract_path = ROOT / "docs/business-acceptance" / f"{module}.json"
|
|
contract = read(contract_path)
|
|
command = contract.get("command", [])
|
|
if not isinstance(command, list) or len(command) < 2 or command[0] not in {"node", "python3"}:
|
|
print(json.dumps({"status": "acceptance_required", "contract": str(contract_path.relative_to(ROOT)), "reason": "module must supply an executable business case contract"}))
|
|
return 78
|
|
script = (ROOT / command[1]).resolve()
|
|
if not script.is_relative_to(ROOT) or not script.is_file():
|
|
print(json.dumps({"status": "failed", "reason": "business case script must exist inside the project"}))
|
|
return 2
|
|
before = {record_id: fingerprint(rows[record_id]) for record_id in selected_ids if record_id in rows}
|
|
failure_fingerprints = {key: occurrence(entry) for key, entry in failure_entries(read(FAILURE_SOURCE)).items()}
|
|
issue_fingerprints = {issue["id"]: observation(issue) for issue in read(ROOT / "docs/current/runtime-issue-ledger.json").get("issues", [])
|
|
if issue.get("id") and issue.get("capabilityId") in selected_ids}
|
|
try:
|
|
inputs = case_inputs(contract_path, command, contract.get("dependencies", []))
|
|
except (ValueError, OSError) as error:
|
|
print(json.dumps({"status": "case_configuration_error", "error": str(error)}))
|
|
return 78
|
|
# Reuse valid evidence per capability. A large module contract often has
|
|
# one changed or newly implemented ID; rerunning every already-verified
|
|
# browser/WASM case made that single repair pay the full group startup
|
|
# cost. Each reusable subset is still checked against its own source,
|
|
# artifact, dependency and issue fingerprints.
|
|
reusable_ids: set[str] = set()
|
|
reusable = []
|
|
if options.reuse:
|
|
reusable_candidates = accepted_ids([rows[record_id] for record_id in selected_ids])
|
|
by_path = {}
|
|
for record_id in sorted(reusable_candidates):
|
|
proofs = reusable_reports([rows[record_id]], command, inputs, failure_fingerprints, issue_fingerprints)
|
|
if not proofs:
|
|
continue
|
|
reusable_ids.add(record_id)
|
|
for evidence, proof, covered in proofs:
|
|
if evidence not in by_path:
|
|
by_path[evidence] = (evidence, proof, set())
|
|
by_path[evidence][2].update(covered)
|
|
reusable = list(by_path.values())
|
|
pending_ids = selected_ids - reusable_ids
|
|
if reusable and not pending_ids:
|
|
closed = sum(close_tested_issues(report, evidence, issue_fingerprints, covered, refresh_resolved=False)
|
|
for evidence, report, covered in reusable)
|
|
evidence_paths = [str(evidence.relative_to(ROOT)) for evidence, _, _ in reusable]
|
|
pending = [{key: issue.get(key) for key in ("id", "capabilityId", "failureKeys")}
|
|
for issue in read(ROOT / "docs/current/runtime-issue-ledger.json").get("issues", [])
|
|
if issue.get("capabilityId") in selected_ids and issue.get("status", "open") in ACTIVE_STATUSES]
|
|
mapping_required = options.require_issue_closure and bool(pending)
|
|
print(json.dumps({"status": "issue_coverage_required" if mapping_required else "passed", "passed": len(selected_ids),
|
|
"remaining": [], "resolvedIssues": closed, "pendingIssues": pending, "reused": True,
|
|
"evidencePath": evidence_paths[0] if len(evidence_paths) == 1 else None,
|
|
"evidencePaths": evidence_paths, "totalSeconds": round(time.perf_counter() - started, 3)}))
|
|
return 78 if mapping_required else 0
|
|
environment = dict(os.environ, FREECAD_BUSINESS_IDS=json.dumps(sorted(pending_ids)))
|
|
run = None
|
|
try:
|
|
run = execute_case(command, environment)
|
|
report = json.loads(run.stdout)
|
|
except (OSError, ValueError, subprocess.TimeoutExpired) as error:
|
|
def diagnostic_text(value):
|
|
return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value or ""
|
|
stdout = diagnostic_text(run.stdout if run is not None else getattr(error, "stdout", None))
|
|
stderr = diagnostic_text(run.stderr if run is not None else getattr(error, "stderr", None))
|
|
reason = stderr[-3000:] or str(error)
|
|
record_failures(pending_ids, reason)
|
|
print(json.dumps({"status": "failed", "reason": reason, "parseOrExecutionError": str(error),
|
|
"passed": len(reusable_ids), "remaining": sorted(pending_ids),
|
|
"reusedCount": len(reusable_ids), "executedCount": len(pending_ids),
|
|
"exitCode": run.returncode if run is not None else None,
|
|
"stdout": stdout[-3000:], "stderr": stderr[-3000:]}))
|
|
return 1
|
|
results = report.get("results") if isinstance(report, dict) else None
|
|
valid_report = (isinstance(report, dict) and report.get("schema") == "freecad-web-business-cases/v1"
|
|
and isinstance(results, list) and all(isinstance(result, dict) and isinstance(result.get("capabilityId"), str) for result in results)
|
|
and len({result["capabilityId"] for result in results}) == len(results))
|
|
selected_results = [result for result in results if result["capabilityId"] in pending_ids] if valid_report else []
|
|
# A nonzero aggregate exit may describe failed or mixed cases.
|
|
# Preserve their evidence, but never accept an all-pass
|
|
# report whose process failed (for example during cleanup).
|
|
reported_failure_exit = (run.returncode > 0
|
|
and any(not successful_result(result) for result in selected_results))
|
|
if not valid_report or (run.returncode != 0 and not reported_failure_exit):
|
|
results = report.get("results", []) if isinstance(report, dict) else []
|
|
failures = [{"capabilityId": result.get("capabilityId"), "error": result["error"]}
|
|
for result in (results if isinstance(results, list) else [])
|
|
if isinstance(result, dict) and result.get("capabilityId") in selected_ids
|
|
and isinstance(result.get("error"), str) and result["error"]]
|
|
errors = list(dict.fromkeys(result["error"] for result in failures))
|
|
reason = "\n".join(errors) or run.stderr[-2000:] or "invalid business case report"
|
|
record_failures(pending_ids, f"exit={run.returncode}: {reason}")
|
|
print(json.dumps({"status": "failed", "exitCode": run.returncode, "reason": reason,
|
|
"passed": len(reusable_ids), "remaining": sorted(pending_ids),
|
|
"reusedCount": len(reusable_ids), "executedCount": len(pending_ids),
|
|
"failures": failures, "stdout": run.stdout[-3000:], "stderr": run.stderr[-3000:]}))
|
|
return 1
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
digest.cache_clear()
|
|
module_inputs.cache_clear()
|
|
changed_inputs = []
|
|
for record_id, value in sorted(before.items()):
|
|
current = fingerprint(rows[record_id])
|
|
if current != value:
|
|
changed_inputs.append({"capabilityId": record_id, "before": value, "after": current})
|
|
if changed_inputs:
|
|
record_failures(selected_ids, "business inputs changed during execution")
|
|
print(json.dumps({"status": "failed", "reason": "business inputs changed during case execution; run against a stable artifact",
|
|
"changedInputs": changed_inputs}))
|
|
return 1
|
|
if case_inputs(contract_path, command, contract.get("dependencies", [])) != inputs:
|
|
record_failures(selected_ids, "business case dependencies changed during execution")
|
|
print(json.dumps({"status": "failed", "reason": "business case dependencies changed during execution"}))
|
|
return 1
|
|
report["results"] = selected_results
|
|
report.update(module=module, batchId=batch["batchId"], executedAt=now, command=command, processExitCode=run.returncode, scriptSha256=digest(script), failureFingerprints=failure_fingerprints, issueFingerprints=issue_fingerprints, caseInputs=inputs)
|
|
evidence = ROOT / "docs/current/business-evidence" / f"{batch['batchId']}-{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%f')}.json"
|
|
evidence.parent.mkdir(parents=True, exist_ok=True)
|
|
evidence.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
digest.cache_clear()
|
|
ledger = read(LEDGER)
|
|
ledger.update(schema="freecad-web-business-acceptance/v1", updatedAt=now)
|
|
entries = ledger.setdefault("records", {})
|
|
passed = set(reusable_ids)
|
|
for result in report.get("results", []):
|
|
record_id = result.get("capabilityId")
|
|
if record_id not in pending_ids or not successful_result(result):
|
|
continue
|
|
passed.add(record_id)
|
|
entries[record_id] = {"status": "passed", "fingerprint": before[record_id], "evidencePath": str(evidence.relative_to(ROOT)), "evidenceSha256": digest(evidence), "updatedAt": now}
|
|
temporary = LEDGER.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(LEDGER)
|
|
failed_results = [{"capabilityId": result["capabilityId"], "error": result.get("error") or
|
|
("business route not implemented" if result.get("status") == "not_implemented"
|
|
else "business case assertions failed")}
|
|
for result in selected_results if result["capabilityId"] not in passed]
|
|
failure_reason = "\n".join(dict.fromkeys(str(result["error"]) for result in failed_results))
|
|
if selected_ids - passed:
|
|
record_failures(selected_ids - passed, failure_reason or f"business cases missing or failed: {evidence.relative_to(ROOT)}")
|
|
if any(result.get("resolvedFailureKeys") for result in report.get("results", [])):
|
|
adopt_business_evidence(evidence)
|
|
closed = close_tested_issues(report, evidence, issue_fingerprints, passed)
|
|
closed += sum(close_tested_issues(proof, path, issue_fingerprints, covered, refresh_resolved=False)
|
|
for path, proof, covered in reusable)
|
|
pending_issues = [{key: issue.get(key) for key in ("id", "capabilityId", "failureKeys")}
|
|
for issue in read(ROOT / "docs/current/runtime-issue-ledger.json").get("issues", [])
|
|
if issue.get("capabilityId") in selected_ids and issue.get("status", "open") in ACTIVE_STATUSES]
|
|
mapping_required = options.require_issue_closure and passed == selected_ids and bool(pending_issues)
|
|
status = "issue_coverage_required" if mapping_required else "passed" if passed == selected_ids else "acceptance_incomplete"
|
|
print(json.dumps({"status": status, "passed": len(passed), "remaining": sorted(selected_ids - passed),
|
|
"exitCode": run.returncode, "failures": failed_results, "reason": failure_reason,
|
|
"resolvedIssues": closed, "pendingIssues": pending_issues, "evidencePath": str(evidence.relative_to(ROOT)),
|
|
"reused": bool(reusable_ids), "reusedCount": len(reusable_ids),
|
|
"executedCount": len(pending_ids), "totalSeconds": round(time.perf_counter() - started, 3)}))
|
|
return 78 if mapping_required else 0 if passed == selected_ids else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main(sys.argv[1:]))
|