247 lines
16 KiB
Python
247 lines
16 KiB
Python
"""Failure closure requires evidence for the latest occurrence, not metadata."""
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from threading import Barrier
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import failure_reconciliation as reconciliation
|
|
import run_freecad_web_scheduler as scheduler
|
|
import business_acceptance as acceptance
|
|
import prepare_implementation_business_batch as planner
|
|
import run_module_batch_pipeline as pipeline
|
|
|
|
|
|
class ReconciliationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temp.name)
|
|
self.stack = contextlib.ExitStack()
|
|
for name, value in {"ROOT": self.root, "SOURCE": self.root / "failures.json", "OUTPUT": self.root / "reconciled.json", "REPORT": self.root / "report.md", "MODULES": self.root / "modules.json"}.items():
|
|
self.stack.enter_context(patch.object(reconciliation, name, value))
|
|
reconciliation.MODULES.write_text(json.dumps({"modules": {"Part": {}, "PartDesign": {}, "Mesh": {}, "CAM": {}, "Assembly": {}}}))
|
|
self.entry = {"key": "one", "command": "python3 -c 'print(1)'", "firstSeenAt": "2020-01-01T00:00:00+00:00", "lastSeenAt": "2020-01-01T00:00:01+00:00", "lastExitCode": 1, "firstError": "error", "count": 1, "resolved": False}
|
|
self.save_source(self.entry)
|
|
|
|
def tearDown(self):
|
|
self.stack.close()
|
|
self.temp.cleanup()
|
|
|
|
def save_source(self, *rows):
|
|
reconciliation.SOURCE.write_text(json.dumps({"schema": "freecad-web-scheduler-failures/v1", "failures": {row["key"]: row for row in rows}}))
|
|
|
|
def test_concurrent_writes_keep_independent_staging_files(self):
|
|
target = self.root / "concurrent.json"
|
|
barrier = Barrier(2)
|
|
replace = Path.replace
|
|
|
|
def simultaneous_replace(source, destination):
|
|
barrier.wait(timeout=5)
|
|
return replace(source, destination)
|
|
|
|
values = [{"writer": index, "payload": str(index) * 4096} for index in range(2)]
|
|
with patch.object(Path, "replace", simultaneous_replace), ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = [pool.submit(reconciliation.write, target, value) for value in values]
|
|
for future in futures:
|
|
future.result(timeout=10)
|
|
self.assertIn(json.loads(target.read_text()), values)
|
|
self.assertEqual(list(self.root.glob("*.tmp")), [])
|
|
|
|
def test_failed_replace_preserves_destination_and_cleans_staging(self):
|
|
target = self.root / "preserved.json"
|
|
reconciliation.write(target, {"original": True})
|
|
with patch.object(Path, "replace", side_effect=OSError("replace failed")):
|
|
with self.assertRaisesRegex(OSError, "replace failed"):
|
|
reconciliation.write(target, {"replacement": True})
|
|
self.assertEqual(json.loads(target.read_text()), {"original": True})
|
|
self.assertEqual(list(self.root.glob("*.tmp")), [])
|
|
|
|
def test_classification_preserves_original_and_does_not_imply_resolution(self):
|
|
original = reconciliation.SOURCE.read_bytes()
|
|
self.entry.update(lastReplayExitCode=0, resolved=True, resolutionEvidence="old.json")
|
|
self.save_source(self.entry)
|
|
original = reconciliation.SOURCE.read_bytes()
|
|
report = reconciliation.synchronize()
|
|
self.assertEqual(report["summary"]["closed"], 0)
|
|
self.assertEqual(report["records"]["one"]["sourceSnapshot"], self.entry)
|
|
self.assertEqual(reconciliation.SOURCE.read_bytes(), original)
|
|
self.assertFalse(reconciliation.summary_current()["complete"])
|
|
|
|
def test_real_matching_command_closes_and_new_occurrence_reopens(self):
|
|
run = subprocess.run([sys.executable, "-c", "print(1)"], capture_output=True, text=True, check=True)
|
|
self.entry["resolutionProof"] = reconciliation.record_success(self.entry, self.entry["command"], run.stdout)
|
|
self.save_source(self.entry)
|
|
report = reconciliation.synchronize()
|
|
self.assertEqual(report["records"]["one"]["status"], "resolved")
|
|
self.assertTrue(reconciliation.summary_current()["complete"])
|
|
self.entry.update(count=2, lastSeenAt=datetime.now(timezone.utc).isoformat(), lastError="new failure")
|
|
self.save_source(self.entry)
|
|
reopened = reconciliation.synchronize()
|
|
self.assertEqual(reopened["records"]["one"]["status"], "pending")
|
|
self.assertEqual(len(reopened["records"]["one"]["previousOccurrences"]), 1)
|
|
self.assertFalse(reconciliation.summary_current()["complete"])
|
|
|
|
def test_command_normalization_preserves_quoted_argument_semantics(self):
|
|
self.assertNotEqual(reconciliation.command_tokens("printf '%s' 'a b'"), reconciliation.command_tokens("printf '%s' ab"))
|
|
self.assertEqual(reconciliation.command_tokens(f"/bin/bash -lc 'cd {self.root} && python3 tools/check.py'"), reconciliation.command_tokens("python3 tools/check.py"))
|
|
|
|
def test_longest_module_target_match_and_no_module_guess_from_error_dump(self):
|
|
modules, evidence = reconciliation.owners({"command": "cmake --build build --target FreeCADWebPartDesignRuntimeProbe", "firstError": "Part Mesh CAM"}, ["Part", "PartDesign", "Mesh", "CAM"])
|
|
self.assertEqual(modules, ["PartDesign"])
|
|
self.assertTrue(evidence)
|
|
self.assertEqual(reconciliation.owners({"command": "python3 tools/check.py", "firstError": "Part Mesh CAM"}, ["Part", "Mesh", "CAM"])[0], ["Shared"])
|
|
|
|
def test_build_failure_is_not_archived_as_a_simple_diagnostic(self):
|
|
self.assertEqual(reconciliation.classify({"command": "sed -n 1p file && cmake --build build"})[0], "build_link")
|
|
self.assertNotEqual(reconciliation.classify({"command": "sed -n 1p file && node tools/run_p6_part_module_batch_gate.mjs"})[0], "diagnostic_tool")
|
|
|
|
def test_review_requires_current_sources_and_preserves_failure(self):
|
|
self.entry.update(command="sed -n 1p old-manifest.json")
|
|
self.save_source(self.entry)
|
|
source = self.root / "current-workflow.py"
|
|
source.write_text("manifest_path = 'context/active.json'\n")
|
|
reconciliation.review(["one"], "superseded", "The old single manifest query is replaced by per-task manifests", [source])
|
|
self.assertEqual(reconciliation.read(reconciliation.OUTPUT)["records"]["one"]["status"], "superseded")
|
|
self.assertFalse(reconciliation.read(reconciliation.SOURCE)["failures"]["one"]["resolved"])
|
|
source.write_text("changed source\n")
|
|
self.assertFalse(reconciliation.summary_current()["complete"])
|
|
|
|
def test_review_cannot_hide_build_failures_with_documentation(self):
|
|
self.entry.update(command="cmake --build build --target FreeCADWebPartProbe")
|
|
self.save_source(self.entry)
|
|
source = self.root / "policy.md"
|
|
source.write_text("business first")
|
|
with self.assertRaises(ValueError):
|
|
reconciliation.review(["one"], "not_applicable", "obsolete", [source])
|
|
|
|
def test_business_evidence_requires_explicit_failure_mapping_and_current_fingerprint(self):
|
|
self.entry.update(module="CAM", capabilityIds=["CAM:cam.import"])
|
|
self.save_source(self.entry)
|
|
proof = self.root / "business.json"
|
|
value = {"schema": "freecad-web-business-cases/v1", "executedAt": datetime.now(timezone.utc).isoformat(), "failureFingerprints": {"one": reconciliation.occurrence(self.entry)},
|
|
"results": [{"capabilityId": "CAM:cam.import", "status": "passed", "resolvedFailureKeys": ["one"], "cases": [{"input": {}, "assertions": [{"name": "reproduced input succeeds", "passed": True}]}]}]}
|
|
proof.write_text(json.dumps(value))
|
|
resolution = {"disposition": "resolved", "failureFingerprint": reconciliation.occurrence(self.entry), "evidencePath": "business.json", "evidenceSha256": reconciliation.digest(proof)}
|
|
with patch.object(acceptance, "accepted_ids", return_value={"CAM:cam.import"}):
|
|
self.assertTrue(reconciliation.valid_resolution("one", self.entry, resolution))
|
|
value["results"][0]["resolvedFailureKeys"] = []
|
|
proof.write_text(json.dumps(value))
|
|
resolution["evidenceSha256"] = reconciliation.digest(proof)
|
|
self.assertFalse(reconciliation.valid_resolution("one", self.entry, resolution))
|
|
|
|
def test_module_work_packet_excludes_unrelated_module(self):
|
|
self.entry.update(module="CAM", command="node tools/run_p6_cam_runtime.mjs")
|
|
other = {**self.entry, "key": "other", "module": "Mesh", "command": "node tools/run_p6_mesh_runtime.mjs"}
|
|
self.save_source(self.entry, other)
|
|
reconciliation.synchronize()
|
|
context = reconciliation.module_context("CAM", set())
|
|
self.assertEqual(context["pendingForModule"], 1)
|
|
self.assertEqual(context["groups"][0]["key"], "one")
|
|
|
|
def test_product_and_format_proofs_match_the_actual_module(self):
|
|
reconciliation.MODULES.write_text(json.dumps({"modules": {"Web": {}, "Part": {}}}))
|
|
for capability, module in [("product:task-panel", "Web"), ("business-format:Part:export:.stl", "Part")]:
|
|
self.entry.update(module=module, capabilityIds=[capability])
|
|
proof = self.root / "business.json"
|
|
value = {"schema": "freecad-web-business-cases/v1", "executedAt": datetime.now(timezone.utc).isoformat(),
|
|
"failureFingerprints": {"one": reconciliation.occurrence(self.entry)}, "results": [
|
|
{"capabilityId": capability, "status": "passed", "resolvedFailureKeys": ["one"],
|
|
"cases": [{"input": {}, "assertions": [{"name": "actual business", "passed": True}]}]}]}
|
|
proof.write_text(json.dumps(value))
|
|
resolution = {"disposition": "resolved", "failureFingerprint": reconciliation.occurrence(self.entry),
|
|
"evidencePath": "business.json", "evidenceSha256": reconciliation.digest(proof)}
|
|
with patch.object(acceptance, "accepted_ids", return_value={capability}):
|
|
self.assertTrue(reconciliation.valid_resolution("one", self.entry, resolution))
|
|
self.entry["module"] = "Part" if module == "Web" else "Web"
|
|
self.assertFalse(reconciliation.valid_resolution("one", self.entry, resolution))
|
|
|
|
def test_issue_registration_uses_exact_business_argument_not_failure_task_ids(self):
|
|
entry = {"capabilityIds": ["failure:Web:first"],
|
|
"command": '/bin/bash -lc "python3 tools/record_runtime_issue.py product:task-panel --error bad"'}
|
|
self.assertEqual(reconciliation.declared_business_ids(entry), {"product:task-panel"})
|
|
entry["command"] = "python3 other.py product:task-panel"
|
|
self.assertEqual(reconciliation.declared_business_ids(entry), {"failure:Web:first"})
|
|
entry["capabilityIds"] = ["product:other"]
|
|
entry["command"] = "python3 tools/record_runtime_issue.py product:task-panel --error bad"
|
|
self.assertEqual(reconciliation.declared_business_ids(entry), {"product:other"})
|
|
|
|
def test_linked_business_issue_suspends_review_until_business_fix_is_deployed(self):
|
|
issue_path = self.root / "docs/current/runtime-issue-ledger.json"
|
|
issue_path.parent.mkdir(parents=True)
|
|
issue = {"failureKeys": ["one"], "capabilityId": "CAM:cam.import", "status": "open"}
|
|
issue_path.write_text(json.dumps({"issues": [issue]}))
|
|
self.assertTrue(reconciliation.pending_business_repair(["one"]))
|
|
self.assertFalse(reconciliation.pending_business_repair(["other"]))
|
|
issue["status"] = "fix_deployed_pending_use"
|
|
issue_path.write_text(json.dumps({"issues": [issue]}))
|
|
self.assertFalse(reconciliation.pending_business_repair(["one"]))
|
|
|
|
def test_corrupt_source_does_not_satisfy_final_reconciliation(self):
|
|
reconciliation.SOURCE.write_text('{"schema":"freecad-web-scheduler-failures/v1","failures":{"bad":null}}')
|
|
reconciliation.synchronize()
|
|
self.assertFalse(reconciliation.summary_current()["complete"])
|
|
|
|
def test_recording_preserves_first_error_and_resolved_history(self):
|
|
old = {str(index): {"key": str(index), "resolved": True, "lastSeenAt": "2020"} for index in range(125)}
|
|
state = {"failures": old}
|
|
first = scheduler.record_command_failure("task", 1, "command", 1, "first", state=state, persist=False, module="CAM", capability_ids=["CAM:cam.import"])
|
|
scheduler.record_command_failure("task", 2, "command", 1, "second", state=state, persist=False)
|
|
self.assertEqual(first["firstError"], "first")
|
|
self.assertEqual(first["lastError"], "second")
|
|
self.assertEqual(len(state["failures"]), 126)
|
|
self.assertEqual(first["capabilityIds"], ["CAM:cam.import"])
|
|
|
|
def test_full_completion_requires_current_failure_reconciliation(self):
|
|
audit = self.root / "docs/current/web-business-completion-audit.json"
|
|
audit.parent.mkdir(parents=True)
|
|
audit.write_text('{"businessFunctionalComplete":true}')
|
|
reconciliation.synchronize()
|
|
with patch.object(acceptance, "ROOT", self.root), patch.object(acceptance, "requirements", return_value=[{"id": "CAM:cam.import"}]), patch.object(acceptance, "accepted_ids", return_value={"CAM:cam.import"}):
|
|
self.assertFalse(acceptance.business_complete())
|
|
self.entry["resolutionProof"] = reconciliation.record_success(self.entry, self.entry["command"], "1")
|
|
self.save_source(self.entry)
|
|
reconciliation.synchronize()
|
|
self.assertTrue(acceptance.business_complete())
|
|
|
|
def test_module_handoff_keeps_ready_business_ahead_of_historical_review(self):
|
|
output = self.root / "batch.json"
|
|
output.write_text(json.dumps({"selected": {"module": "CAM", "tier": "feature-workflow"}}))
|
|
calls = []
|
|
def run(command, **kwargs):
|
|
calls.append(command)
|
|
if command[1] == "tools/prepare_business_acceptance_batch.py":
|
|
value = {"status": "no_ready_batch"}
|
|
else:
|
|
value = {"status": "ready", "selected": {"module": "CAM", "tier": "failure-reconciliation"}} if "--module" in command else {"status": "ready", "selected": {"module": "Mesh", "tier": "feature-workflow"}}
|
|
output.write_text(json.dumps(value))
|
|
return subprocess.CompletedProcess(command, 0, "", "")
|
|
with patch.object(planner, "BATCH", output), patch.object(planner, "feature_phase_remaining", return_value=1), patch.object(planner.subprocess, "run", side_effect=run), contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(planner.main(), 0)
|
|
self.assertEqual(json.loads(output.read_text())["selected"], {"module": "Mesh", "tier": "feature-workflow"})
|
|
self.assertEqual(planner.main(), 0)
|
|
self.assertEqual(json.loads(output.read_text())["selected"]["module"], "Mesh")
|
|
self.assertEqual(sum("--module" in command for command in calls), 0)
|
|
self.assertFalse(any("tools/prepare_failure_reconciliation_batch.py" in command for command in calls))
|
|
|
|
def test_pending_review_does_not_start_build_or_report_a_new_command_failure(self):
|
|
packet_path, batch_path = self.root / "packet.json", self.root / "batch.json"
|
|
packet_path.write_text(json.dumps({"batchId": "review", "outerPipelineAvailable": True}))
|
|
batch_path.write_text(json.dumps({"batchId": "review", "selected": {"tier": "failure-reconciliation"}, "failureKeys": ["one"]}))
|
|
report = {}
|
|
with patch.object(pipeline, "PACKET_PATH", packet_path), patch.object(pipeline, "BATCH_PATH", batch_path), patch.object(pipeline, "run_step", return_value={"name": "failure-reconciliation", "exitCode": 1}) as run, patch.object(pipeline, "save", side_effect=lambda value: report.update(value)), contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(pipeline.main(), 0)
|
|
self.assertEqual(run.call_count, 1)
|
|
self.assertEqual(report["status"], "reconciliation_incomplete")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|