216 lines
13 KiB
Python
216 lines
13 KiB
Python
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import run_freecad_web_scheduler as scheduler
|
|
import run_module_batch_pipeline as pipeline
|
|
import run_business_acceptance as acceptance_runner
|
|
import business_acceptance as acceptance
|
|
|
|
|
|
class RecoveryTests(unittest.TestCase):
|
|
def test_business_exception_survives_large_geometry_output(self):
|
|
import subprocess
|
|
message = "TypeError: kernel.FS.readFile is not a function at cases.mjs:43"
|
|
stdout = json.dumps({"status": "failed", "stdout": "<svg>" + "geometry " * 2000, "stderr": message})
|
|
result = subprocess.CompletedProcess(["python3", "cases.py"], 1, stdout, "")
|
|
with patch.object(pipeline, "run_business_process", return_value=result):
|
|
step = pipeline.run_step("business-acceptance", ["python3", "cases.py"], {}, 1)
|
|
self.assertIn(message, step["stderr"])
|
|
self.assertNotIn("geometry", step["stderr"])
|
|
self.assertLess(len(step["stderr"]), 500)
|
|
self.assertEqual(step["exitCode"], 1)
|
|
|
|
def test_business_error_extraction_handles_nested_and_non_json_output(self):
|
|
value = json.dumps({"status": "failed", "stdout": json.dumps({"results": [
|
|
{"capabilityId": "Part:case", "status": "failed", "error": "invalid batch schema"}]})})
|
|
diagnostics = pipeline.business_failure_diagnostics(value)
|
|
self.assertIn("invalid batch schema", diagnostics)
|
|
self.assertIn("Part:case", diagnostics)
|
|
self.assertEqual(pipeline.business_failure_diagnostics("compiler error"), "")
|
|
|
|
def test_provider_backoff_allows_local_cases_but_not_repeated_unchanged_failures(self):
|
|
import hashlib
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
source = root / "cases.py"
|
|
source.write_text("original")
|
|
batch = {"status": "ready", "batchId": "b", "selected": {"module": "CAM", "tier": "business-acceptance"}}
|
|
packet = {"batchId": "b", "outerPipelineAvailable": True}
|
|
previous = {}
|
|
documents = {scheduler.MODULE_BATCH_PATH: batch, scheduler.IMPLEMENTATION_PACKET_PATH: packet,
|
|
scheduler.MODULE_PIPELINE_PATH: previous}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "read_progress", side_effect=lambda p: documents[p]), \
|
|
patch.object(scheduler, "runtime_pipeline_ready", return_value=True), patch.object(scheduler, "batch_retry_details", return_value={}):
|
|
self.assertTrue(scheduler.local_pipeline_available_during_backoff())
|
|
previous.update(batchId="b", status="failed", failedStage="preflight",
|
|
supportHashes={"cases.py": hashlib.sha256(source.read_bytes()).hexdigest()})
|
|
self.assertFalse(scheduler.local_pipeline_available_during_backoff())
|
|
source.write_text("fixed")
|
|
self.assertTrue(scheduler.local_pipeline_available_during_backoff())
|
|
packet["batchId"] = "other"
|
|
self.assertFalse(scheduler.local_pipeline_available_during_backoff())
|
|
packet["batchId"] = "b"
|
|
batch["status"] = "waiting_for_retry"
|
|
self.assertFalse(scheduler.local_pipeline_available_during_backoff())
|
|
|
|
def test_model_launch_is_guarded_after_local_pipeline_and_before_session_preparation(self):
|
|
source = (scheduler.ROOT / "tools/run_freecad_web_scheduler.py").read_text()
|
|
gate = source.index('localPipelineDuringBackoff=ran_static_pipeline')
|
|
self.assertLess(source.index('ran_static_pipeline = False'), gate)
|
|
self.assertLess(gate, source.index('previous_session = resumable_session(', gate))
|
|
|
|
def test_acceptance_failure_never_enters_compiler_recovery(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
batch = {"batchId": "b", "selected": {"module": "Part", "tier": "feature-workflow", "family": "runtime-issue"}}
|
|
report = {"batchId": "b", "status": "failed", "failedStage": "business-acceptance",
|
|
"steps": [{"name": "business-acceptance", "exitCode": 78, "stdout": '{"status":"issue_coverage_required"}'}]}
|
|
packet = {"batchId": "b", "acceptanceContract": "docs/business-acceptance/Part.json", "records": [], "editableExcerpts": []}
|
|
paths = {}
|
|
for key, value in (("MODULE_BATCH_PATH", batch), ("MODULE_PIPELINE_PATH", report), ("IMPLEMENTATION_PACKET_PATH", packet)):
|
|
paths[key] = root / f"{key}.json"
|
|
paths[key].write_text(json.dumps(value))
|
|
with patch.multiple(scheduler, **paths), patch.object(scheduler, "diagnostic_cmake_excerpt", side_effect=AssertionError("not a compiler error")):
|
|
prompt = scheduler.build_prompt("task", failures=[])
|
|
self.assertIn("issue_coverage_required", prompt)
|
|
self.assertIn("docs/business-acceptance/Part.json", prompt)
|
|
self.assertNotIn("targetCMake", prompt)
|
|
self.assertEqual(scheduler.compile_failure_steps(report, "b"), [])
|
|
report.update(failedStage="build", steps=[{"name": "build", "exitCode": 1}])
|
|
self.assertEqual(len(scheduler.compile_failure_steps(report, "b")), 1)
|
|
self.assertEqual(scheduler.compile_failure_steps(report, "another-batch"), [])
|
|
|
|
def test_runtime_fast_path_requires_real_script_and_declared_business(self):
|
|
with tempfile.TemporaryDirectory() as directory, patch.object(scheduler, "ROOT", Path(directory)):
|
|
root = Path(directory)
|
|
contract = root / "docs/business-acceptance/Part.json"
|
|
contract.parent.mkdir(parents=True)
|
|
batch = {"selected": {"module": "Part", "family": "runtime-issue"}, "records": [["Part:create"]]}
|
|
self.assertFalse(scheduler.runtime_pipeline_ready(batch))
|
|
contract.write_text(json.dumps({"command": ["python3", "cases.py"], "coveredIds": ["Part:create"]}))
|
|
self.assertFalse(scheduler.runtime_pipeline_ready(batch))
|
|
(root / "cases.py").write_text("pass\n")
|
|
self.assertTrue(scheduler.runtime_pipeline_ready(batch))
|
|
batch["records"].append(["Part:other"])
|
|
self.assertFalse(scheduler.runtime_pipeline_ready(batch))
|
|
|
|
def test_failed_case_is_not_replayed_until_relevant_input_changes(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
source = root / "cases.py"
|
|
source.write_text("old\n")
|
|
report_path = root / "pipeline.json"
|
|
report_path.write_text(json.dumps({"batchId": "b", "status": "failed", "failedStage": "business-acceptance", "supportHashes": {"cases.py": "old"}}))
|
|
packet = {"supportFiles": [{"path": "cases.py", "sha256": "old"}]}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "MODULE_PIPELINE_PATH", report_path):
|
|
self.assertTrue(scheduler.failed_pipeline_is_unchanged("b", packet))
|
|
packet["supportFiles"][0]["sha256"] = "new"
|
|
self.assertFalse(scheduler.failed_pipeline_is_unchanged("b", packet))
|
|
|
|
def test_content_change_with_preserved_size_and_mtime_invalidates_hash_cache(self):
|
|
with tempfile.TemporaryDirectory() as directory, patch.object(pipeline, "ROOT", Path(directory)):
|
|
source = Path(directory) / "source.cpp"
|
|
source.write_text("first")
|
|
before, _ = pipeline.file_signature(source)
|
|
stat = source.stat()
|
|
source.write_text("other")
|
|
os.utime(source, ns=(stat.st_atime_ns, stat.st_mtime_ns))
|
|
after, hit = pipeline.file_signature(source, before)
|
|
self.assertFalse(hit)
|
|
self.assertNotEqual(before["sha256"], after["sha256"])
|
|
|
|
|
|
class PipelineTests(unittest.TestCase):
|
|
def test_successful_build_survives_failed_acceptance_and_ninja_still_checks_dependencies(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
tree = root / "build"
|
|
tree.mkdir()
|
|
runtime, wasm = tree / "probe.js", tree / "probe.wasm"
|
|
source = root / "source.cpp"
|
|
source.write_text("source")
|
|
batch = {"batchId": "b", "selected": {"module": "Part", "family": "runtime-issue"}, "records": [["Part:create"]]}
|
|
packet = {"batchId": "b", "module": "Part", "outerPipelineAvailable": True, "build": {"target": "probe", "tree": "build", "bin": "build", "runtime": "build/probe.js", "wasm": "build/probe.wasm"},
|
|
"supportFiles": [{"path": "source.cpp", "sha256": "source"}]}
|
|
paths = {key: root / f"{key}.json" for key in ("BATCH_PATH", "PACKET_PATH", "OUTPUT_PATH", "BUILD_CACHE_PATH", "PENDING_BUILD_PATH")}
|
|
paths["BATCH_PATH"].write_text(json.dumps(batch))
|
|
paths["PACKET_PATH"].write_text(json.dumps(packet))
|
|
calls = []
|
|
fail_case, native_dirty = True, True
|
|
native_generation = 0
|
|
def run(name, command, environment, timeout):
|
|
nonlocal native_dirty, native_generation
|
|
calls.append(name)
|
|
if name == "build" and native_dirty:
|
|
native_generation += 1
|
|
runtime.write_text("built " + str(native_generation))
|
|
wasm.write_text("wasm " + str(native_generation))
|
|
native_dirty = False
|
|
code = 78 if name == "business-acceptance" and fail_case else 0
|
|
if name == "business-acceptance":
|
|
self.assertIn("--require-issue-closure", command)
|
|
return {"name": name, "command": command, "exitCode": code, "seconds": 0.01, "stdout": "", "stderr": ""}
|
|
with patch.multiple(pipeline, ROOT=root, **paths), patch.object(pipeline, "implementation_first_enabled", return_value=True), \
|
|
patch.object(pipeline, "run_step", side_effect=run), \
|
|
patch.object(pipeline, "record_implementation_batch", return_value={"remainingRecordCount": 0}), \
|
|
contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(pipeline.main(), 78)
|
|
self.assertTrue(paths["BUILD_CACHE_PATH"].is_file())
|
|
initial = runtime.read_text()
|
|
fail_case = False
|
|
calls.clear()
|
|
self.assertEqual(pipeline.main(), 0)
|
|
reused = json.loads(paths["OUTPUT_PATH"].read_text())
|
|
self.assertFalse(reused["buildRequired"])
|
|
self.assertEqual(runtime.read_text(), initial)
|
|
self.assertIn("build", calls)
|
|
self.assertIn("artifact-sync", calls)
|
|
native_dirty = True
|
|
calls.clear()
|
|
self.assertEqual(pipeline.main(), 0)
|
|
self.assertTrue(json.loads(paths["OUTPUT_PATH"].read_text())["buildRequired"])
|
|
self.assertIn("artifact-sync", calls)
|
|
|
|
|
|
class IssueCoverageTests(unittest.TestCase):
|
|
def test_passing_business_does_not_claim_uncovered_issue_is_closed(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
(root / "docs/current").mkdir(parents=True)
|
|
(root / "docs/business-acceptance").mkdir()
|
|
row = {"id": "Part:create", "module": "Part"}
|
|
issue = {"id": "runtime:one", "capabilityId": row["id"], "status": "fix_deployed_pending_use", "occurrences": 1}
|
|
issue_path = root / "docs/current/runtime-issue-ledger.json"
|
|
issue_path.write_text(json.dumps({"issues": [issue]}))
|
|
result = {"schema": "freecad-web-business-cases/v1", "results": [{"capabilityId": row["id"], "status": "passed", "cases": [{"input": {"length": 2}, "assertions": [{"name": "volume", "passed": True}]}]}]}
|
|
script = root / "case.py"
|
|
script.write_text(f"print({json.dumps(result)!r})\n")
|
|
(root / "docs/business-acceptance/Part.json").write_text(json.dumps({"command": ["python3", "case.py"]}))
|
|
ledger = root / "docs/current/business-acceptance.json"
|
|
with patch.object(acceptance_runner, "ROOT", root), patch.object(acceptance_runner, "LEDGER", ledger), \
|
|
patch.object(acceptance_runner, "requirements", return_value=[row]), \
|
|
patch.object(acceptance, "ROOT", root), patch.object(acceptance, "LEDGER", ledger), \
|
|
contextlib.redirect_stdout(io.StringIO()):
|
|
args = ["--module", "Part", "--ids", row["id"], "--require-issue-closure"]
|
|
self.assertEqual(acceptance_runner.main(args), 78)
|
|
self.assertEqual(json.loads(ledger.read_text())["records"][row["id"]]["status"], "passed")
|
|
self.assertEqual(json.loads(issue_path.read_text())["issues"][0]["status"], "fix_deployed_pending_use")
|
|
result["results"][0]["resolvedIssueIds"] = [issue["id"]]
|
|
script.write_text(f"print({json.dumps(result)!r})\n")
|
|
self.assertEqual(acceptance_runner.main(args), 0)
|
|
self.assertEqual(json.loads(issue_path.read_text())["issues"][0]["status"], "resolved")
|
|
acceptance.digest.cache_clear()
|
|
acceptance.module_inputs.cache_clear()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|