380 lines
26 KiB
Python
380 lines
26 KiB
Python
"""Unattended scheduling must move past stalls without claiming completion."""
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import scheduler_progress as progress
|
|
import prepare_feature_capability_batch as features
|
|
import prepare_implementation_business_batch as business
|
|
import run_module_batch_pipeline as pipeline
|
|
import flush_deferred_module_builds as flush
|
|
import continue_web_task_cycle as continuation
|
|
import run_freecad_web_scheduler as scheduler
|
|
import watch_freecad_web_scheduler as watchdog
|
|
import show_freecad_web_scheduler as viewer
|
|
|
|
|
|
class ProgressTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.directory = tempfile.TemporaryDirectory()
|
|
self.path = Path(self.directory.name) / "progress.json"
|
|
self.patcher = patch.object(progress, "PROGRESS_PATH", self.path)
|
|
self.patcher.start()
|
|
self.batch = {"batchId": "batch", "selected": {"module": "Part"}, "records": [["Part:A"], ["Part:B"]]}
|
|
|
|
def tearDown(self):
|
|
self.patcher.stop()
|
|
self.directory.cleanup()
|
|
|
|
def test_three_attempts_without_outcome_cool_down_only_unfinished_records(self):
|
|
before = {"Part:A": {}, "Part:B": {}}
|
|
after = {"Part:A": {"receipt": {"status": "implementation_ready", "batchId": "new"}}, "Part:B": {}}
|
|
for cycle in range(1, 4):
|
|
result = progress.record_outcome(self.batch, before, after, cycle=cycle, now=1000 + cycle)
|
|
self.assertEqual(progress.cooling_ids(now=1004), {"Part:B"})
|
|
self.assertEqual(result["deferredIds"], ["Part:B"])
|
|
self.assertEqual(progress.cooling_ids(now=1004 + progress.COOLDOWN_SECONDS), set())
|
|
self.assertNotIn("completed", self.path.read_text())
|
|
|
|
def test_source_edits_alone_do_not_reset_stall_counter(self):
|
|
for cycle in range(1, 4):
|
|
result = progress.record_outcome(self.batch, {}, {}, cycle=cycle, file_changes=1, now=1000 + cycle)
|
|
self.assertEqual(result["outcome"], "source_edits_pending_outcome")
|
|
self.assertEqual(progress.cooling_ids(now=1004), {"Part:A", "Part:B"})
|
|
|
|
def test_evidence_refresh_and_new_receipt_batch_are_not_new_business(self):
|
|
old = {"receipt": {"status": "implementation_ready", "batchId": "old"}}
|
|
current = {"receipt": {"status": "implementation_ready", "batchId": "new"}}
|
|
self.assertFalse(progress.advanced(old, current))
|
|
before = {"Part:A": {"acceptance": {"status": "failed", "evidencePath": "old.json"}}, "Part:B": {}}
|
|
after = {name: {"acceptance": {"status": "passed", "evidencePath": "current.json"}} for name in ("Part:A", "Part:B")}
|
|
result = progress.record_outcome(self.batch, before, after, cycle=1, now=1000)
|
|
self.assertEqual(result["firstBusinessPassIds"], ["Part:B"])
|
|
self.assertEqual(result["refreshedBusinessIds"], ["Part:A"])
|
|
summary = progress.progress_summary(progress.read(self.path))
|
|
self.assertEqual(summary["firstBusinessPasses"], 1)
|
|
self.assertEqual(summary["businessRefreshes"], 1)
|
|
self.assertEqual(summary["deliveryMeasuredCycles"], 1)
|
|
|
|
def test_old_activity_history_does_not_claim_new_business_delivery(self):
|
|
summary = progress.progress_summary({"history": [{"outcome": "progress", "seconds": 1}]})
|
|
self.assertEqual(summary["deliveryMeasuredCycles"], 0)
|
|
self.assertEqual(summary["firstBusinessPasses"], 0)
|
|
self.assertTrue(summary["verdict"].startswith("ACTIVITY:"))
|
|
|
|
def test_old_evidence_refresh_does_not_reset_module_stall(self):
|
|
feature = {"batchId": "new-work", "selected": {"module": "Part", "tier": "feature-workflow"},
|
|
"records": [["Part:unfinished"]]}
|
|
accepted = {"batchId": "old-work", "selected": {"module": "Part", "tier": "business-acceptance"},
|
|
"records": [["Part:already-passed"]]}
|
|
old = {"Part:already-passed": {"acceptance": {"status": "passed", "evidencePath": "old.json"}}}
|
|
new = {"Part:already-passed": {"acceptance": {"status": "passed", "evidencePath": "new.json"}}}
|
|
for attempt in range(progress.FEATURE_NO_PROGRESS_LIMIT):
|
|
cycle = attempt * 2 + 1
|
|
progress.record_outcome(feature, {}, {}, cycle=cycle, now=1000 + cycle, file_changes=1)
|
|
previous = progress.read(self.path)["modules"]["Part"].copy()
|
|
refresh = progress.record_outcome(accepted, old, new, cycle=cycle + 1, now=1001 + cycle)
|
|
self.assertTrue(refresh["evidenceRefreshOnly"])
|
|
self.assertEqual(refresh["outcome"], "progress")
|
|
self.assertEqual(refresh["substantiveProgressIds"], [])
|
|
self.assertEqual(refresh["moduleRecovery"]["consecutiveNoProgress"], previous["consecutiveNoProgress"])
|
|
self.assertEqual(refresh["moduleRecovery"].get("retryAt", 0), previous.get("retryAt", 0))
|
|
self.assertIn("Part:unfinished", progress.cooling_ids(now=1010))
|
|
|
|
def test_refresh_only_history_preserves_recovered_stall(self):
|
|
events = []
|
|
for index in range(progress.MODULE_NO_PROGRESS_LIMIT):
|
|
events.extend([{"module": "Part", "outcome": "no_progress", "at": 1000 + index},
|
|
{"module": "Part", "outcome": "progress", "evidenceRefreshOnly": True, "at": 1000.5 + index}])
|
|
modules = progress.module_states({"history": events})
|
|
self.assertEqual(modules["Part"]["consecutiveNoProgress"], progress.MODULE_NO_PROGRESS_LIMIT)
|
|
self.assertGreater(modules["Part"]["retryAt"], 1005)
|
|
|
|
def test_actual_new_delivery_or_repair_resets_stall(self):
|
|
for old in ({}, {"acceptance": {"status": "failed", "evidencePath": "failed.json"}}):
|
|
self.path.write_text(json.dumps({"modules": {"Part": {"consecutiveNoProgress": 2, "retryAt": 1100}}}))
|
|
after = {"Part:A": {"acceptance": {"status": "passed", "evidencePath": "new.json"}}}
|
|
result = progress.record_outcome(self.batch, {"Part:A": old}, after, cycle=1, now=1000)
|
|
self.assertEqual(result["moduleRecovery"]["consecutiveNoProgress"], 0)
|
|
self.assertEqual(result["moduleRecovery"]["retryAt"], 0)
|
|
self.assertFalse(result["evidenceRefreshOnly"])
|
|
self.assertIn("Part:A", result["substantiveProgressIds"])
|
|
|
|
def test_feature_workflow_cools_after_two_ineffective_turns(self):
|
|
batch = {**self.batch, "selected": {"module": "Part", "tier": "feature-workflow"}}
|
|
for cycle in range(1, progress.FEATURE_NO_PROGRESS_LIMIT + 1):
|
|
result = progress.record_outcome(batch, {}, {}, cycle=cycle, now=1000 + cycle)
|
|
self.assertEqual(set(result["deferredIds"]), {"Part:A", "Part:B"})
|
|
self.assertEqual(progress.cooling_ids(now=1004), {"Part:A", "Part:B"})
|
|
|
|
def test_development_lane_is_retained_without_claiming_progress(self):
|
|
batch = {**self.batch, "businessLane": "development"}
|
|
result = progress.record_outcome(batch, {}, {}, cycle=1, file_changes=1, now=1001)
|
|
self.assertEqual(result["businessLane"], "development")
|
|
self.assertEqual(result["outcome"], "source_edits_pending_outcome")
|
|
self.assertEqual(progress.read(self.path)["history"][0]["businessLane"], "development")
|
|
|
|
def test_changing_batch_id_does_not_erase_record_stall_history(self):
|
|
for cycle in range(1, 4):
|
|
progress.record_outcome({**self.batch, "batchId": str(cycle)}, {}, {}, cycle=cycle, now=1000 + cycle)
|
|
self.assertEqual(progress.cooling_ids(now=1004), {"Part:A", "Part:B"})
|
|
|
|
def test_rotating_commands_does_not_bypass_module_stall_detection(self):
|
|
for cycle in range(progress.MODULE_NO_PROGRESS_LIMIT):
|
|
batch = {"batchId": str(cycle), "selected": {"module": "TechDraw"}, "records": [[f"TechDraw:command{cycle}"]]}
|
|
progress.record_outcome(batch, {}, {}, cycle=cycle, now=1000 + cycle, file_changes=1)
|
|
ids = {"TechDraw:new-command", "CAM:cam.import"}
|
|
self.assertEqual(progress.cooling_ids(now=1010, candidates=ids) & ids, {"TechDraw:new-command"})
|
|
self.assertTrue(progress.batch_retry_details({"records": [["TechDraw:new-command"]]}, now=1010))
|
|
self.assertEqual(progress.cooling_modules(now=1010 + progress.MODULE_COOLDOWN_SECONDS), {})
|
|
|
|
def test_previous_history_activates_module_recovery_without_resetting_on_read(self):
|
|
history = [{"module": "TechDraw", "outcome": "no_progress", "at": 1000 + cycle} for cycle in range(8)]
|
|
self.path.write_text(json.dumps({"history": history}))
|
|
first = progress.cooling_modules(now=1010)
|
|
second = progress.cooling_modules(now=1050)
|
|
self.assertEqual(first, second)
|
|
self.assertEqual(first["TechDraw"]["retryAt"], 1007 + progress.MODULE_COOLDOWN_SECONDS)
|
|
|
|
def test_startup_restores_old_module_stall_once(self):
|
|
self.path.write_text(json.dumps({"history": [{"module": "TechDraw", "outcome": "no_progress", "at": cycle} for cycle in range(8)]}))
|
|
self.assertEqual(progress.restore_module_recovery(now=1000), ["TechDraw"])
|
|
retry_at = progress.read(self.path)["modules"]["TechDraw"]["retryAt"]
|
|
self.assertEqual(progress.restore_module_recovery(now=1050), [])
|
|
self.assertEqual(progress.read(self.path)["modules"]["TechDraw"]["retryAt"], retry_at)
|
|
|
|
def test_provider_failures_do_not_create_module_stall(self):
|
|
for cycle in range(8):
|
|
progress.record_outcome(self.batch, {}, {}, cycle=cycle, provider_failure=True, now=1000 + cycle)
|
|
self.assertEqual(progress.cooling_modules(now=1010), {})
|
|
|
|
def test_maintenance_and_business_share_backoff_without_business_mutation(self):
|
|
progress.record_outcome(self.batch, {}, {}, cycle=1, provider_failure=True, now=1000)
|
|
before = progress.read(self.path)
|
|
provider = progress.record_provider_attempt(failed=True, responded=False, now=1001, reason="no first command")
|
|
after = progress.read(self.path)
|
|
self.assertEqual(provider["consecutiveFailures"], 2)
|
|
self.assertEqual(provider["retryAt"], 1041)
|
|
self.assertEqual(after["history"], before["history"])
|
|
self.assertEqual(after["records"], before["records"])
|
|
progress.record_provider_attempt(failed=False, responded=False, now=1002)
|
|
self.assertEqual(progress.read(self.path), after)
|
|
progress.record_provider_attempt(failed=False, responded=True, now=1003)
|
|
self.assertEqual(progress.read(self.path)["provider"]["retryAt"], 0)
|
|
|
|
def test_provider_failure_backoff_is_persisted_and_does_not_cool_business(self):
|
|
for cycle in range(1, 4):
|
|
result = progress.record_outcome(self.batch, {}, {}, cycle=cycle, provider_failure=True, now=1000)
|
|
self.assertEqual(result["provider"]["retryAt"], 1060)
|
|
self.assertEqual(progress.read(self.path)["provider"]["consecutiveFailures"], 3)
|
|
self.assertEqual(progress.cooling_ids(now=1001), set())
|
|
progress.record_outcome(self.batch, {}, {}, cycle=4, now=1081)
|
|
self.assertEqual(progress.read(self.path)["provider"]["consecutiveFailures"], 3)
|
|
progress.record_outcome(self.batch, {}, {}, cycle=5, model_response=True, now=1082)
|
|
self.assertEqual(progress.read(self.path)["provider"]["retryAt"], 0)
|
|
|
|
def test_regression_and_reopened_issues_are_not_forward_progress(self):
|
|
self.assertFalse(progress.advanced({"route": "routed"}, {"route": "implementation_required"}))
|
|
self.assertFalse(progress.advanced({"acceptance": {"status": "passed"}}, {"acceptance": {"status": "failed"}}))
|
|
self.assertFalse(progress.advanced({"issues": [["issue", "resolved"]]}, {"issues": [["issue", "reopened"]]}))
|
|
self.assertTrue(progress.advanced({"issues": [["issue", "open"]]}, {"issues": [["issue", "fix_deployed_pending_use"]]}))
|
|
|
|
def test_feature_planner_selects_other_module_and_keeps_cooled_work_pending(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
registry, issues, state, output = (root / name for name in ("features.json", "issues.json", "state.json", "batch.json"))
|
|
rows = [{"id": "Part:A", "module": "Part", "command": "A"}, {"id": "Sketcher:B", "module": "Sketcher", "command": "B"}]
|
|
registry.write_text(json.dumps({"records": rows}))
|
|
issues.write_text('{"issues":[]}')
|
|
state.write_text('{"active_task":"task"}')
|
|
self.path.write_text(json.dumps({"records": {"Part:A": {"retryAt": time.time() + 300}}}))
|
|
with patch.object(features, "REGISTRY", registry), patch.object(features, "ISSUES", issues), patch.object(features, "STATE", state), patch.object(features, "OUTPUT", output), patch.object(features, "implementation_ready_ids", return_value=set()), patch.object(features, "capability_source_fingerprint", return_value="fixture"), patch.object(features, "stalled_batch_limit", return_value=None), contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(features.main(), 0)
|
|
selected = json.loads(output.read_text())
|
|
self.assertEqual(selected["selected"]["module"], "Sketcher")
|
|
self.assertEqual(selected["deferredRecords"], 1)
|
|
self.path.write_text(json.dumps({"records": {row["id"]: {"retryAt": time.time() + 300} for row in rows}}))
|
|
self.assertEqual(features.main(), 0)
|
|
waiting = json.loads(output.read_text())
|
|
self.assertEqual(waiting["status"], "waiting_for_retry")
|
|
self.assertEqual(waiting["deferredRecords"], 2)
|
|
self.assertEqual(json.loads(registry.read_text())["records"], rows)
|
|
|
|
|
|
class BuildAndPhaseTests(unittest.TestCase):
|
|
def test_excluded_target_retains_pending_work_and_current_diagnostics(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
pending, diagnostics, report = (root / name for name in ("pending.json", "pipeline.json", "flush.json"))
|
|
pending.write_text(json.dumps({"schema": "freecad-web-module-build-pending/v1", "targets": {"build:keep": [{"batchId": "one"}]}}))
|
|
diagnostics.write_text('{"status":"failed","batchId":"current"}')
|
|
with patch.object(flush, "ROOT", root), patch.object(flush, "PENDING_PATH", pending), patch.object(flush, "OUTPUT_PATH", diagnostics), patch.object(flush, "run", side_effect=AssertionError("excluded target must not build")), contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(flush.main(["--exclude-target", "build:keep", "--report-path", str(report)]), 0)
|
|
self.assertEqual(json.loads(report.read_text())["status"], "coalescing")
|
|
self.assertIn("build:keep", json.loads(pending.read_text())["targets"])
|
|
self.assertEqual(json.loads(diagnostics.read_text())["batchId"], "current")
|
|
|
|
def test_mixed_batches_share_link_but_final_batch_is_not_deferred(self):
|
|
batch = {"selected": {"module": "Part", "family": "desktop-command-mixed", "recordCount": 8},
|
|
"queue": [{"module": "Part", "family": "desktop-command-a", "unresolved": 5},
|
|
{"module": "Part", "family": "desktop-command-b", "unresolved": 5}]}
|
|
self.assertTrue(pipeline.should_defer_feature_build(batch, implementation_first=True, pending_count=0, artifacts_exist=True))
|
|
self.assertFalse(pipeline.should_defer_feature_build(batch, implementation_first=True, pending_count=1, artifacts_exist=True))
|
|
batch["queue"][1]["unresolved"] = 3
|
|
self.assertFalse(pipeline.should_defer_feature_build(batch, implementation_first=True, pending_count=0, artifacts_exist=True))
|
|
|
|
def test_waiting_feature_phase_does_not_starve_ready_format_work(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = Path(directory) / "batch.json"
|
|
calls = []
|
|
def run(command, **kwargs):
|
|
calls.append(command[1])
|
|
if command[1] == "tools/prepare_business_acceptance_batch.py":
|
|
value = {"status": "no_ready_batch"}
|
|
elif command[1] == "tools/prepare_feature_capability_batch.py":
|
|
value = {"status": "waiting_for_retry", "retryAt": time.time() + 300}
|
|
elif command[1] == "tools/prepare_product_business_batch.py":
|
|
value = {"status": "no_ready_batch"}
|
|
else:
|
|
value = {"status": "ready", "selected": {"module": "Mesh"}}
|
|
output.write_text(json.dumps(value))
|
|
return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()
|
|
with patch.object(business, "BATCH", output), patch.object(business, "feature_phase_remaining", return_value=1), patch.object(business.subprocess, "run", side_effect=run), contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(business.main(), 0)
|
|
self.assertEqual(calls[-1], "tools/prepare_format_business_batch.py")
|
|
self.assertIn("tools/prepare_feature_capability_batch.py", calls)
|
|
self.assertEqual(json.loads(output.read_text())["selected"]["module"], "Mesh")
|
|
|
|
|
|
class ContinuationCooldownTests(unittest.TestCase):
|
|
@patch.object(business, "acceptance_preempts", return_value=False)
|
|
def test_full_continuation_replaces_preserved_cooled_batch(self, _preempts):
|
|
with tempfile.TemporaryDirectory() as directory, contextlib.ExitStack() as stack:
|
|
root = Path(directory)
|
|
current = root / "docs/current"
|
|
current.mkdir(parents=True)
|
|
batch_path = current / "module-batch-manifest.json"
|
|
checkpoint = current / "continuation-state.json"
|
|
state = current / "task-state.json"
|
|
registry = current / "feature-capability-registry.json"
|
|
issues = current / "runtime-issue-ledger.json"
|
|
progress_path = current / "scheduler-progress.json"
|
|
stack_ids = ["TechDraw:desktop:StackBottom", "TechDraw:desktop:StackDown"]
|
|
batch = {"status": "ready", "strategy": "feature-first-deferred-validation", "taskId": "task", "batchId": "preserved",
|
|
"selected": {"module": "TechDraw", "tier": "feature-workflow", "family": "desktop-command-stack", "recordCount": 2},
|
|
"records": [[record_id, record_id.rsplit(":", 1)[-1]] for record_id in stack_ids]}
|
|
batch_path.write_text(json.dumps(batch))
|
|
checkpoint.write_text(json.dumps({"activeTask": "task", "batchId": "preserved", "batchPolicy": {"minRecords": 2}}))
|
|
state.write_text(json.dumps({"active_task": "task", "tasks": {"task": "in_progress"}}))
|
|
rows = [{"id": record_id, "command": record_id.rsplit(":", 1)[-1], "module": "TechDraw", "implementationStatus": "implementation_required"} for record_id in stack_ids]
|
|
rows.append({"id": "Mesh:desktop:Mesh_Import", "command": "Mesh_Import", "module": "Mesh", "implementationStatus": "implementation_required"})
|
|
registry.write_text(json.dumps({"records": rows}))
|
|
issues.write_text('{"issues":[]}')
|
|
for module, name, value in [
|
|
(progress, "PROGRESS_PATH", progress_path),
|
|
(continuation, "ROOT", root), (continuation, "STATE", state), (continuation, "CHECKPOINT", checkpoint),
|
|
(continuation, "CAPABILITY_REGISTRY", registry),
|
|
(features, "REGISTRY", registry), (features, "ISSUES", issues), (features, "STATE", state), (features, "OUTPUT", batch_path),
|
|
]:
|
|
stack.enter_context(patch.object(module, name, value))
|
|
for module, name, value in [
|
|
(continuation, "implementation_first_enabled", True), (continuation, "load_registry", {"batches": {}}),
|
|
(continuation, "feature_batch_replan_size", None), (continuation, "refresh_feature_registry_if_sources_changed", False),
|
|
(continuation, "run_continuation_check", False), (continuation, "context_requires_refresh", False),
|
|
(continuation, "load_cards", {"task": (root / "card.md", "in_progress", "business task")}),
|
|
(features, "implementation_ready_ids", set()), (features, "stalled_batch_limit", None),
|
|
(features, "capability_source_fingerprint", "fixture"),
|
|
]:
|
|
stack.enter_context(patch.object(module, name, return_value=value))
|
|
self.assertFalse(continuation.batch_requires_refresh())
|
|
for cycle in range(3):
|
|
progress.record_outcome(batch, {}, {}, cycle=cycle, now=time.time())
|
|
self.assertTrue(continuation.batch_requires_refresh())
|
|
calls = []
|
|
def run(command, **kwargs):
|
|
calls.append(command[-1])
|
|
with contextlib.redirect_stdout(io.StringIO()) as output:
|
|
code = features.main()
|
|
return subprocess.CompletedProcess(command, code, output.getvalue(), "")
|
|
stack.enter_context(patch.object(continuation.subprocess, "run", side_effect=run))
|
|
stack.enter_context(patch.object(sys, "argv", ["continue_web_task_cycle.py"]))
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(continuation.main(), 0)
|
|
selected = json.loads(batch_path.read_text())
|
|
self.assertEqual(calls, ["tools/prepare_implementation_business_batch.py"])
|
|
self.assertEqual(selected["selected"]["module"], "Mesh")
|
|
self.assertTrue(set(stack_ids).isdisjoint(progress.selected_ids(selected)))
|
|
self.assertEqual(json.loads(registry.read_text())["records"], rows)
|
|
|
|
def test_launch_guard_rejects_cooled_batch_even_if_replanning_fails(self):
|
|
batch = {"batchId": "stale"}
|
|
with patch.object(scheduler, "batch_retry_details", return_value={"deferredRecords": 4, "retryAt": time.time() + 60}), patch.object(scheduler, "run_local", return_value=subprocess.CompletedProcess([], 1, "", "planner failed")), patch.object(scheduler.time, "sleep"):
|
|
self.assertTrue(scheduler.replan_cooled_batch(batch, lambda **value: None))
|
|
|
|
def test_launch_guard_backs_off_when_planner_returns_success_but_preserves_cooled_batch(self):
|
|
cooling = {"deferredRecords": 4, "retryAt": time.time() + 60}
|
|
updates = []
|
|
with patch.object(scheduler, "batch_retry_details", return_value=cooling), patch.object(scheduler, "run_local", return_value=subprocess.CompletedProcess([], 0, "preserved", "")), patch.object(scheduler.time, "sleep") as sleep:
|
|
self.assertTrue(scheduler.replan_cooled_batch({"batchId": "stale"}, lambda **value: updates.append(value)))
|
|
sleep.assert_called_once_with(30)
|
|
self.assertTrue(any(row.get("status") == "cooldown_replan_backoff" for row in updates))
|
|
|
|
def test_business_continuation_does_not_open_legacy_evidence(self):
|
|
with patch.object(continuation, "implementation_first_enabled", return_value=True), patch.object(continuation.Path, "read_text", side_effect=AssertionError("legacy evidence must not be read")):
|
|
self.assertFalse(continuation.batch_records_have_evidence({"batchId": "current", "records": [["Part:command"]]}))
|
|
|
|
|
|
class AutomaticRecoveryTests(unittest.TestCase):
|
|
def fixture(self):
|
|
started = datetime.fromtimestamp(1000, timezone.utc).isoformat()
|
|
status = {"pid": 42, "status": "codex_turn_running", "moduleBatchId": "batch", "turnStartedAt": started, "cycle": 4}
|
|
batch = {"batchId": "batch", "records": [["TechDraw:Stack"]]}
|
|
ledger = {"records": {"TechDraw:Stack": {"retryAt": 1400, "lastAttemptAt": 990}}}
|
|
return status, batch, ledger
|
|
|
|
def test_watchdog_detects_new_turn_on_cooled_work_not_inflight_completion(self):
|
|
status, batch, ledger = self.fixture()
|
|
self.assertTrue(progress.stalled_launch(status, batch, ledger, now=1010))
|
|
ledger["records"]["TechDraw:Stack"]["lastAttemptAt"] = 1010
|
|
self.assertEqual(progress.stalled_launch(status, batch, ledger, now=1020), {})
|
|
self.assertEqual(progress.stalled_launch({**status, "status": "provider_backoff"}, batch, ledger, now=1020), {})
|
|
|
|
def test_watchdog_requests_one_repair_per_process_without_editing_business_ledger(self):
|
|
status, batch, ledger = self.fixture()
|
|
original = json.dumps(ledger)
|
|
with tempfile.TemporaryDirectory() as directory, patch.object(watchdog, "RECOVERY_PATH", Path(directory) / "recovery.json"), patch.object(watchdog, "scheduler_pid_alive", return_value=True), patch.object(progress.time, "time", return_value=1010), patch.object(watchdog.os, "kill") as kill, contextlib.redirect_stdout(io.StringIO()):
|
|
first = watchdog.request_stall_repair(42, status, batch, ledger)
|
|
second = watchdog.request_stall_repair(42, status, batch, ledger)
|
|
self.assertEqual(first, second)
|
|
self.assertEqual(first["status"], "requested")
|
|
self.assertEqual(kill.call_count, 1)
|
|
self.assertEqual(json.dumps(ledger), original)
|
|
|
|
def test_zero_outcomes_report_stalled_despite_successful_processes(self):
|
|
state = {"history": [{"outcome": "no_progress", "seconds": 30} for _ in range(16)] + [{"outcome": "provider_wait", "seconds": 60} for _ in range(4)]}
|
|
result = progress.progress_summary(state)
|
|
self.assertEqual(result["progressRate"], 0)
|
|
self.assertTrue(result["verdict"].startswith("STALLED:"))
|
|
self.assertEqual(result["noProgressSeconds"], 720)
|
|
|
|
def test_log_tail_is_bounded_and_starts_on_complete_line(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory) / "scheduler.out"
|
|
path.write_text("old text\n" * 10000 + "latest event\n")
|
|
tail = viewer.read_log_tail(path, limit=64)
|
|
self.assertLessEqual(len(tail.encode()), 64)
|
|
self.assertTrue(tail.endswith("latest event\n"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|