409 lines
31 KiB
Python
409 lines
31 KiB
Python
"""Performance controls must retain active work and accurate timing."""
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import analyze_scheduler_timings as timing
|
|
import run_freecad_web_scheduler as scheduler
|
|
import generate_feature_capability_registry as registry
|
|
import prepare_module_implementation_packet as packet
|
|
import scheduler_progress as progress
|
|
import show_freecad_web_scheduler as viewer
|
|
|
|
|
|
class ProviderTests(unittest.TestCase):
|
|
def test_active_transport_survives_cli_first_event_and_post_edit_idle_limits(self):
|
|
for edited in (False, True):
|
|
with self.subTest(edited=edited), tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
events = [{"type": "turn.started"}]
|
|
if edited:
|
|
events.append({"type": "item.completed", "item": {"type": "file_change", "changes": [{"path": "source.cpp", "kind": "update"}]}})
|
|
tail = [{"type": "item.completed", "item": {"type": "agent_message", "text": "completed stream"}}, {"type": "turn.completed"}]
|
|
script = "import sys,time\nsys.stdin.read()\n" + "\n".join(f"print({json.dumps(e)!r},flush=True)" for e in events)
|
|
script += "\ntime.sleep(1.5)\n" + "\n".join(f"print({json.dumps(e)!r},flush=True)" for e in tail)
|
|
metadata = {}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), \
|
|
patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "update_timing_summary"), \
|
|
patch.object(scheduler, "record_unbuilt_edit"), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 8, 1, "test", root / "cycle.log",
|
|
lambda **value: None, turn_metadata=metadata, agent_idle_limit_seconds=0.1, first_activity_timeout_seconds=0.1,
|
|
transport_activity=lambda: {"canWait": True, "phase": "streaming", "responseBudgetSeconds": 5}, cancel_transport=lambda: 1)
|
|
self.assertEqual(result, (0, False, False, False, False))
|
|
self.assertEqual(metadata["cancelledModelRequests"], 1)
|
|
|
|
def test_fresh_http_retries_cannot_reset_first_activity_silence_budget(self):
|
|
state = {"canWait": True, "phase": "request", "elapsedSeconds": 0.01, "responseBudgetSeconds": 130.5}
|
|
self.assertTrue(scheduler.transport_can_continue(state, 60, 60))
|
|
self.assertFalse(scheduler.transport_can_continue(state, 61, 60))
|
|
self.assertFalse(scheduler.transport_can_continue(state, 131, 60))
|
|
self.assertFalse(scheduler.transport_can_continue({**state, "canWait": False}, 1, 60))
|
|
|
|
def test_transport_context_limit_rotates_without_provider_failure(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
metadata = {}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), \
|
|
patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "update_timing_summary"), \
|
|
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", "import sys,time;sys.stdin.read();time.sleep(10)"],
|
|
"fixture", 8, 1, "test", root / "cycle.log", lambda **value: None, turn_metadata=metadata,
|
|
transport_activity=lambda: {"phase": "context_rejected", "canWait": False})
|
|
self.assertTrue(metadata["contextBudgetExceeded"])
|
|
self.assertTrue(metadata["sessionCorrupt"])
|
|
self.assertTrue(result[3])
|
|
self.assertFalse(result[2])
|
|
|
|
def test_slow_first_response_survives_short_post_activity_idle_limit(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
events = [{"type": "thread.started", "thread_id": "fixture"}, {"type": "turn.started"}]
|
|
tail = [{"type": "item.completed", "item": {"type": "agent_message", "text": "ready"}}, {"type": "turn.completed"}]
|
|
script = "import sys,time\nsys.stdin.read()\n" + "\n".join(f"print({json.dumps(e)!r},flush=True)" for e in events)
|
|
script += "\ntime.sleep(1.5)\n" + "\n".join(f"print({json.dumps(e)!r},flush=True)" for e in tail)
|
|
metadata = {}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), \
|
|
patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "update_timing_summary"), \
|
|
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 10, 1, "test", root / "cycle.log",
|
|
lambda **value: None, turn_metadata=metadata,
|
|
agent_idle_limit_seconds=0.2, first_activity_timeout_seconds=4)
|
|
self.assertEqual(result, (0, False, False, False, False))
|
|
self.assertTrue(metadata["turnCompleted"])
|
|
|
|
def test_silent_request_has_bounded_timeout_and_attributable_log(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
metadata = {}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), \
|
|
patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "update_timing_summary"), \
|
|
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", "import sys,time; sys.stdin.read(); time.sleep(20)"],
|
|
"fixture", 10, 1, "test", root / "cycle.log", lambda **value: None,
|
|
turn_metadata=metadata, first_activity_timeout_seconds=0.2)
|
|
self.assertTrue(result[2])
|
|
final = json.loads((root / "cycle.log").read_text().splitlines()[-1])
|
|
self.assertTrue(final["firstActivityTimedOut"])
|
|
self.assertIn("no first command", final["failureReason"])
|
|
self.assertEqual(final["firstActivityTimeoutSeconds"], 0.2)
|
|
|
|
def test_first_response_budget_grows_only_after_silent_timeouts(self):
|
|
with patch.object(scheduler, "FIRST_ACTIVITY_TIMEOUT_SECONDS", 30), patch.object(scheduler, "FIRST_ACTIVITY_MAX_SECONDS", 180):
|
|
self.assertEqual(scheduler.first_activity_budget({}), 30)
|
|
self.assertEqual(scheduler.first_activity_budget({"reason": "no first command", "consecutiveFailures": 1}), 60)
|
|
self.assertEqual(scheduler.first_activity_budget({"reason": "no first command", "consecutiveFailures": 40}), 180)
|
|
self.assertEqual(scheduler.first_activity_budget({"reason": "overloaded", "consecutiveFailures": 40}), 30)
|
|
|
|
def test_transport_silent_wait_is_capped_but_recovery_budget_grows(self):
|
|
with patch.object(scheduler, "FIRST_ACTIVITY_TIMEOUT_SECONDS", 30), patch.object(scheduler, "FIRST_ACTIVITY_TRANSPORT_CAP_SECONDS", 60):
|
|
self.assertEqual(scheduler.first_activity_limit({}, 130.5), 60)
|
|
self.assertEqual(scheduler.first_activity_limit({"reason": "no first command", "consecutiveFailures": 1}, 130.5), 60)
|
|
self.assertEqual(scheduler.first_activity_limit({"reason": "no first command", "consecutiveFailures": 2}, 130.5), 120)
|
|
self.assertEqual(scheduler.first_activity_limit({"reason": "no first command", "consecutiveFailures": 5}, 130.5), 130.5)
|
|
|
|
def test_repeated_unavailability_backs_off_without_clearing_progress(self):
|
|
with patch.object(progress, "PROVIDER_BACKOFF_MAX_SECONDS", 600):
|
|
self.assertEqual([progress.provider_retry_delay(n, 20) for n in range(1, 8)], [20, 40, 60, 120, 240, 480, 600])
|
|
|
|
def test_automatic_recovery_waits_for_active_tool_result(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
request_path = root / "recovery.json"
|
|
start_events = [{"type": "thread.started", "thread_id": "fixture"}, {"type": "turn.started"},
|
|
{"type": "item.started", "item": {"id": "tool", "type": "command_execution", "command": "sleep 1"}}]
|
|
end_event = {"type": "item.completed", "item": {"id": "tool", "type": "command_execution", "command": "sleep 1", "exit_code": 0}}
|
|
request = {"schedulerPid": os.getpid(), "batchId": "batch", "status": "requested"}
|
|
script = "import sys,time,json\nfrom pathlib import Path\nsys.stdin.read()\n"
|
|
script += "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in start_events)
|
|
script += f"\nPath({str(request_path)!r}).write_text({json.dumps(request)!r})\ntime.sleep(1.5)\nprint({json.dumps(end_event)!r},flush=True)\ntime.sleep(20)\n"
|
|
metadata = {"batchId": "batch"}
|
|
with patch.object(progress, "RECOVERY_PATH", request_path), patch.object(scheduler, "ROOT", root), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "update_timing_summary"), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 10, 1, "test", root / "cycle.log", lambda **value: None, turn_metadata=metadata)
|
|
self.assertTrue(metadata["automaticStallRecovery"])
|
|
self.assertEqual(metadata["commandsCompleted"], 1)
|
|
self.assertFalse(result[1])
|
|
self.assertFalse(result[2])
|
|
|
|
def test_overload_is_detected_without_classifying_tool_errors(self):
|
|
self.assertTrue(scheduler.is_stream_disconnect_error("Reconnecting... 1/5 (stream disconnected before completion: Our servers are currently overloaded. Please try again later.)"))
|
|
self.assertTrue(scheduler.is_stream_disconnect_error("concurrency limit exceeded"))
|
|
self.assertFalse(scheduler.is_stream_disconnect_error("Part recompute failed: invalid dimensions"))
|
|
|
|
def test_wait_budget_never_stops_an_active_command(self):
|
|
self.assertFalse(scheduler.provider_wait_expired(True, True, 600))
|
|
self.assertFalse(scheduler.provider_wait_expired(False, False, 600))
|
|
self.assertTrue(scheduler.provider_wait_expired(True, False, 46))
|
|
|
|
def test_recovery_turn_limits_bound_repeated_no_progress(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
progress_path = root / "progress.json"
|
|
pipeline_path = root / "pipeline.json"
|
|
progress_path.write_text(json.dumps({"modules": {"BIM": {"consecutiveNoProgress": 3}}}))
|
|
pipeline_path.write_text(json.dumps({"staticReadiness": {"selected": 16, "ready": 0, "missing": 16}}))
|
|
with patch.object(scheduler, "PROGRESS_PATH", progress_path), patch.object(scheduler, "MODULE_PIPELINE_PATH", pipeline_path):
|
|
self.assertEqual(scheduler.recovery_turn_limits("BIM", True, {}), (2, 4, 45.0))
|
|
|
|
def test_idle_provider_process_stops_before_turn_timeout(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
events = [{"type": "thread.started", "thread_id": "fixture"}, {"type": "turn.started"},
|
|
{"type": "error", "message": "Our servers are currently overloaded"}]
|
|
script = "import sys,time\nsys.stdin.read()\n" + "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in events) + "\ntime.sleep(20)\n"
|
|
statuses = []
|
|
summaries = []
|
|
started = time.monotonic()
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "PROVIDER_NO_PROGRESS_SECONDS", 0.05), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "update_timing_summary", side_effect=lambda *args, **kwargs: summaries.append(args)), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 10, 1, "test", root / "cycle.log", lambda **value: statuses.append(value), turn_metadata={})
|
|
self.assertLess(time.monotonic() - started, 5)
|
|
self.assertTrue(result[2])
|
|
self.assertFalse(result[1])
|
|
self.assertTrue(any(value.get("status") == "codex_turn_stream_error_stopping" for value in statuses))
|
|
self.assertIn("provider/stream-wait", summaries[0][3])
|
|
|
|
def test_stream_error_does_not_wait_for_full_transport_retry_budget(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
events = [{"type": "thread.started", "thread_id": "fixture"}, {"type": "turn.started"},
|
|
{"type": "error", "message": "stream disconnected before completion"}]
|
|
script = "import sys,time\nsys.stdin.read()\n" + "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in events) + "\ntime.sleep(40)\n"
|
|
statuses = []
|
|
started = time.monotonic()
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "PROVIDER_NO_PROGRESS_SECONDS", 0.05), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "update_timing_summary"), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 60, 1, "test", root / "cycle.log", lambda **value: statuses.append(value), turn_metadata={}, transport_activity=lambda: {"phase": "request", "canWait": True, "responseBudgetSeconds": 130.5})
|
|
self.assertLess(time.monotonic() - started, 5)
|
|
self.assertTrue(result[2])
|
|
self.assertTrue(any(value.get("status") == "codex_turn_stream_error_stopping" for value in statuses))
|
|
|
|
def test_provider_notice_does_not_kill_running_tool(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
events = [{"type": "thread.started", "thread_id": "fixture"}, {"type": "turn.started"},
|
|
{"type": "item.started", "item": {"id": "build", "type": "command_execution", "command": "cmake --build fixture"}},
|
|
{"type": "error", "message": "Our servers are currently overloaded"}]
|
|
tail = [{"type": "item.completed", "item": {"id": "build", "type": "command_execution", "command": "cmake --build fixture", "exit_code": 0}}, {"type": "turn.completed"}]
|
|
script = "import sys,time\nsys.stdin.read()\n" + "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in events)
|
|
script += "\ntime.sleep(1.5)\n" + "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in tail)
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "PROVIDER_NO_PROGRESS_SECONDS", 0.05), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "implementation_first_enabled", return_value=False), patch.object(scheduler, "update_timing_summary"), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 10, 1, "test", root / "cycle.log", lambda **value: None, turn_metadata={})
|
|
self.assertEqual(result[0], 0)
|
|
self.assertFalse(result[1])
|
|
self.assertFalse(result[2])
|
|
|
|
|
|
class TimingTests(unittest.TestCase):
|
|
def test_restart_does_not_reuse_logless_pipeline_cycle_ids(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
(root / "cycle-000005.log").write_text("")
|
|
documents = {scheduler.HEARTBEAT_PATH: {"cycle": 6}, scheduler.TIMING_SUMMARY_PATH: {"cycles": [{"cycle": 8}]},
|
|
scheduler.PROGRESS_PATH: {"history": [{"cycle": 7}]}}
|
|
with patch.object(scheduler, "LOG_DIR", root), patch.object(scheduler, "read_progress", side_effect=lambda p: documents[p]):
|
|
self.assertEqual(scheduler.next_cycle_number(), 9)
|
|
|
|
def test_retry_wait_is_included_once_in_total_but_not_in_command_time(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory) / "timing.json"
|
|
path.write_text(json.dumps({"cycles": [{"cycle": 1, "turnSeconds": 30, "phaseTiming": {"request/first-activity-wait": 30},
|
|
"preparationSeconds": 2, "preparationTiming": {"continue": 2}}]}))
|
|
with patch.object(scheduler, "TIMING_SUMMARY_PATH", path):
|
|
scheduler.record_scheduler_wait(1, 30)
|
|
scheduler.record_scheduler_wait(1, 30)
|
|
report = timing.collect_cached(json.loads(path.read_text()), 20)
|
|
self.assertEqual(report["averageEffectiveCycleSeconds"], 92)
|
|
self.assertEqual(report["accountedSeconds"], 92)
|
|
self.assertEqual(report["timedCommandSeconds"], 0)
|
|
self.assertEqual(report["phases"]["scheduler/provider-backoff"]["seconds"], 60)
|
|
summary = json.loads(path.read_text())
|
|
summary["schema"] = "freecad-web-scheduler-timing/v2"
|
|
summary["cycles"][0]["accountedSeconds"] = 30
|
|
with patch.object(viewer, "read_json", return_value=summary):
|
|
visible = viewer.recent_timing(1, 1)
|
|
self.assertEqual(visible["accountedSeconds"], 92)
|
|
self.assertEqual(visible["averageEffectiveCycleSeconds"], 92)
|
|
|
|
def test_wall_timeline_recovers_historical_gaps_without_double_counting(self):
|
|
result = timing.wall_timeline({"1": {"startedAtEpoch": 0, "endedAtEpoch": 30},
|
|
"2": {"startedAtEpoch": 92, "endedAtEpoch": 122}})
|
|
self.assertEqual(result["wallSeconds"], 122)
|
|
self.assertEqual(result["betweenCycleSeconds"], 62)
|
|
self.assertEqual(result["observedCycleSeconds"], 60)
|
|
self.assertEqual(timing.wall_timeline({})["wallSeconds"], 0)
|
|
|
|
def test_idle_cycles_do_not_dilute_active_cycle_average(self):
|
|
report = timing.collect_cached({"cycles": [
|
|
{"cycle": 1, "turnSeconds": 300, "phaseTiming": {"provider/stream-wait": 300}},
|
|
{"cycle": 2, "turnSeconds": 0, "phaseTiming": {"scheduler/no-ready": 0}, "preparationSeconds": 0.5},
|
|
]}, 20)
|
|
self.assertEqual(report["activeCycles"], 1)
|
|
self.assertEqual(report["idleCycles"], 1)
|
|
self.assertEqual(report["averageEffectiveCycleSeconds"], 300)
|
|
|
|
def test_raw_log_counts_each_command_once(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
events = [{"type": "item.started", "item": {"type": "command_execution", "command": "true"}},
|
|
{"type": "item.completed", "item": {"type": "command_execution", "command": "true", "schedulerDurationSeconds": 2}}]
|
|
(root / "cycle-000001.log").write_text("\n".join(json.dumps(row) for row in events))
|
|
report = timing.collect(root, 1, from_logs=True)
|
|
self.assertEqual(report["cycles"][0]["commands"], 1)
|
|
self.assertEqual(report["unmeasuredCommands"], 0)
|
|
|
|
def test_idle_preparation_is_not_double_counted(self):
|
|
with patch.object(scheduler, "update_timing_summary") as update:
|
|
scheduler.record_scheduler_idle_timing(1, "task", 0.5, {"continue": 0.3, "planner": 0.2})
|
|
self.assertEqual(update.call_args.args[2], 0)
|
|
|
|
|
|
class ContextReuseTests(unittest.TestCase):
|
|
def test_script_written_edit_finalizes_for_pipeline_without_idle_failure(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
source = root / "source.cpp"
|
|
source.write_text("before")
|
|
item = {"id": "edit", "type": "command_execution", "command": "python3 patch_source.py"}
|
|
events = [{"type": "thread.started", "thread_id": "fixture"}, {"type": "turn.started"},
|
|
{"type": "item.started", "item": item}]
|
|
script = "import sys,time\nfrom pathlib import Path\nsys.stdin.read()\n"
|
|
script += "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in events)
|
|
script += "\nPath('source.cpp').write_text('after')\n"
|
|
script += f"print({json.dumps({'type': 'item.completed', 'item': {**item, 'exit_code': 0}})!r}, flush=True)\ntime.sleep(20)\n"
|
|
metadata = {"batchId": "batch", "module": "Part"}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "POST_EDIT_FINALIZE_SECONDS", 0.05), \
|
|
patch.object(scheduler, "build_environment", return_value=dict(os.environ)), \
|
|
patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), \
|
|
patch.object(scheduler, "update_timing_summary"), patch.object(scheduler, "record_unbuilt_edit") as record, \
|
|
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 10, 1, "test", root / "cycle.log",
|
|
lambda **value: None, turn_metadata=metadata, source_paths=[source])
|
|
self.assertEqual(result, (0, False, False, False, False))
|
|
self.assertEqual(metadata["fileChanges"], 1)
|
|
self.assertEqual(metadata["observedSourceChanges"], 1)
|
|
self.assertTrue(metadata["postEditAutoFinalize"])
|
|
self.assertGreater(metadata["sourceChangeScanSeconds"], 0)
|
|
record.assert_called_once_with("batch", "Part")
|
|
log = [json.loads(line) for line in (root / "cycle.log").read_text().splitlines()]
|
|
changed = [row["item"]["schedulerObservedSourceChanges"] for row in log if row.get("item", {}).get("schedulerObservedSourceChanges")]
|
|
self.assertEqual(changed, [[str(source)]])
|
|
|
|
def test_explicit_patch_then_read_does_not_double_count_edit(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
source = root / "source.cpp"
|
|
source.write_text("before")
|
|
events = [{"type": "item.completed", "item": {"type": "file_change", "changes": [{"path": "source.cpp", "kind": "update"}]}},
|
|
{"type": "item.completed", "item": {"type": "command_execution", "command": "true", "exit_code": 0}},
|
|
{"type": "turn.completed"}]
|
|
script = "import sys\nfrom pathlib import Path\nsys.stdin.read()\nPath('source.cpp').write_text('after')\n"
|
|
script += "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in events)
|
|
metadata = {}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), \
|
|
patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), \
|
|
patch.object(scheduler, "update_timing_summary"), patch.object(scheduler, "record_unbuilt_edit"), \
|
|
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 10, 1, "test", root / "cycle.log",
|
|
lambda **value: None, turn_metadata=metadata, source_paths=[source])
|
|
self.assertEqual(result[0], 0)
|
|
self.assertEqual(metadata["fileChanges"], 1)
|
|
self.assertEqual(metadata["observedSourceChanges"], 0)
|
|
|
|
def test_desktop_packet_supplies_application_call_sites(self):
|
|
with tempfile.TemporaryDirectory() as directory, patch.object(packet, "ROOT", Path(directory)):
|
|
source = packet.ROOT / "web/src"
|
|
source.mkdir(parents=True)
|
|
(source / "workbench-shell.tsx").write_text("const desktop = new DesktopCommandDispatcher(router);\nconst result = desktopCommands.execute(command, args);\nreturn <main />;\n")
|
|
(source / "desktop-command-dispatcher.ts").write_text("async execute(record, args) {\nreturn this.router.request(record.module, record.command, args);\n}\n")
|
|
regions = packet.application_integration_excerpts()
|
|
self.assertEqual({row["path"] for row in regions}, {"web/src/workbench-shell.tsx", "web/src/desktop-command-dispatcher.ts"})
|
|
self.assertTrue(all(row["role"] == "application-integration" for row in regions))
|
|
|
|
def test_new_integration_scope_invalidates_old_session(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
session = root / "session.json"
|
|
packet_path = root / "packet.json"
|
|
value = {"batchId": "batch", "module": "TechDraw", "supportFiles": [{"path": "native.cpp"}]}
|
|
packet_path.write_text(json.dumps(value))
|
|
with patch.object(scheduler, "SESSION_CACHE_PATH", session), patch.object(scheduler, "IMPLEMENTATION_PACKET_PATH", packet_path):
|
|
scheduler.save_session_cache("TechDraw", "batch", {"threadId": "thread", "scopeFingerprint": scheduler.implementation_scope(value)}, None, True)
|
|
self.assertIsNotNone(scheduler.resumable_session("TechDraw", "batch", same_batch_only=True))
|
|
value["supportFiles"].append({"path": "web/src/workbench-shell.tsx"})
|
|
packet_path.write_text(json.dumps(value))
|
|
self.assertIsNone(scheduler.resumable_session("TechDraw", "batch", same_batch_only=True))
|
|
def test_fifth_read_can_be_followed_by_an_edit_in_the_same_turn(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
events = [{"type": "thread.started", "thread_id": "fixture"}, {"type": "turn.started"}]
|
|
for index in range(5):
|
|
item = {"id": f"read{index}", "type": "command_execution", "command": f"sed -n '1,2p' source{index}.cpp"}
|
|
events.extend([{"type": "item.started", "item": item}, {"type": "item.completed", "item": {**item, "exit_code": 0}}])
|
|
script = "import sys,time\nsys.stdin.read()\n" + "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in events)
|
|
tail = [{"type": "item.completed", "item": {"type": "file_change", "changes": [{"path": "source.cpp", "kind": "update"}]}}, {"type": "turn.completed"}]
|
|
script += "\ntime.sleep(1.5)\n" + "\n".join(f"print({json.dumps(event)!r}, flush=True)" for event in tail)
|
|
metadata = {}
|
|
with patch.object(scheduler, "ROOT", root), patch.object(scheduler, "build_environment", return_value=dict(os.environ)), patch.object(scheduler, "load_failure_state", return_value={"failures": {}}), patch.object(scheduler, "update_timing_summary"), patch.object(scheduler, "record_unbuilt_edit"), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
result = scheduler.run_codex_turn([sys.executable, "-c", script], "fixture", 10, 1, "test", root / "cycle.log", lambda **value: None, turn_metadata=metadata, source_read_limit=4, source_read_hard_limit=5)
|
|
self.assertEqual(result[0], 0)
|
|
self.assertFalse(result[3])
|
|
self.assertTrue(metadata["sourceBudgetReached"])
|
|
self.assertTrue(metadata["turnCompleted"])
|
|
self.assertEqual(metadata["fileChanges"], 1)
|
|
|
|
def test_empty_retry_preserves_previous_clean_session_but_tool_activity_does_not(self):
|
|
with tempfile.TemporaryDirectory() as directory, patch.object(scheduler, "SESSION_CACHE_PATH", Path(directory) / "session.json"):
|
|
scheduler.save_session_cache("Draft", "batch", {"threadId": "clean", "usage": {"input_tokens": 100}}, None, True)
|
|
previous = scheduler.load_session_cache()
|
|
metadata = {"sessionResumed": True, "threadId": "clean", "commandsStarted": 0, "fileChanges": 0}
|
|
scheduler.save_session_cache("Draft", "batch", metadata, previous, False)
|
|
self.assertEqual(scheduler.load_session_cache(), previous)
|
|
self.assertTrue(metadata["previousSessionPreserved"])
|
|
scheduler.save_session_cache("Draft", "batch", {**metadata, "commandsStarted": 1}, previous, False)
|
|
self.assertFalse(scheduler.SESSION_CACHE_PATH.exists())
|
|
|
|
def test_excerpt_keeps_anchor_and_complete_lines(self):
|
|
lines = ["before " + "x" * 120 for _ in range(12)] + ["execute_target(args);", "return result;", "}"]
|
|
start, excerpt = packet.anchored_excerpt(lines, 13, 180)
|
|
self.assertIn("execute_target(args);", excerpt)
|
|
self.assertEqual(excerpt.splitlines(), lines[start - 1:start - 1 + len(excerpt.splitlines())])
|
|
self.assertLessEqual(len(excerpt.encode()), 180)
|
|
|
|
def test_dense_worker_anchor_is_not_cut_to_fake_patch_context(self):
|
|
line = "scope.onmessage = () => { " + "dispatch(request); " * 80 + "};"
|
|
with patch.object(packet, "MAX_EDIT_REGION_BYTES", 200):
|
|
regions = packet.editable_excerpts(packet.ROOT / "worker.ts", "import {dispatch} from './facade';\n" + line, [])
|
|
self.assertEqual(regions[0]["anchorLine"], 2)
|
|
self.assertEqual(regions[0]["excerpt"], line)
|
|
|
|
def test_nearby_anchor_is_not_skipped_when_previous_excerpt_excludes_it(self):
|
|
line = "scope.onmessage = () => { " + "dispatch(request); " * 80 + "};"
|
|
with patch.object(packet, "MAX_EDIT_REGION_BYTES", 200):
|
|
regions = packet.editable_excerpts(packet.ROOT / "worker.ts", "const COMMAND = 'draft.action';\n" + line, ["draft.action"])
|
|
self.assertEqual(len(regions), 2)
|
|
self.assertEqual(regions[1]["excerpt"], line)
|
|
|
|
|
|
class RegistryCacheTests(unittest.TestCase):
|
|
def test_changed_input_and_added_file_invalidate_stamps(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
source = root / "a.ts"
|
|
source.write_text("old")
|
|
before = registry.file_stamps({source})
|
|
source.write_text("new implementation")
|
|
self.assertNotEqual(before, registry.file_stamps({source}))
|
|
another = root / "b.ts"
|
|
another.write_text("new worker")
|
|
self.assertNotEqual(registry.file_stamps({source}), registry.file_stamps({source, another}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|