261 lines
11 KiB
Python
261 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Keep the FreeCAD Web scheduler alive until the ledger reaches TERMINATE.
|
|
|
|
The watchdog never changes business statuses or claims work. It checks both
|
|
process liveness and launches that violate the persisted business cooldown,
|
|
requesting a drained restart when a stalled batch bypasses the planner.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from implementation_first import implementation_first_enabled, user_validation_deferred, implementation_scope_complete
|
|
from business_acceptance import business_complete
|
|
from scheduler_progress import PROGRESS_PATH, RECOVERY_PATH, stalled_launch
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
STATE = ROOT / "docs/current/task-state.json"
|
|
WATCHDOG_LOCK = ROOT / "docs/current/freecad-web-scheduler-watchdog.lock"
|
|
WATCHDOG_PID = ROOT / "docs/current/freecad-web-scheduler-watchdog.pid"
|
|
WATCHDOG_STATUS = ROOT / "docs/current/freecad-web-scheduler-watchdog-status.json"
|
|
SCHEDULER_PID = ROOT / "docs/current/freecad-web-scheduler.pid"
|
|
SCHEDULER_STATUS = ROOT / "docs/current/freecad-web-scheduler-status.json"
|
|
SCHEDULER_SCRIPT = ROOT / "tools/run_freecad_web_scheduler.py"
|
|
OUTPUT = ROOT / "docs/current/freecad-web-scheduler.out"
|
|
MODULE_BATCH = ROOT / "docs/current/module-batch-manifest.json"
|
|
|
|
|
|
def now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def read_json(path: Path) -> dict:
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
return value if isinstance(value, dict) else {}
|
|
except (FileNotFoundError, json.JSONDecodeError):
|
|
return {}
|
|
|
|
|
|
def terminal() -> bool:
|
|
if user_validation_deferred():
|
|
return implementation_scope_complete()
|
|
if implementation_first_enabled():
|
|
return business_complete()
|
|
state = read_json(STATE)
|
|
counts = state.get("counts", {})
|
|
return not state.get("active_task") and all(counts.get(key, 0) == 0 for key in ("planned", "in_progress", "blocked"))
|
|
|
|
|
|
def pid_alive(pid: int | None) -> bool:
|
|
return bool(pid and pid > 0 and os.path.exists(f"/proc/{pid}"))
|
|
|
|
|
|
def scheduler_pid_alive(pid: int | None) -> bool:
|
|
"""Require both liveness and the expected scheduler command line."""
|
|
if not pid_alive(pid):
|
|
return False
|
|
try:
|
|
command_line = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace")
|
|
except (FileNotFoundError, OSError):
|
|
return False
|
|
return str(SCHEDULER_SCRIPT) in command_line or "tools/run_freecad_web_scheduler.py" in command_line
|
|
|
|
|
|
def discover_scheduler_pid() -> int | None:
|
|
"""Find a scheduler process when its PID file is between atomic writes."""
|
|
for entry in Path("/proc").iterdir():
|
|
if not entry.name.isdigit():
|
|
continue
|
|
pid = int(entry.name)
|
|
if scheduler_pid_alive(pid):
|
|
return pid
|
|
return None
|
|
|
|
|
|
def read_pid(path: Path) -> int | None:
|
|
try:
|
|
return int(path.read_text(encoding="ascii").strip())
|
|
except (FileNotFoundError, ValueError):
|
|
return None
|
|
|
|
|
|
def write_status(value: dict) -> None:
|
|
value["updatedAt"] = now()
|
|
temporary = WATCHDOG_STATUS.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(WATCHDOG_STATUS)
|
|
|
|
|
|
def scheduler_args(args: argparse.Namespace) -> list[str]:
|
|
command = [sys.executable, str(SCHEDULER_SCRIPT), "--timeout", str(args.timeout), "--idle-delay", str(args.idle_delay)]
|
|
if args.model:
|
|
command.extend(["--model", args.model])
|
|
return command
|
|
|
|
|
|
def request_stall_repair(pid: int, status: dict, batch: dict, progress: dict) -> dict:
|
|
if status.get("pid") != pid or not scheduler_pid_alive(pid):
|
|
return {}
|
|
if status.get("executionMode") == "periodic-optimization":
|
|
return {}
|
|
fault = stalled_launch(status, batch, progress)
|
|
if not fault:
|
|
return {}
|
|
previous = read_json(RECOVERY_PATH)
|
|
if previous.get("schedulerPid") == pid and previous.get("status") == "requested":
|
|
return previous
|
|
request = {**fault, "schedulerPid": pid, "status": "requested", "detectedAt": now(),
|
|
"repair": "drain active tools, restart scheduler, replan cooled records",
|
|
"requestCount": int(previous.get("requestCount", 0)) + 1}
|
|
RECOVERY_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = RECOVERY_PATH.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(request, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(RECOVERY_PATH)
|
|
try:
|
|
os.kill(pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
pass
|
|
print(f"WATCHDOG_STALL_REPAIR: pid={pid} batch={fault['batchId']} records={len(fault['recordIds'])}", flush=True)
|
|
return request
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--interval", type=float, default=10.0, help="seconds between scheduler liveness checks")
|
|
parser.add_argument("--timeout", type=float, default=1800.0, help="maximum seconds per Codex turn (default: 1800)")
|
|
parser.add_argument("--idle-delay", type=float, default=30.0, help="scheduler delay when no ready leaf exists")
|
|
parser.add_argument("--model", default=None, help="optional Codex model override")
|
|
args = parser.parse_args()
|
|
|
|
WATCHDOG_PID.parent.mkdir(parents=True, exist_ok=True)
|
|
with WATCHDOG_LOCK.open("a+", encoding="ascii") as lock:
|
|
try:
|
|
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
print("WATCHDOG_ALREADY_RUNNING", file=sys.stderr)
|
|
return 2
|
|
|
|
WATCHDOG_PID.write_text(str(os.getpid()) + "\n", encoding="ascii")
|
|
state = {
|
|
"schema": "freecad-web-scheduler-watchdog/v1",
|
|
"watchdogPid": os.getpid(),
|
|
"status": "starting",
|
|
"intervalSeconds": args.interval,
|
|
"schedulerScript": str(SCHEDULER_SCRIPT.relative_to(ROOT)),
|
|
"restartCount": 0,
|
|
"startedAt": now(),
|
|
"lastSchedulerPid": None,
|
|
"lastSchedulerExitCode": None,
|
|
"lastRestartAt": None,
|
|
"lastRestartReason": None,
|
|
}
|
|
stopping = False
|
|
|
|
def stop(_signum: int, _frame: object) -> None:
|
|
nonlocal stopping
|
|
stopping = True
|
|
|
|
signal.signal(signal.SIGINT, stop)
|
|
signal.signal(signal.SIGTERM, stop)
|
|
|
|
scheduler_process: subprocess.Popen[str] | None = None
|
|
try:
|
|
while not stopping:
|
|
if terminal():
|
|
if scheduler_process and scheduler_process.poll() is None:
|
|
scheduler_process.terminate()
|
|
state.update(status="terminated", schedulerPid=None, terminatedAt=now())
|
|
write_status(state)
|
|
print("WATCHDOG_TERMINATE: ledger reached TERMINATE", flush=True)
|
|
return 0
|
|
|
|
scheduler_pid = read_pid(SCHEDULER_PID)
|
|
if not scheduler_pid_alive(scheduler_pid):
|
|
scheduler_pid = discover_scheduler_pid()
|
|
process_alive = scheduler_process is not None and scheduler_process.poll() is None
|
|
pid_file_alive = scheduler_pid_alive(scheduler_pid)
|
|
if process_alive or pid_file_alive:
|
|
effective_pid = scheduler_process.pid if process_alive and scheduler_process else scheduler_pid
|
|
scheduler_status = read_json(SCHEDULER_STATUS)
|
|
repair = request_stall_repair(effective_pid, scheduler_status, read_json(MODULE_BATCH), read_json(PROGRESS_PATH))
|
|
state.update(status="repairing_stall" if repair else "watching", schedulerPid=effective_pid, schedulerStatus=scheduler_status.get("status"))
|
|
if repair:
|
|
state.update(lastAutomaticRepair=repair, automaticRepairCount=repair["requestCount"])
|
|
write_status(state)
|
|
time.sleep(max(1.0, args.interval))
|
|
continue
|
|
|
|
if scheduler_process is not None:
|
|
exit_code = scheduler_process.poll()
|
|
state.update(lastSchedulerPid=scheduler_process.pid, lastSchedulerExitCode=exit_code)
|
|
reason = "scheduler_exited"
|
|
else:
|
|
exit_code = None
|
|
reason = "scheduler_missing"
|
|
recovery = read_json(RECOVERY_PATH)
|
|
if recovery.get("status") == "requested":
|
|
reason = "stalled_batch_recovery"
|
|
recovery.update(status="restarting", restartedAt=now())
|
|
temporary = RECOVERY_PATH.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(recovery, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(RECOVERY_PATH)
|
|
state["restartCount"] = int(state.get("restartCount", 0)) + 1
|
|
state.update(
|
|
status="restarting",
|
|
schedulerPid=None,
|
|
lastRestartAt=now(),
|
|
lastRestartReason=reason,
|
|
)
|
|
write_status(state)
|
|
print(
|
|
f"WATCHDOG_RESTART: count={state['restartCount']} reason={reason} "
|
|
f"previous_pid={state.get('lastSchedulerPid')} exit={exit_code}",
|
|
flush=True,
|
|
)
|
|
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
|
output = OUTPUT.open("a", encoding="utf-8")
|
|
try:
|
|
scheduler_process = subprocess.Popen(
|
|
scheduler_args(args),
|
|
cwd=ROOT,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=output,
|
|
stderr=subprocess.STDOUT,
|
|
start_new_session=True,
|
|
text=True,
|
|
env={**os.environ, "FREECAD_CCACHE_RAM": os.environ.get("FREECAD_CCACHE_RAM", "1")},
|
|
)
|
|
finally:
|
|
output.close()
|
|
state.update(schedulerPid=scheduler_process.pid, status="watching")
|
|
write_status(state)
|
|
time.sleep(max(1.0, args.interval))
|
|
|
|
if scheduler_process and scheduler_process.poll() is None:
|
|
scheduler_process.terminate()
|
|
state.update(status="stop_signal", schedulerPid=None, stoppedAt=now())
|
|
write_status(state)
|
|
print("WATCHDOG_STOP_SIGNAL", flush=True)
|
|
return 130
|
|
finally:
|
|
try:
|
|
if WATCHDOG_PID.read_text(encoding="ascii").strip() == str(os.getpid()):
|
|
WATCHDOG_PID.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|