workinf_Blender_Wasm/my-tools/watch_blender_web_scheduler.py

#!/usr/bin/env python3
"""Watch and safely restart the Blender/WBV2 scheduler process.

The watchdog only restarts an unexpectedly missing scheduler. It never edits
the queue, claims tasks, retries blocked checkpoints, or deletes evidence.
"""

from __future__ import annotations

import argparse
import json
import os
import signal
import subprocess
import time
from datetime import datetime, timezone
from pathlib import Path

from wbv2_runtime import ROOT, SCHEDULER_CHECKPOINT, SCHEDULER_LOCK, lock_owner, queue_pointer, scheduler_args, terminal_pointer


WATCHDOG_LOCK = ROOT / "tests/golden/WBV2/my-tools-watchdog.lock"
WATCHDOG_PID = ROOT / "tests/golden/WBV2/my-tools-watchdog.pid"
WATCHDOG_STATUS = ROOT / "tests/golden/WBV2/my-tools-watchdog.json"


def now() -> str:
    return datetime.now(timezone.utc).isoformat()


def write_json(path: Path, value: dict) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_suffix(path.suffix + f".tmp-{os.getpid()}")
    temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    temporary.replace(path)


def process_alive(pid: object) -> bool:
    if not isinstance(pid, int) or pid <= 0:
        return False
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        return True
    return True


def acquire_lock() -> int:
    try:
        handle = os.open(WATCHDOG_LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
    except FileExistsError:
        existing = {}
        try:
            existing = json.loads(WATCHDOG_LOCK.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError):
            pass
        owner = existing.get("pid") if isinstance(existing, dict) else None
        if isinstance(owner, int) and not process_alive(owner):
            WATCHDOG_LOCK.unlink(missing_ok=True)
            return acquire_lock()
        raise SystemExit(f"another WBV2 watchdog appears to be running: {WATCHDOG_LOCK}")
    os.write(handle, f"{{\"pid\":{os.getpid()},\"startedAt\":{json.dumps(now())}}}\n".encode())
    return handle


def checkpoint() -> dict:
    try:
        value = json.loads(SCHEDULER_CHECKPOINT.read_text(encoding="utf-8"))
    except (FileNotFoundError, json.JSONDecodeError):
        return {}
    return value if isinstance(value, dict) else {}


def checkpoint_task(value: dict) -> str | None:
    for key in ("current", "before", "after"):
        candidate = value.get(key)
        if isinstance(candidate, dict) and isinstance(candidate.get("task"), str):
            return candidate["task"]
    return None


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--interval", type=float, default=10.0)
    parser.add_argument("--restart-delay", type=float, default=5.0)
    parser.add_argument("--max-restarts", type=int, default=0, help="0 means unlimited safe restarts")
    parser.add_argument("--max-iterations", type=int, default=0)
    parser.add_argument("--commit", action="store_true", default=os.getenv("WBV2_AUTO_COMMIT", "1") != "0")
    parser.add_argument("--push", action="store_true", default=os.getenv("WBV2_AUTO_PUSH", "1") != "0")
    parser.add_argument("--push-retries", type=int, default=6)
    args = parser.parse_args()
    if args.interval <= 0 or args.restart_delay < 0 or args.max_restarts < 0 or args.max_iterations < 0:
        parser.error("interval/delay/limits must be non-negative; interval must be positive")
    if args.push and not args.commit:
        parser.error("--push requires --commit")

    WATCHDOG_LOCK.parent.mkdir(parents=True, exist_ok=True)
    handle = acquire_lock()
    os.close(handle)
    WATCHDOG_PID.write_text(f"{os.getpid()}\n", encoding="ascii")
    restarts = 0
    child: subprocess.Popen[bytes] | None = None
    stopped = False

    def stop(signum: int, _frame: object) -> None:
        nonlocal stopped
        stopped = True
        if child and child.poll() is None:
            try:
                os.killpg(child.pid, signum)
            except ProcessLookupError:
                pass

    signal.signal(signal.SIGINT, stop)
    signal.signal(signal.SIGTERM, stop)
    try:
        while not stopped:
            pointer = queue_pointer()
            saved_checkpoint = checkpoint()
            state = saved_checkpoint.get("state")
            applies = checkpoint_task(saved_checkpoint) == pointer.get("task")
            if terminal_pointer(pointer) or (applies and state in {"complete", "blocked", "stopped"}):
                write_json(
                    WATCHDOG_STATUS,
                    {
                        "schemaVersion": 1,
                        "status": "stopped",
                        "reason": state or "terminal",
                        "task": pointer.get("task"),
                        "updatedAt": now(),
                        "restartCount": restarts,
                    },
                )
                break
            if not lock_owner(SCHEDULER_LOCK)["alive"] and child is None and applies and state in {"started", "running", "preflight"}:
                write_json(
                    WATCHDOG_STATUS,
                    {
                        "schemaVersion": 1,
                        "status": "blocked",
                        "reason": f"unsafe_scheduler_checkpoint_{state}",
                        "task": pointer.get("task"),
                        "updatedAt": now(),
                        "restartCount": restarts,
                    },
                )
                return 2
            if not lock_owner(SCHEDULER_LOCK)["alive"] and (child is None or child.poll() is not None):
                if args.max_restarts and restarts >= args.max_restarts:
                    write_json(
                        WATCHDOG_STATUS,
                        {
                            "schemaVersion": 1,
                            "status": "blocked",
                            "reason": "max_safe_restarts_exceeded",
                            "task": pointer.get("task"),
                            "updatedAt": now(),
                            "restartCount": restarts,
                        },
                    )
                    return 2
                child = subprocess.Popen(
                    scheduler_args(commit=args.commit, push=args.push, max_iterations=args.max_iterations),
                    cwd=ROOT,
                    start_new_session=True,
                )
                restarts += 1
                write_json(
                    WATCHDOG_STATUS,
                    {
                        "schemaVersion": 1,
                        "status": "restarted",
                        "reason": "scheduler_missing",
                        "task": pointer.get("task"),
                        "schedulerPid": child.pid,
                        "updatedAt": now(),
                        "restartCount": restarts,
                    },
                )
                time.sleep(args.restart_delay)
            else:
                write_json(
                    WATCHDOG_STATUS,
                    {
                        "schemaVersion": 1,
                        "status": "watching",
                        "task": pointer.get("task"),
                        "schedulerPid": lock_owner(SCHEDULER_LOCK)["pid"],
                        "updatedAt": now(),
                        "restartCount": restarts,
                    },
                )
                time.sleep(args.interval)
    finally:
        if child and child.poll() is None:
            try:
                os.killpg(child.pid, signal.SIGTERM)
            except ProcessLookupError:
                pass
        WATCHDOG_PID.unlink(missing_ok=True)
        WATCHDOG_LOCK.unlink(missing_ok=True)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())