947 lines
58 KiB
Python
947 lines
58 KiB
Python
#!/usr/bin/env python3
|
|
"""Persistent, independently verified Blender Web migration supervisor."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from wbv2_timing import WorkerTrace, command_phase, display_times, format_report, minute_label, timing_report
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CONTROL = "tests/golden/WBV2/autopilot"
|
|
POLICY = "my-tools/wbv2_autopilot_policy.json"
|
|
REQUIRED = {"source_build", "typecheck", "lint", "chromium", "positive", "negative", "undo_redo", "save_reopen", "determinism"}
|
|
ACTIVE = {"implementing", "validating", "finalizing", "committing", "committed", "pushing", "pushed"}
|
|
|
|
|
|
class ProviderUnavailable(ValueError):
|
|
"""No agent action started before a bounded provider failure."""
|
|
|
|
|
|
def now():
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def read_json(path, default=None):
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except FileNotFoundError:
|
|
if default is not None:
|
|
return default
|
|
raise
|
|
|
|
|
|
def write_json(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_name(path.name + f".tmp-{os.getpid()}")
|
|
with temporary.open("w", encoding="utf-8") as stream:
|
|
json.dump(value, stream, ensure_ascii=True, indent=2)
|
|
stream.write("\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
temporary.replace(path)
|
|
|
|
|
|
def sha(path):
|
|
if not path.is_file():
|
|
return None
|
|
with path.open("rb") as stream:
|
|
return hashlib.file_digest(stream, "sha256").hexdigest()
|
|
|
|
|
|
def contained(root, relative):
|
|
if not isinstance(relative, str) or not relative or Path(relative).is_absolute() or ".." in Path(relative).parts:
|
|
raise ValueError(f"invalid relative path: {relative!r}")
|
|
result = (root / relative).resolve()
|
|
if not result.is_relative_to(root.resolve()):
|
|
raise ValueError(f"path escapes repository: {relative}")
|
|
return result
|
|
|
|
|
|
def alive(pid):
|
|
if not isinstance(pid, int) or pid <= 0:
|
|
return False
|
|
try:
|
|
os.kill(pid, 0)
|
|
return True
|
|
except ProcessLookupError:
|
|
return False
|
|
except PermissionError:
|
|
return True
|
|
|
|
|
|
def identity(pid):
|
|
try:
|
|
return Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[19]
|
|
except (OSError, IndexError):
|
|
return None
|
|
|
|
|
|
class Lease:
|
|
def __init__(self, path, inherited=None):
|
|
self.path = path
|
|
self.inherited = inherited
|
|
self.transferred = False
|
|
|
|
def __enter__(self):
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
self.stream = os.fdopen(self.inherited, "a+") if self.inherited is not None else self.path.open("a+")
|
|
if os.fstat(self.stream.fileno()).st_ino != self.path.stat().st_ino:
|
|
self.stream.close()
|
|
raise RuntimeError("inherited lease does not match the owner lock")
|
|
try:
|
|
fcntl.flock(self.stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
self.stream.close()
|
|
raise RuntimeError("another Python autopilot owns the project") from None
|
|
self.stream.seek(0)
|
|
self.stream.truncate()
|
|
json.dump({"pid": os.getpid(), "identity": identity(os.getpid()), "startedAt": now()}, self.stream)
|
|
self.stream.flush()
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
# Never unlink a flock inode while another process can hold it open.
|
|
if not self.transferred:
|
|
fcntl.flock(self.stream, fcntl.LOCK_UN)
|
|
self.stream.close()
|
|
|
|
|
|
class ProjectFence:
|
|
"""Share the existing scheduler's exclusive marker with Python runs."""
|
|
def __init__(self, root):
|
|
self.directory = root / "tests/golden/WBV2"
|
|
self.path = self.directory / "continuous-scheduler.lock"
|
|
|
|
def __enter__(self):
|
|
self.directory.mkdir(parents=True, exist_ok=True)
|
|
for name in ("continuous-scheduler.lock", "continuous-guardian.lock", "continuous-supervisor.lock", "web-product-migration.lock"):
|
|
owner = read_json(self.directory / name, {})
|
|
if alive(owner.get("pid")):
|
|
raise RuntimeError(f"another project runner is active: {name}")
|
|
try:
|
|
with self.path.open("x", encoding="utf-8") as stream:
|
|
json.dump({"pid": os.getpid(), "identity": identity(os.getpid()), "runtime": "python-autopilot", "startedAt": now()}, stream)
|
|
except FileExistsError:
|
|
raise RuntimeError("stale scheduler lock requires reconciliation before Python can claim it") from None
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
owner = read_json(self.path, {})
|
|
if owner.get("pid") == os.getpid() and owner.get("runtime") == "python-autopilot":
|
|
self.path.unlink()
|
|
|
|
|
|
def queue_context(root):
|
|
queue = (root / "docs/EXECUTION_QUEUE.md").read_text(encoding="utf-8")
|
|
task = re.search(r"\| 当前任务 \| `([^`]+)` \|", queue)
|
|
parent = re.search(r"\| parent manifest \| `([^`]+)` \|", queue)
|
|
if not task or not parent or not re.fullmatch(r"WBV2-P\d+-\d+", task[1]):
|
|
raise ValueError("authoritative queue pointer is malformed")
|
|
manifest = read_json(contained(root, parent[1]))
|
|
if manifest.get("status") != "done" or manifest.get("nextTask") != task[1]:
|
|
raise ValueError("parent manifest does not authorize the current task")
|
|
card = f"docs/tasks/{task[1]}.md"
|
|
if not (root / card).is_file():
|
|
raise ValueError("current task card is missing")
|
|
return {"task": task[1], "parentManifest": parent[1], "card": card, "parentSha256": sha(root / parent[1])}
|
|
|
|
|
|
def manifest_proof(root, task):
|
|
manifest = read_json(contained(root, f"tests/golden/{task}/manifest.json"), {})
|
|
reasons = []
|
|
if manifest.get("task") != task or manifest.get("status") != "done":
|
|
reasons.append("task_not_done")
|
|
if manifest.get("verificationDeferred") or manifest.get("acceptanceComplete") is not True:
|
|
reasons.append("runtime_acceptance_not_complete")
|
|
artifacts = manifest.get("artifacts", {})
|
|
if not isinstance(artifacts, dict) or not artifacts:
|
|
reasons.append("artifacts_missing")
|
|
else:
|
|
for name, item in artifacts.items():
|
|
try:
|
|
if not item.get("sha256") or sha(contained(root, item["path"])) != item["sha256"]:
|
|
reasons.append(f"artifact_drift:{name}")
|
|
except (KeyError, ValueError, AttributeError):
|
|
reasons.append(f"artifact_invalid:{name}")
|
|
return {"ok": not reasons, "reasons": reasons, "nextTask": manifest.get("nextTask")}
|
|
|
|
|
|
def inventory_proof(root):
|
|
inventory = read_json(root / "tests/golden/WBV2/dynamic-inventory.json")
|
|
units = inventory.get("units", [])
|
|
unverified = sum(unit.get("coverageStatus") != "VERIFIED" for unit in units)
|
|
invalid, checked = 0, {}
|
|
for unit in units:
|
|
if unit.get("coverageStatus") != "VERIFIED":
|
|
continue
|
|
matches = True
|
|
for key in ("evidenceManifest", "productionEvidence"):
|
|
item = unit.get("coverage", {}).get(key, {})
|
|
try:
|
|
signature = (item["path"], item["sha256"])
|
|
if signature not in checked:
|
|
checked[signature] = bool(item["sha256"]) and sha(contained(root, item["path"])) == item["sha256"]
|
|
matches = matches and checked[signature]
|
|
except (KeyError, ValueError):
|
|
matches = False
|
|
invalid += not matches
|
|
return {"ok": bool(units) and unverified == 0 and invalid == 0, "unitCount": len(units), "recordedVerified": len(units) - unverified, "unverifiedUnits": unverified, "unitsWithInvalidReferences": invalid}
|
|
|
|
|
|
def result_schema():
|
|
strings = {"type": "array", "items": {"type": "string"}}
|
|
def obj(properties):
|
|
return {"type": "object", "additionalProperties": False, "required": list(properties), "properties": properties}
|
|
check = obj({"id": {"type": "string"}, "kinds": strings, "argv": strings, "cwd": {"type": "string"}})
|
|
case = obj({"id": {"type": "string"}, "kind": {"type": "string"}, "checkId": {"type": "string"}, "receipt": {"type": "string"}, "unitIds": strings})
|
|
return obj({"status": {"type": "string", "enum": ["ready", "blocked"]}, "summary": {"type": "string"}, "workflow": {"type": "string"}, "routeIds": strings, "sourceReferences": strings, "changedFiles": strings, "sourceUnits": strings, "dependencies": strings, "artifacts": strings, "checks": {"type": "array", "items": check}, "cases": {"type": "array", "items": case}})
|
|
|
|
|
|
def validate_command(root, argv, cwd):
|
|
if not argv or not all(isinstance(v, str) and v and "\0" not in v for v in argv):
|
|
raise ValueError("command must be an argv array")
|
|
directory = root if cwd == "." else contained(root, cwd)
|
|
if not directory.is_dir():
|
|
raise ValueError("command cwd does not exist")
|
|
if argv[0] not in {"node", "python3", "npm", "cmake", "ninja", "ctest", "git"}:
|
|
raise ValueError("unsupported validation executable")
|
|
if any(arg in {"-e", "--eval", "-c", "--command", "--input-type=module"} for arg in argv[1:]):
|
|
raise ValueError("inline code is not a validation entrypoint")
|
|
if any(re.search(r"firefox|webkit", arg, re.I) for arg in argv):
|
|
raise ValueError("only Chromium is permitted")
|
|
if argv[0] == "git" and argv != ["git", "diff", "--check"]:
|
|
raise ValueError("validation cannot change Git state")
|
|
if argv[0] in {"node", "python3"}:
|
|
scripts = [arg for arg in argv[1:] if arg.endswith((".mjs", ".js", ".py"))]
|
|
if not scripts or any(not (directory / p).resolve().is_relative_to(root) or not (directory / p).is_file() for p in scripts):
|
|
raise ValueError("a concrete repository validation script is required")
|
|
if argv[0] == "npm" and "run" not in argv:
|
|
raise ValueError("only repository npm run scripts are permitted")
|
|
return argv, directory
|
|
|
|
|
|
class Autopilot:
|
|
def __init__(self, root=ROOT, policy=None):
|
|
self.root = root.resolve()
|
|
self.directory = self.root / CONTROL
|
|
self.policy = policy or read_json(self.root / POLICY)
|
|
if self.policy.get("sourceFirst") is not True:
|
|
raise ValueError("source-first business workflow policy is mandatory")
|
|
if not REQUIRED.issubset(self.policy.get("requiredChecks", [])):
|
|
raise ValueError("policy cannot omit mandatory behavioral gates")
|
|
for key in ("workerTimeoutSeconds", "checkTimeoutSeconds", "heartbeatSeconds", "maxAttempts", "maxTokensPerRun", "maxLogBytes", "maxBuildJobs", "monitorPollSeconds", "modelStartTimeoutSeconds", "modelIdleTimeoutSeconds", "providerBackoffSeconds", "providerMaxBackoffSeconds", "maxProviderFailures"):
|
|
if not isinstance(self.policy.get(key), (float, int)) or self.policy[key] <= 0:
|
|
raise ValueError(f"invalid positive policy limit: {key}")
|
|
self.state_path = self.directory / "state.json"
|
|
self.state = read_json(self.state_path, {"schemaVersion": 1, "state": "idle", "jobs": {}, "tokensUsed": 0})
|
|
self.stop_requested = False
|
|
self.child = None
|
|
self.run_tokens = 0
|
|
|
|
def checkpoint(self, state, **values):
|
|
self.state.update(state=state, updatedAt=now(), pid=os.getpid(), identity=identity(os.getpid()), **values)
|
|
write_json(self.state_path, self.state)
|
|
write_json(self.root / "tests/golden/WBV2/web-product-migration-status.json", {"schemaVersion": 2, "pid": os.getpid(), "status": state, "updatedAt": now(), "activeTask": self.state.get("activeJob"), "checkpoint": str(self.state_path.relative_to(self.root)), "verifiedJobs": sum(j.get("state") == "verified" for j in self.state["jobs"].values()), "reason": self.state.get("reason")})
|
|
|
|
def event(self, kind, **values):
|
|
self.directory.mkdir(parents=True, exist_ok=True)
|
|
record = {"time": now(), "event": kind, **values}
|
|
line = json.dumps(record, ensure_ascii=True)
|
|
with (self.directory / "events.jsonl").open("a", encoding="utf-8") as stream:
|
|
stream.write(line + "\n")
|
|
print(json.dumps(display_times({**record, "time": minute_label(record["time"])}), ensure_ascii=True), flush=True)
|
|
|
|
@contextmanager
|
|
def timed(self, phase):
|
|
started, wall = time.monotonic(), now()
|
|
try:
|
|
yield
|
|
finally:
|
|
self.event("timing.span", phase=phase, startedAt=wall, finishedAt=now(), durationSeconds=round(time.monotonic() - started, 6))
|
|
|
|
def request_stop(self, *_):
|
|
self.stop_requested = True
|
|
if self.child and self.child.poll() is None:
|
|
try:
|
|
os.killpg(self.child.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
pass
|
|
|
|
def run_process(self, argv, directory, log, timeout, stdin=None, env=None, monitor_worker=False, phase=None):
|
|
log.parent.mkdir(parents=True, exist_ok=True)
|
|
started = time.monotonic()
|
|
phase = phase or command_phase(argv)
|
|
result = {"argv": argv, "cwd": str(directory.relative_to(self.root)), "phase": phase, "startedAt": now(), "log": str(log.relative_to(self.root))}
|
|
trace = WorkerTrace() if monitor_worker else None
|
|
provider_abort = None
|
|
next_heartbeat = started + self.policy["heartbeatSeconds"]
|
|
last_provider_error = None
|
|
interrupted = timed_out = log_limit = False
|
|
self.event("process.started", **result)
|
|
with log.open("wb") as output:
|
|
self.child = subprocess.Popen(argv, cwd=directory, env={**os.environ, **(env or {})}, stdin=subprocess.PIPE if stdin is not None else subprocess.DEVNULL, stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
|
|
self.checkpoint(self.state["state"], childPid=self.child.pid, childIdentity=identity(self.child.pid), currentProcess={**result, "status": "running", "elapsedSeconds": 0, "phase": "provider_wait" if trace else phase})
|
|
if stdin is not None:
|
|
try:
|
|
self.child.stdin.write(stdin.encode())
|
|
self.child.stdin.close()
|
|
except BrokenPipeError:
|
|
pass
|
|
while self.child.poll() is None:
|
|
elapsed = time.monotonic() - started
|
|
if trace:
|
|
trace.read(log, elapsed)
|
|
if trace.provider_error and not trace.actions_seen:
|
|
provider_abort = "provider_unavailable"
|
|
elif not trace.actions_seen and elapsed >= self.policy.get("modelStartTimeoutSeconds", 60):
|
|
provider_abort = "model_start_timeout"
|
|
elif trace.actions_seen and not trace.active_items and elapsed - trace.last_progress >= self.policy.get("modelIdleTimeoutSeconds", 180):
|
|
provider_abort = "model_idle_timeout"
|
|
interrupted = self.stop_requested or (self.directory / "pause.request").exists()
|
|
timed_out = elapsed > timeout
|
|
log_limit = log.stat().st_size > self.policy["maxLogBytes"]
|
|
if interrupted or timed_out or log_limit or provider_abort:
|
|
try:
|
|
os.killpg(self.child.pid, signal.SIGTERM)
|
|
self.child.wait(timeout=5)
|
|
except ProcessLookupError:
|
|
pass
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(self.child.pid, signal.SIGKILL)
|
|
self.child.wait()
|
|
break
|
|
changed_error = trace and trace.provider_error != last_provider_error
|
|
if time.monotonic() >= next_heartbeat or changed_error:
|
|
current = {**result, "status": "running", "elapsedSeconds": round(elapsed, 3)}
|
|
if trace:
|
|
current.update(phase="provider_wait" if not trace.actions_seen else "agent_work", providerError=trace.provider_error, firstActionSeconds=trace.first_action, idleSeconds=round(elapsed - trace.last_progress, 3), activeItems=list(trace.active_items.values()))
|
|
last_provider_error = trace.provider_error
|
|
self.checkpoint(self.state["state"], heartbeatAt=now(), currentProcess=current)
|
|
next_heartbeat = time.monotonic() + self.policy["heartbeatSeconds"]
|
|
wait_for = max(0.001, min(self.policy.get("monitorPollSeconds", 0.25), next_heartbeat - time.monotonic(), timeout - elapsed))
|
|
try:
|
|
self.child.wait(timeout=wait_for)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
code = self.child.wait()
|
|
interrupted = interrupted or self.stop_requested or (self.directory / "pause.request").exists()
|
|
if trace:
|
|
trace.read(log, time.monotonic() - started)
|
|
try:
|
|
os.killpg(self.child.pid, signal.SIGTERM)
|
|
time.sleep(0.1)
|
|
os.killpg(self.child.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
self.child = None
|
|
elapsed = time.monotonic() - started
|
|
result.update(exitCode=code, durationSeconds=round(elapsed, 6), interrupted=interrupted, timedOut=timed_out, logLimit=log_limit, providerAbort=provider_abort, logSha256=sha(log), finishedAt=now())
|
|
if trace:
|
|
result["workerTrace"] = trace.snapshot(elapsed)
|
|
self.state.pop("childPid", None)
|
|
self.state.pop("childIdentity", None)
|
|
write_json(log.with_suffix(log.suffix + ".process.json"), result)
|
|
self.event("process.finished", **result)
|
|
self.checkpoint(self.state["state"], currentProcess={"phase": phase, "startedAt": result["startedAt"], "finishedAt": result["finishedAt"], "status": "finished", "durationSeconds": result["durationSeconds"], "providerAbort": provider_abort})
|
|
return result
|
|
|
|
def protected_hashes(self, context):
|
|
files = ["docs/EXECUTION_QUEUE.md", "nextTask.md", context["parentManifest"], "tests/golden/WBV2/web-product-backlog.json", "tests/golden/WBV2/dynamic-inventory.json", "tests/golden/WBV2/task-index.json", "tests/golden/WBV2-P0-002/parity-map.json", "my-tools/wbv2_autopilot.py", POLICY, f"tests/golden/{context['task']}/manifest.json"]
|
|
inventory = read_json(self.root / "tests/golden/WBV2/dynamic-inventory.json", {})
|
|
for unit in inventory.get("units", []):
|
|
for key in ("evidenceManifest", "productionEvidence"):
|
|
artifact = unit.get("coverage", {}).get(key)
|
|
if artifact:
|
|
files.append(artifact["path"])
|
|
files.append("my-tools/wbv2_timing.py")
|
|
with self.timed("evidence_hashing"):
|
|
return {p: sha(self.root / p) for p in sorted(set(files))}
|
|
|
|
def jobs(self, context):
|
|
backlog = read_json(self.root / "tests/golden/WBV2/web-product-backlog.json", {})
|
|
jobs = [dict(j) for j in backlog.get("tasks", [])] if backlog.get("queueTask") == context["task"] else [{"id": context["task"], "title": "Implement the current source-backed module", "ownedPaths": [], "implementation": "Follow the current module card and its source/case contract."}]
|
|
priority = self.policy.get("priority", [])
|
|
jobs.sort(key=lambda j: priority.index(j["id"]) if j["id"] in priority else len(priority))
|
|
for job in jobs:
|
|
if not re.fullmatch(r"[A-Z0-9_-]+", job["id"]):
|
|
raise ValueError("unsafe job identifier")
|
|
job["queueTask"] = context["task"]
|
|
return jobs
|
|
|
|
def receipt_valid(self, record):
|
|
if record.get("state") != "verified" or not record.get("receipt"):
|
|
return False
|
|
try:
|
|
receipt = read_json(contained(self.root, record["receipt"]))
|
|
if sha(self.root / record["receipt"]) != record.get("receiptSha256"):
|
|
return False
|
|
return all(sha(contained(self.root, p)) == digest for p, digest in receipt["inputs"].items()) and all(sha(contained(self.root, a["path"])) == a["sha256"] for a in receipt["artifacts"])
|
|
except (OSError, KeyError, ValueError):
|
|
return False
|
|
|
|
def choose_job(self, jobs):
|
|
# Complete untouched workflows before revisiting shared-file changes.
|
|
for job in jobs:
|
|
if self.state["jobs"].get(job["id"], {}).get("state") != "verified":
|
|
return job
|
|
return next((job for job in jobs if not self.receipt_valid(self.state["jobs"][job["id"]])), None)
|
|
|
|
def worker_prompt(self, context, job, run_dir, previous):
|
|
receipt_dir = str((run_dir / "cases").relative_to(self.root))
|
|
return f"""Implement one bounded Blender Web workflow in {self.root}.
|
|
Queue: {json.dumps(context, ensure_ascii=True)}
|
|
Workflow inside this task: {json.dumps(job, ensure_ascii=True)}
|
|
Previous attempt: {json.dumps(previous, ensure_ascii=True)[:4000]}
|
|
Known findings: {json.dumps(self.policy.get('knownFindings', []), ensure_ascii=True)}
|
|
|
|
Follow AGENTS.md: read the queue, print-task-context, check-task-context, then only named owned paths and bounded source searches.
|
|
Drive this task by its business module, user workflow and concrete function routes, then the exact source/operator case matrix.
|
|
FIRST search the supplied Blender 5.2 sources and task-scoped related references to resolve behavior before editing the Web implementation.
|
|
Use node tools/web/search-wbv2-batch-source.mjs --unit <source-unit-id> --pattern <symbol> to read bounded native source locations.
|
|
Blender 5.2 source/Main is authoritative. Never simulate Blender in JS or count route classification, timing, builds or bookkeeping as completed behavior.
|
|
Preserve dirty files. Reconcile previous edits before changing code. Never replay an unknown generator or external write.
|
|
Implement production code and executable focused tests, adding only directly related helpers/tests.
|
|
Do not modify queue, parent manifests, nextTask.md, backlog, historical evidence, source inventory or autopilot control files.
|
|
Do not commit, push, SSH, deploy, install services, start agents or change the live website.
|
|
Do not run expensive validation yourself: the independent supervisor executes your checks after your process exits successfully.
|
|
Return ready only when implementation and runnable tests are prepared; ready is NOT completion.
|
|
Checks use argv arrays and repo-relative cwd, with no shell or inline code and only Chromium.
|
|
Required check kinds: {', '.join(sorted(REQUIRED))}. Reuse existing build/test entrypoints; one command may cover multiple kinds.
|
|
Tests must prove positive Main mutation or exact Main reader, negative no-mutation, undo/redo, save/reopen and determinism.
|
|
Declare cases for positive, negative, undo_redo, save_reopen and determinism, bound to checkId and exact source inventory unitIds.
|
|
Tests must write fresh case receipts under {receipt_dir}/ at validation time using WBV2_RECEIPT_DIR. Do NOT write these receipts now.
|
|
Case JSON: status=PASS, runtime=CHROMIUM_WEBENGINE or BLENDER_NATIVE, caseId, kind, unitIds, observations, artifacts=[{{path,sha256}}].
|
|
observations use SHA-256 of actual canonical native Main snapshots: positive before/after differ, or reader expected/actual equal with reader=true; negative before/after equal; undo_redo before/after/undo/redo prove reversal; save_reopen saved/reopened equal; determinism first/second equal. At least one case must run in Chromium.
|
|
Use actual native/Chromium receipts, screenshots, .blend or WASM artifacts. Test assertions must verify behavior, not static markers.
|
|
List source/protocol/test/fixture/build dependencies for hash invalidation and compiled WASM/native paths in artifacts.
|
|
Return workflow (the concrete user workflow), routeIds (exact current route catalog IDs) and sourceReferences (native Blender files actually used).
|
|
sourceUnits must be existing dynamic inventory IDs. Do not claim verified coverage; the supervisor checks and hashes results.
|
|
Return blocked with a concrete reason if prerequisites cannot be established.
|
|
"""
|
|
|
|
def worker(self, prompt, run_dir):
|
|
schema_path, output_path = run_dir / "result.schema.json", run_dir / "worker-result.json"
|
|
write_json(schema_path, result_schema())
|
|
command = ["codex", "exec", "--sandbox", self.policy["sandbox"], "-c", 'approval_policy="never"', "--ephemeral", "--cd", str(self.root), "--json", "--output-schema", str(schema_path), "--output-last-message", str(output_path), "-"]
|
|
proc = self.run_process(command, self.root, run_dir / "worker.jsonl", self.policy["workerTimeoutSeconds"], stdin=prompt, monitor_worker=True, phase="handoff_model" if self.state["state"] == "finalizing" else "model_request")
|
|
completed = False
|
|
for line in (run_dir / "worker.jsonl").read_text(encoding="utf-8", errors="replace").splitlines():
|
|
try:
|
|
event = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if event.get("type") == "turn.completed":
|
|
completed = True
|
|
usage = event.get("usage", {})
|
|
tokens = int(usage.get("input_tokens", 0)) + int(usage.get("output_tokens", 0))
|
|
self.run_tokens += tokens
|
|
self.state["tokensUsed"] = self.state.get("tokensUsed", 0) + tokens
|
|
if proc["interrupted"] or proc["timedOut"] or proc["logLimit"]:
|
|
self.checkpoint("needs_reconcile", reason="worker interrupted; inspect edits and process receipt before resuming")
|
|
raise InterruptedError(self.state["reason"])
|
|
trace = proc.get("workerTrace", {})
|
|
if proc.get("providerAbort") or trace.get("providerError"):
|
|
if trace.get("safeToRetry"):
|
|
raise ProviderUnavailable(trace.get("providerError") or proc["providerAbort"])
|
|
if proc.get("providerAbort"):
|
|
raise InterruptedError("model stalled after actions or unrecognized output; inspect the bounded trace before retry")
|
|
if proc["exitCode"] != 0 or not completed:
|
|
raise ValueError(f"worker failed: exit={proc['exitCode']}, turnCompleted={completed}")
|
|
return read_json(output_path)
|
|
|
|
def validate_result(self, result, run_dir):
|
|
if result.get("status") != "ready":
|
|
raise ValueError(result.get("summary", "worker reported blocked"))
|
|
known = {u["id"] for u in read_json(self.root / "tests/golden/WBV2/dynamic-inventory.json")["units"]}
|
|
units = set(result.get("sourceUnits", []))
|
|
if not units or not units.issubset(known):
|
|
raise ValueError("missing or unknown source unit bindings")
|
|
catalog = read_json(self.root / "web/app/src/capabilities/blender-feature-route-catalog.json")
|
|
route_ids = set(result.get("routeIds", []))
|
|
if not result.get("workflow", "").strip() or not route_ids or not route_ids.issubset({entry["id"] for entry in catalog["entries"]}) or units != route_ids:
|
|
raise ValueError("a concrete business workflow and source-bound function routes are required")
|
|
references = result.get("sourceReferences", [])
|
|
if not references or any(not p.startswith("blender-5.2.0/source/blender/") or not contained(self.root, p).is_file() for p in references):
|
|
raise ValueError("native Blender source references are required before Web implementation")
|
|
kinds, ids = set(), {}
|
|
for check in result.get("checks", []):
|
|
if not re.fullmatch(r"[A-Za-z0-9_-]+", check["id"]) or check["id"] in ids:
|
|
raise ValueError("check IDs must be unique and path-safe")
|
|
ids[check["id"]] = check
|
|
kinds.update(check["kinds"])
|
|
validate_command(self.root, check["argv"], check["cwd"])
|
|
if not REQUIRED.issubset(kinds):
|
|
raise ValueError(f"missing validation kinds: {sorted(REQUIRED - kinds)}")
|
|
case_kinds, case_ids, paths, covered = set(), set(), set(), set()
|
|
positive_units = set()
|
|
for case in result.get("cases", []):
|
|
destination = contained(self.root, case["receipt"])
|
|
if not destination.is_relative_to(run_dir / "cases") or destination.exists():
|
|
raise ValueError("behavioral receipts must be fresh validation output in this attempt")
|
|
if case["checkId"] not in ids or case["kind"] not in ids[case["checkId"]]["kinds"] or case["id"] in case_ids or str(destination) in paths:
|
|
raise ValueError("invalid or duplicate case/check binding")
|
|
case_ids.add(case["id"])
|
|
paths.add(str(destination))
|
|
case_kinds.add(case["kind"])
|
|
if not case["unitIds"] or not set(case["unitIds"]).issubset(units):
|
|
raise ValueError("invalid source-unit case binding")
|
|
covered.update(case["unitIds"])
|
|
if case["kind"] == "positive":
|
|
positive_units.update(case["unitIds"])
|
|
if not {"positive", "negative", "undo_redo", "save_reopen", "determinism"}.issubset(case_kinds) or covered != units or positive_units != units:
|
|
raise ValueError("incomplete behavioral case matrix")
|
|
for p in result.get("changedFiles", []) + result.get("dependencies", []) + result.get("artifacts", []):
|
|
contained(self.root, p)
|
|
if not any(p.endswith((".wasm", "/blender", "/web_engine")) for p in result.get("artifacts", [])):
|
|
raise ValueError("a compiled Blender/WASM artifact is required")
|
|
|
|
def verify_cases(self, result):
|
|
artifacts = []
|
|
chromium = False
|
|
for case in result["cases"]:
|
|
p = contained(self.root, case["receipt"])
|
|
receipt = read_json(p)
|
|
if receipt.get("status") != "PASS" or receipt.get("runtime") not in {"CHROMIUM_WEBENGINE", "BLENDER_NATIVE"}:
|
|
raise ValueError(f"runtime case failed: {case['id']}")
|
|
if receipt.get("caseId") != case["id"] or receipt.get("kind") != case["kind"] or set(receipt.get("unitIds", [])) != set(case["unitIds"]):
|
|
raise ValueError("runtime receipt does not match the requested case")
|
|
chromium = chromium or receipt["runtime"] == "CHROMIUM_WEBENGINE"
|
|
observations = receipt.get("observations", {})
|
|
def hashes(*keys):
|
|
values = [observations.get(key, "") for key in keys]
|
|
if not all(isinstance(v, str) and re.fullmatch(r"[a-f0-9]{64}", v) for v in values):
|
|
raise ValueError(f"missing native state hashes: {case['id']}")
|
|
return values
|
|
kind = case["kind"]
|
|
if kind == "positive" and observations.get("reader"):
|
|
left, right = hashes("expected", "actual")
|
|
valid = left == right
|
|
elif kind in {"positive", "negative"}:
|
|
left, right = hashes("before", "after")
|
|
valid = (left != right) if kind == "positive" else (left == right)
|
|
elif kind == "undo_redo":
|
|
before, after, undo, redo = hashes("before", "after", "undo", "redo")
|
|
valid = before != after and before == undo and after == redo
|
|
elif kind == "save_reopen":
|
|
saved, reopened = hashes("saved", "reopened")
|
|
valid = saved == reopened
|
|
elif kind == "determinism":
|
|
first, second = hashes("first", "second")
|
|
valid = first == second
|
|
else:
|
|
valid = False
|
|
if not valid:
|
|
raise ValueError(f"native behavior assertion failed: {case['id']}")
|
|
if not receipt.get("artifacts"):
|
|
raise ValueError("runtime case has no production artifacts")
|
|
for artifact in receipt["artifacts"]:
|
|
if not artifact.get("sha256") or sha(contained(self.root, artifact["path"])) != artifact["sha256"]:
|
|
raise ValueError(f"runtime artifact hash drift: {case['id']}")
|
|
artifacts.append(artifact)
|
|
artifacts.append({"path": case["receipt"], "sha256": sha(p)})
|
|
if not chromium:
|
|
raise ValueError("no Chromium runtime case receipt")
|
|
return artifacts
|
|
|
|
def attempt(self, context, job):
|
|
previous = self.state["jobs"].get(job["id"], {})
|
|
count = int(previous.get("attempts", 0)) + 1
|
|
run_dir = self.directory / "runs" / f"{job['id']}-{uuid.uuid4().hex[:12]}"
|
|
run_dir.mkdir(parents=True)
|
|
record = {"state": "implementing", "attempts": count, "providerFailures": previous.get("providerFailures", 0), "runDirectory": str(run_dir.relative_to(self.root))}
|
|
self.state["jobs"][job["id"]] = record
|
|
protected = self.protected_hashes(context)
|
|
with self.timed("source_fingerprint"):
|
|
owned_before = {p: sha(contained(self.root, p)) for p in job.get("ownedPaths", [])}
|
|
write_json(run_dir / "before-worker.json", {"protected": protected, "owned": owned_before})
|
|
self.checkpoint("implementing", activeJob=job["id"], queue=context, reason=None)
|
|
try:
|
|
result = self.worker(self.worker_prompt(context, job, run_dir, previous), run_dir)
|
|
if self.protected_hashes(context) != protected:
|
|
raise InterruptedError("worker changed protected queue/evidence files")
|
|
self.validate_result(result, run_dir)
|
|
inputs = sorted(set(job.get("ownedPaths", []) + result["changedFiles"] + result["dependencies"] + result["sourceReferences"] + [context["card"], context["parentManifest"], POLICY]))
|
|
before = {p: sha(contained(self.root, p)) for p in inputs}
|
|
if any(value is None for value in before.values()):
|
|
raise ValueError("declared dependency does not exist")
|
|
record["state"] = "validating"
|
|
self.checkpoint("validating")
|
|
checks, cache = [], {}
|
|
env = {"WBV2_RECEIPT_DIR": str(run_dir / "cases"), "WBV2_DEPLOYED_ORIGIN": self.policy["origin"], "WEB_BUILD_JOBS": str(min(self.policy["maxBuildJobs"], 8)), "CMAKE_BUILD_PARALLEL_LEVEL": str(min(self.policy["maxBuildJobs"], 8))}
|
|
for check in result["checks"]:
|
|
argv, cwd = validate_command(self.root, check["argv"], check["cwd"])
|
|
key = json.dumps([argv, str(cwd)])
|
|
if key not in cache:
|
|
cache[key] = self.run_process(argv, cwd, run_dir / f"check-{check['id']}.log", self.policy["checkTimeoutSeconds"], env=env, phase="validation:" + "+".join(sorted(check["kinds"])))
|
|
else:
|
|
self.event("check.reused", checkId=check["id"], originalLog=cache[key]["log"], avoidedSeconds=cache[key]["durationSeconds"])
|
|
receipt = {**cache[key], "id": check["id"], "kinds": check["kinds"]}
|
|
checks.append(receipt)
|
|
if receipt["exitCode"] != 0 or receipt["interrupted"] or receipt["timedOut"] or receipt["logLimit"]:
|
|
raise ValueError(f"check {check['id']} failed: exit={receipt['exitCode']}, timeout={receipt['timedOut']}")
|
|
if self.protected_hashes(context) != protected or any(sha(self.root / p) != digest for p, digest in before.items()):
|
|
raise InterruptedError("source/protected evidence changed during validation")
|
|
artifacts = self.verify_cases(result)
|
|
for p in result["artifacts"]:
|
|
digest = sha(contained(self.root, p))
|
|
if digest is None:
|
|
raise ValueError(f"compiled artifact missing: {p}")
|
|
artifacts.append({"path": p, "sha256": digest})
|
|
for check in checks:
|
|
artifacts.append({"path": check["log"], "sha256": check["logSha256"]})
|
|
receipt_path = run_dir / "verified-receipt.json"
|
|
write_json(receipt_path, {"schemaVersion": 1, "queueTask": context["task"], "job": job["id"], "workflow": result["workflow"], "routeIds": result["routeIds"], "sourceReferences": result["sourceReferences"], "status": "verified", "verifiedAt": now(), "sourceUnits": result["sourceUnits"], "inputs": before, "checks": checks, "artifacts": artifacts, "coverageIncrementClaimed": 0, "summary": result["summary"]})
|
|
record.update(state="verified", receipt=str(receipt_path.relative_to(self.root)), receiptSha256=sha(receipt_path), changedFiles=result["changedFiles"], attempts=0, providerFailures=0)
|
|
self.checkpoint("ready", activeJob=None)
|
|
self.event("job.verified", job=job["id"], receipt=record["receipt"])
|
|
return True
|
|
except ProviderUnavailable as error:
|
|
if self.protected_hashes(context) != protected or any(sha(self.root / p) != value for p, value in owned_before.items()):
|
|
record.update(state="needs_reconcile", reason="provider failure coincided with source/evidence changes")
|
|
self.checkpoint("needs_reconcile", reason=record["reason"])
|
|
return False
|
|
failures = int(previous.get("providerFailures", 0)) + 1
|
|
delay = min(self.policy["providerMaxBackoffSeconds"], self.policy["providerBackoffSeconds"] * 2 ** min(failures - 1, 20))
|
|
retry_at = datetime.fromtimestamp(time.time() + delay, timezone.utc).isoformat()
|
|
state = "provider_blocked" if failures >= self.policy["maxProviderFailures"] else "waiting_provider"
|
|
record.update(state=state, attempts=int(previous.get("attempts", 0)), providerFailures=failures, reason=str(error), nextRetryAt=retry_at)
|
|
self.checkpoint(state, reason=str(error), nextRetryAt=retry_at, providerFailures=failures)
|
|
self.event("provider.unavailable", job=job["id"], failures=failures, nextRetryAt=retry_at, reason=str(error), noAgentActions=True)
|
|
return False
|
|
except InterruptedError as error:
|
|
record.update(state="needs_reconcile", reason=str(error))
|
|
self.checkpoint("needs_reconcile", reason=str(error))
|
|
return False
|
|
except (ValueError, OSError, KeyError, TypeError) as error:
|
|
record.update(state="blocked" if count >= self.policy["maxAttempts"] else "retry_pending", reason=str(error))
|
|
self.checkpoint("paused" if self.stop_requested else record["state"], reason=str(error))
|
|
self.event("job.failed", job=job["id"], attempts=count, reason=str(error))
|
|
return False
|
|
|
|
def git(self, args):
|
|
log = self.directory / "git" / f"{uuid.uuid4().hex}.log"
|
|
result = self.run_process(["git", *args], self.root, log, 120)
|
|
if result["interrupted"] or result["timedOut"] or result["logLimit"]:
|
|
raise InterruptedError("Git command interrupted; reconcile its outcome before continuing")
|
|
output = log.read_text(encoding="utf-8", errors="surrogateescape")
|
|
if result["exitCode"] != 0:
|
|
raise subprocess.CalledProcessError(result["exitCode"], ["git", *args], output=output)
|
|
return output
|
|
|
|
def remember_baseline(self, refresh=False):
|
|
location = self.directory / "baseline.json"
|
|
if location.exists() and not refresh:
|
|
return
|
|
dirty = set(self.git(["diff", "--name-only", "-z", "HEAD"]).split("\0"))
|
|
dirty.update(self.git(["ls-files", "--others", "--exclude-standard", "-z"]).split("\0"))
|
|
write_json(location, {"head": self.git(["rev-parse", "HEAD"]).strip(), "dirtyPaths": sorted(dirty - {""}), "recordedAt": now()})
|
|
|
|
def publish(self, context, changed_files):
|
|
if not self.policy.get("autoCommit"):
|
|
self.checkpoint("ready_to_publish", reason="automatic commit disabled")
|
|
return False
|
|
baseline = read_json(self.directory / "baseline.json")
|
|
paths = sorted(set(changed_files))
|
|
if not paths:
|
|
raise ValueError("handoff has no declared files to commit")
|
|
for p in paths:
|
|
contained(self.root, p)
|
|
if p == ".git" or p.startswith(".git/"):
|
|
raise ValueError("cannot publish Git internals")
|
|
dirty_overlap = set(paths) & set(baseline["dirtyPaths"])
|
|
if dirty_overlap:
|
|
raise ValueError(f"publication would include pre-existing edits: {sorted(dirty_overlap)[:12]}")
|
|
if self.git(["branch", "--show-current"]).strip() != "main":
|
|
raise ValueError("automatic publication requires branch main")
|
|
if self.git(["rev-parse", "HEAD"]).strip() != baseline["head"]:
|
|
raise ValueError("HEAD changed since the supervisor baseline")
|
|
self.checkpoint("committing", publishPaths=paths)
|
|
self.git(["add", "-f", "--", *paths])
|
|
# --only excludes unrelated staged changes from this commit.
|
|
self.git(["commit", "--only", "-m", f"feat(web): verified workflow {context['task']}", "--", *paths])
|
|
head = self.git(["rev-parse", "HEAD"]).strip()
|
|
self.checkpoint("committed", commit=head)
|
|
if self.policy.get("autoPush"):
|
|
self.checkpoint("pushing")
|
|
pushed = False
|
|
for attempt in range(self.policy.get("pushRetries", 6) + 1):
|
|
result = self.run_process(["git", "push", "origin", "HEAD:refs/heads/main"], self.root, self.directory / f"push-{attempt}.log", 120)
|
|
try:
|
|
remote = self.git(["ls-remote", "--refs", "origin", "refs/heads/main"]).split()[0]
|
|
except (subprocess.CalledProcessError, IndexError):
|
|
remote = None
|
|
if remote == head:
|
|
pushed = True
|
|
self.checkpoint("pushed", remoteHead=remote)
|
|
break
|
|
if result["interrupted"] or self.stop_requested:
|
|
break
|
|
deadline = time.monotonic() + min(30, 2 ** attempt)
|
|
while time.monotonic() < deadline and not self.stop_requested:
|
|
time.sleep(0.2)
|
|
if not pushed:
|
|
raise InterruptedError("push outcome not confirmed; reconcile remote HEAD before continuing")
|
|
baseline["head"] = head
|
|
write_json(self.directory / "baseline.json", baseline)
|
|
return True
|
|
|
|
def finalize(self, context):
|
|
run_dir = self.directory / "runs" / f"handoff-{uuid.uuid4().hex[:12]}"
|
|
run_dir.mkdir(parents=True)
|
|
records = self.state["jobs"]
|
|
if not records or not all(self.receipt_valid(r) for r in records.values()):
|
|
raise ValueError("cannot finalize stale workflow receipts")
|
|
references = [{"path": r["receipt"], "sha256": r["receiptSha256"]} for r in records.values()]
|
|
self.checkpoint("finalizing")
|
|
prompt = f"""Finalize only the already verified Blender Web task {context['task']}.
|
|
Read the minimal queue/context/card first. Current pointer: {json.dumps(context)}.
|
|
Independent runtime receipts: {json.dumps(references)}.
|
|
Use repository generators to write aggregate report, status, manifest, task-context and publish the successor pointer.
|
|
Bind these verified receipts and their source/unit/build/test hashes. Set acceptanceComplete=true only if the module contract is fully met.
|
|
No source changes, tests, historical inventory/provenance rewrites, blind hash refresh, commit, push, SSH or production deployment.
|
|
Do not rewrite the parent manifest. Preserve dirty files. Never use deferred verification or route counts as completion.
|
|
If source inventory remains unverified and the current module is complete, use the repository generator to select the next source-backed module; do not invent task IDs or declare full parity.
|
|
Return structured status=ready and exact changedFiles; other array fields may be empty for this handoff-only turn.
|
|
If a required generator/evidence is unavailable, return blocked with the exact cause. Do not advance the queue on failure.
|
|
"""
|
|
result = self.worker(prompt, run_dir)
|
|
if result.get("status") != "ready":
|
|
raise ValueError(result.get("summary", "handoff worker blocked"))
|
|
if sha(self.root / context["parentManifest"]) != context["parentSha256"]:
|
|
raise InterruptedError("handoff changed the parent manifest")
|
|
for record in records.values():
|
|
receipt = read_json(self.root / record["receipt"])
|
|
for p, digest in receipt["inputs"].items():
|
|
if p == context["card"]:
|
|
continue
|
|
if sha(self.root / p) != digest:
|
|
raise InterruptedError("handoff changed a verified implementation dependency")
|
|
for index, argv in enumerate(self.policy["handoffChecks"]):
|
|
process = self.run_process(argv, self.root, run_dir / f"gate-{index}.log", self.policy["checkTimeoutSeconds"])
|
|
if process["exitCode"] != 0 or process["interrupted"] or process["timedOut"] or process["logLimit"]:
|
|
raise ValueError(f"handoff gate failed: {argv}")
|
|
proof = manifest_proof(self.root, context["task"])
|
|
if not proof["ok"]:
|
|
raise ValueError(f"aggregate acceptance missing: {proof['reasons']}")
|
|
after = queue_context(self.root)
|
|
if proof["nextTask"] is not None and after["task"] != proof["nextTask"]:
|
|
raise ValueError("generator did not publish the authorized successor")
|
|
if proof["nextTask"] is None and after["task"] != context["task"]:
|
|
raise ValueError("terminal handoff changed the queue unexpectedly")
|
|
changed = list(result["changedFiles"])
|
|
for record in records.values():
|
|
changed.extend(record.get("changedFiles", []))
|
|
receipt = read_json(self.root / record["receipt"])
|
|
changed.append(record["receipt"])
|
|
changed.extend(a["path"] for a in receipt["artifacts"])
|
|
if not self.publish(context, changed):
|
|
return False
|
|
self.checkpoint("handed_off", before=context, after=after)
|
|
return True
|
|
|
|
def doctor(self):
|
|
context = queue_context(self.root)
|
|
jobs = self.jobs(context)
|
|
return {"schemaVersion": 1, "queue": context, "taskCompletion": manifest_proof(self.root, context["task"]), "inventory": inventory_proof(self.root), "jobCount": len(jobs), "nextJob": (self.choose_job(jobs) or {}).get("id"), "tools": {n: bool(shutil.which(n)) for n in ("python3", "node", "git", "codex", "cmake")}, "policy": POLICY, "deployment": "implementation workers do not alter the live server", "state": self.state.get("state")}
|
|
|
|
def wait_for_provider(self, record):
|
|
retry_at = record.get("nextRetryAt")
|
|
if not retry_at:
|
|
return
|
|
remaining = max(0, datetime.fromisoformat(retry_at).timestamp() - time.time())
|
|
if remaining == 0:
|
|
return
|
|
self.checkpoint("waiting_provider", nextRetryAt=retry_at, reason=record.get("reason"))
|
|
deadline, heartbeat = time.monotonic() + remaining, time.monotonic()
|
|
with self.timed("provider_backoff"):
|
|
while time.monotonic() < deadline and not self.stop_requested and not (self.directory / "pause.request").exists():
|
|
if time.monotonic() >= heartbeat:
|
|
self.checkpoint("waiting_provider", heartbeatAt=now())
|
|
heartbeat = time.monotonic() + self.policy["heartbeatSeconds"]
|
|
time.sleep(min(0.25, max(0, deadline - time.monotonic())))
|
|
|
|
def run(self, max_cycles=0):
|
|
inherited = os.environ.pop("WBV2_AUTOPILOT_LEASE_FD", None)
|
|
with Lease(self.directory / "owner.lock", int(inherited) if inherited else None), ProjectFence(self.root):
|
|
if self.state.get("state") in ACTIVE | {"needs_reconcile"}:
|
|
self.checkpoint("needs_reconcile", reason="previous writer has no completed handoff; run read-only reconcile")
|
|
return 3
|
|
if self.state.get("state") in {"blocked", "provider_blocked"}:
|
|
return 2
|
|
if (self.directory / "pause.request").exists():
|
|
self.checkpoint("paused")
|
|
return 0
|
|
signal.signal(signal.SIGINT, self.request_stop)
|
|
signal.signal(signal.SIGTERM, self.request_stop)
|
|
self.remember_baseline()
|
|
cycles = 0
|
|
last_success = True
|
|
self.checkpoint("preflight", reason=None)
|
|
try:
|
|
while not self.stop_requested and (max_cycles == 0 or cycles < max_cycles):
|
|
if self.run_tokens >= self.policy["maxTokensPerRun"]:
|
|
self.checkpoint("budget_exhausted", reason="per-run token budget reached")
|
|
return 4
|
|
if (self.directory / "pause.request").exists():
|
|
self.checkpoint("paused")
|
|
return 0
|
|
context = queue_context(self.root)
|
|
prior = self.state.get("queue")
|
|
if prior and prior["task"] != context["task"]:
|
|
proof = manifest_proof(self.root, prior["task"])
|
|
if not proof["ok"] or proof["nextTask"] != context["task"]:
|
|
raise ValueError("queue advanced without an accepted parent handoff")
|
|
self.state["jobs"] = {}
|
|
self.checkpoint("preflight", queue=context)
|
|
for index, argv in enumerate(self.policy["preflight"]):
|
|
proc = self.run_process(argv, self.root, self.directory / f"preflight-{index}.log", 120)
|
|
if proc["exitCode"] != 0 or proc["interrupted"] or proc["timedOut"] or proc["logLimit"]:
|
|
raise ValueError(f"context preflight failed: {argv}")
|
|
job = self.choose_job(self.jobs(context))
|
|
if job is None:
|
|
if not self.finalize(context):
|
|
return 2
|
|
proof = manifest_proof(self.root, context["task"])
|
|
if proof["nextTask"] is None:
|
|
inventory = inventory_proof(self.root)
|
|
if not inventory["ok"]:
|
|
self.checkpoint("planning_required", reason="no successor but source inventory remains unverified; parity not complete")
|
|
return 2
|
|
self.checkpoint("complete", reason="accepted terminal manifest and no unverified source units")
|
|
return 0
|
|
continue
|
|
record = self.state["jobs"].get(job["id"], {})
|
|
if record.get("state") in {"blocked", "provider_blocked"}:
|
|
self.checkpoint(record["state"], activeJob=job["id"], reason=record.get("reason"))
|
|
return 2
|
|
if record.get("state") == "waiting_provider":
|
|
self.wait_for_provider(record)
|
|
if self.stop_requested or (self.directory / "pause.request").exists():
|
|
self.checkpoint("paused")
|
|
return 2
|
|
success = self.attempt(context, job)
|
|
last_success = success
|
|
cycles += 1
|
|
if self.state["state"] in {"needs_reconcile", "blocked", "provider_blocked", "paused"}:
|
|
return 3 if self.state["state"] == "needs_reconcile" else 2
|
|
if not success and self.state["state"] != "waiting_provider":
|
|
with self.timed("retry_backoff"):
|
|
deadline = time.monotonic() + self.policy.get("retryDelaySeconds", 10)
|
|
while time.monotonic() < deadline and not self.stop_requested:
|
|
time.sleep(0.2)
|
|
final_state = "paused" if self.stop_requested else "idle" if last_success else self.state["state"]
|
|
self.checkpoint(final_state, reason="stop requested" if self.stop_requested else self.state.get("reason") or "cycle limit reached")
|
|
return 0 if last_success and not self.stop_requested else 2
|
|
except InterruptedError as error:
|
|
self.checkpoint("needs_reconcile", reason=str(error))
|
|
return 3
|
|
except (ValueError, OSError, KeyError, RuntimeError, subprocess.CalledProcessError) as error:
|
|
self.checkpoint("blocked", reason=str(error))
|
|
self.event("supervisor.blocked", reason=str(error))
|
|
return 2
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("action", choices=["run", "start", "status", "timings", "doctor", "pause", "stop", "resume", "reconcile", "service"], nargs="?", default="run")
|
|
parser.add_argument("--once", action="store_true")
|
|
parser.add_argument("--max-cycles", type=int, default=0)
|
|
parser.add_argument("--retry-blocked", action="store_true")
|
|
parser.add_argument("--refresh-baseline", action="store_true", help="adopt a reviewed maintenance commit before any workflow edits/receipts")
|
|
parser.add_argument("--json", action="store_true", help="JSON timing report")
|
|
parser.add_argument("--write", action="store_true", help="write timing report snapshots")
|
|
args = parser.parse_args(argv)
|
|
if args.max_cycles < 0:
|
|
parser.error("--max-cycles must be non-negative")
|
|
runner = Autopilot()
|
|
directory = runner.directory
|
|
if args.action == "doctor":
|
|
print(json.dumps(runner.doctor(), ensure_ascii=False, indent=2))
|
|
return 0
|
|
if args.action == "status":
|
|
state = dict(runner.state)
|
|
state["alive"] = alive(state.get("pid")) and identity(state["pid"]) == state.get("identity")
|
|
state["timezone"] = "Asia/Shanghai"
|
|
print(json.dumps(display_times(state), ensure_ascii=False, indent=2))
|
|
return 0
|
|
if args.action == "timings":
|
|
report = timing_report(directory, runner.state)
|
|
if args.write:
|
|
write_json(directory / "timing-report.json", report)
|
|
(directory / "timing-report.md").write_text(format_report(report) + "\n", encoding="utf-8")
|
|
print(json.dumps(report, ensure_ascii=False, indent=2) if args.json else format_report(report))
|
|
return 0
|
|
if args.action == "service":
|
|
def quote(value):
|
|
if "\n" in value or "\r" in value:
|
|
raise ValueError("newline in systemd value")
|
|
return '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%") + '"'
|
|
unit = Path.home() / ".config/systemd/user/blender-web-autopilot.service"
|
|
unit.parent.mkdir(parents=True, exist_ok=True)
|
|
executable = Path(__file__).with_name("run_web_product_migration.py")
|
|
unit.write_text("[Unit]\nDescription=Blender Web source migration supervisor\nAfter=network-online.target\n\n[Service]\nType=simple\n" + f"WorkingDirectory={quote(str(ROOT))}\nExecStart={quote(sys.executable)} {quote(str(executable))} run\nEnvironment={quote('PATH=' + os.environ.get('PATH', '/usr/bin:/bin'))}\n" + "Restart=on-failure\nRestartPreventExitStatus=2 3 4\nRestartSec=10\nKillMode=control-group\nTimeoutStopSec=15\n\n[Install]\nWantedBy=default.target\n", encoding="utf-8")
|
|
print(f"Created {unit}; enable with systemctl --user daemon-reload and systemctl --user enable --now blender-web-autopilot")
|
|
return 0
|
|
if args.action in {"pause", "stop"}:
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
write_json(directory / "pause.request", {"requestedAt": now()})
|
|
owner = read_json(directory / "owner.lock", {})
|
|
if alive(owner.get("pid")) and identity(owner["pid"]) == owner.get("identity"):
|
|
os.kill(owner["pid"], signal.SIGTERM)
|
|
deadline = time.monotonic() + 10
|
|
while alive(owner["pid"]) and identity(owner["pid"]) == owner.get("identity") and time.monotonic() < deadline:
|
|
time.sleep(0.1)
|
|
print("Pause requested; interrupted work requires reconciliation before retry.")
|
|
return 0
|
|
if args.action == "reconcile":
|
|
print(json.dumps({"queue": queue_context(ROOT), "state": runner.state, "childAlive": alive(runner.state.get("childPid")), "receipts": {key: runner.receipt_valid(record) for key, record in runner.state["jobs"].items()}, "decision": "read-only; inspect interrupted edits before resume --retry-blocked"}, ensure_ascii=False, indent=2))
|
|
return 0
|
|
if args.action == "resume":
|
|
with Lease(directory / "owner.lock"):
|
|
if alive(runner.state.get("childPid")) and identity(runner.state["childPid"]) == runner.state.get("childIdentity"):
|
|
parser.error("previous child is alive; stop and reconcile first")
|
|
if runner.state["state"] in ACTIVE | {"needs_reconcile", "blocked", "provider_blocked"} and not args.retry_blocked:
|
|
parser.error("run reconcile; --retry-blocked acknowledges the inspected repository state")
|
|
for record in runner.state["jobs"].values():
|
|
if record.get("state") in ACTIVE | {"needs_reconcile", "blocked", "provider_blocked"} or (args.retry_blocked and record.get("state") != "verified"):
|
|
record.update(state="retry_pending", attempts=0, providerFailures=0)
|
|
record.pop("nextRetryAt", None)
|
|
if args.refresh_baseline and any(record.get("receipt") or record.get("changedFiles") for record in runner.state["jobs"].values()):
|
|
parser.error("cannot refresh a baseline containing workflow edits or verified receipts")
|
|
(directory / "pause.request").unlink(missing_ok=True)
|
|
if args.refresh_baseline:
|
|
runner.remember_baseline(refresh=True)
|
|
runner.checkpoint("idle", reason="explicit resume")
|
|
if args.action in {"start", "resume"}:
|
|
with Lease(directory / "owner.lock") as lease:
|
|
if (directory / "pause.request").exists():
|
|
parser.error("paused; use resume")
|
|
if runner.state["state"] in ACTIVE | {"needs_reconcile", "blocked", "provider_blocked"}:
|
|
parser.error("run reconcile before restarting interrupted or blocked work")
|
|
command = [sys.executable, str(Path(__file__).with_name("run_web_product_migration.py")), "run", "--max-cycles", str(1 if args.once else args.max_cycles)]
|
|
fd = lease.stream.fileno()
|
|
with (directory / "daemon.log").open("ab") as log:
|
|
child = subprocess.Popen(command, cwd=ROOT, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, pass_fds=(fd,), env={**os.environ, "WBV2_AUTOPILOT_LEASE_FD": str(fd)})
|
|
lease.transferred = True
|
|
print(json.dumps({"launchedPid": child.pid, "statusCommand": "python3 my-tools/run_web_product_migration.py status", "log": str(directory / "daemon.log")}))
|
|
return 0
|
|
return runner.run(1 if args.once else args.max_cycles)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|