161 lines
9.9 KiB
JavaScript
161 lines
9.9 KiB
JavaScript
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { execFileSync } from "node:child_process";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const abs = (file) => path.join(root, file);
|
|
const controlRoot = "tests/golden/WBV2";
|
|
const schedulerLockPath = `${controlRoot}/continuous-scheduler.lock`;
|
|
const schedulerCheckpointPath = `${controlRoot}/continuous-scheduler.json`;
|
|
const guardianLockPath = `${controlRoot}/continuous-guardian.lock`;
|
|
const supervisorLockPath = `${controlRoot}/continuous-supervisor.lock`;
|
|
const guardianCheckpointPath = `${controlRoot}/continuous-guardian.json`;
|
|
const supervisorCheckpointPath = `${controlRoot}/continuous-supervisor.json`;
|
|
const supervisorLogPath = `${controlRoot}/continuous-supervisor.log`;
|
|
|
|
const argv = process.argv.slice(2);
|
|
const watch = argv.includes("--watch");
|
|
const jsonOnly = argv.includes("--json");
|
|
const intervalIndex = argv.indexOf("--interval-ms");
|
|
const intervalMs = Number(intervalIndex >= 0 ? argv[intervalIndex + 1] : 2000);
|
|
const knownArgs = new Set(["--watch", "--json", "--interval-ms"]);
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
if (!knownArgs.has(argv[index])) throw new Error(`unknown argument: ${argv[index]}`);
|
|
if (argv[index] === "--interval-ms") index += 1;
|
|
}
|
|
if (!Number.isSafeInteger(intervalMs) || intervalMs < 100) throw new Error("--interval-ms must be an integer >= 100");
|
|
|
|
const readJson = (file) => {
|
|
try { return JSON.parse(fs.readFileSync(abs(file), "utf8")); }
|
|
catch { return null; }
|
|
};
|
|
const pidAlive = (pid) => {
|
|
if (!Number.isSafeInteger(pid) || pid <= 0) return false;
|
|
try {
|
|
process.kill(pid, 0);
|
|
return true;
|
|
}
|
|
catch (error) {
|
|
return error?.code === "EPERM";
|
|
}
|
|
};
|
|
const lockInfo = (file) => {
|
|
const lock = readJson(file);
|
|
return lock ? { pid: lock.pid ?? null, startedAt: lock.startedAt ?? null, alive: pidAlive(lock.pid) } : { pid: null, startedAt: null, alive: false };
|
|
};
|
|
const currentPointer = () => {
|
|
const queue = fs.readFileSync(abs("docs/EXECUTION_QUEUE.md"), "utf8");
|
|
return {
|
|
task: queue.match(/\| 当前任务 \| `(WBV2-P\d+-\d+)` \|/u)?.[1] ?? null,
|
|
parentManifest: queue.match(/\| parent manifest \| `([^`]+)` \|/u)?.[1] ?? null,
|
|
};
|
|
};
|
|
const gitHead = () => {
|
|
try { return execFileSync("git", ["rev-parse", "--verify", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); }
|
|
catch { return null; }
|
|
};
|
|
const tail = (file, maxBytes = 256 * 1024) => {
|
|
try {
|
|
const stat = fs.statSync(abs(file));
|
|
const start = Math.max(0, stat.size - maxBytes);
|
|
const handle = fs.openSync(abs(file), "r");
|
|
const buffer = Buffer.alloc(stat.size - start);
|
|
fs.readSync(handle, buffer, 0, buffer.length, start);
|
|
fs.closeSync(handle);
|
|
return buffer.toString("utf8");
|
|
}
|
|
catch { return ""; }
|
|
};
|
|
const latestHandoff = () => {
|
|
const matches = [...tail(supervisorLogPath).matchAll(/handoff iteration=(\d+) before=(WBV2-P\d+-\d+) after=(WBV2-P\d+-\d+)/gu)];
|
|
const match = matches.at(-1);
|
|
return match ? { iteration: Number(match[1]), before: match[2], after: match[3] } : null;
|
|
};
|
|
const pushVerification = (task) => {
|
|
if (!task) return null;
|
|
const file = `tests/golden/${task}/continuous-push-verify.log`;
|
|
if (!fs.existsSync(abs(file))) return null;
|
|
const output = tail(file, 4096).trim();
|
|
const match = output.match(/local_head=([0-9a-f]+) remote_head=([0-9a-f]+)/u);
|
|
return { path: file, localHead: match?.[1] ?? null, remoteHead: match?.[2] ?? null, equal: Boolean(match && match[1] === match[2]) };
|
|
};
|
|
const collect = () => {
|
|
const autopilot = readJson(`${controlRoot}/autopilot/state.json`);
|
|
const autopilotLock = lockInfo(`${controlRoot}/autopilot/owner.lock`);
|
|
const queue = currentPointer();
|
|
const scheduler = readJson(schedulerCheckpointPath);
|
|
const guardian = readJson(guardianCheckpointPath);
|
|
const supervisor = readJson(supervisorCheckpointPath);
|
|
const schedulerLock = lockInfo(schedulerLockPath);
|
|
const guardianLock = lockInfo(guardianLockPath);
|
|
const supervisorLock = lockInfo(supervisorLockPath);
|
|
const pythonOwnsScheduler = autopilotLock.alive && schedulerLock.pid === autopilotLock.pid;
|
|
const handoff = latestHandoff();
|
|
const active = autopilotLock.alive || schedulerLock.alive || (guardianLock.alive && supervisorLock.alive);
|
|
let terminalQueue = false;
|
|
try {
|
|
const manifest = queue.task ? readJson(`tests/golden/${queue.task}/manifest.json`) : null;
|
|
const index = readJson("tests/golden/WBV2/task-index.json");
|
|
terminalQueue = !active && index?.activeTask === null && manifest?.status === "done" && manifest?.nextTask === null && manifest?.acceptanceComplete === true && manifest?.verificationDeferred !== true;
|
|
} catch { /* A malformed pointer remains visible as a normal checkpoint failure. */ }
|
|
const checkpointBlocked = scheduler?.state === "blocked" || guardian?.state === "blocked" || supervisor?.state === "blocked";
|
|
// A terminal task can leave a historical blocked checkpoint behind (for
|
|
// example, after a push retry). The watchdog owns the next planner attempt;
|
|
// do not report that stale checkpoint as a live execution block.
|
|
const autopilotBlocked = ["blocked", "provider_blocked", "needs_reconcile", "planning_required", "budget_exhausted", "ready_to_publish"].includes(autopilot?.state);
|
|
const blocked = autopilotBlocked || (!autopilotLock.alive && checkpointBlocked && !terminalQueue);
|
|
const state = blocked ? "BLOCKED" : active ? "RUNNING" : "IDLE";
|
|
const phase = autopilotLock.alive || autopilotBlocked ? `python-${autopilot?.state ?? "starting"}` : blocked ? "blocked" : scheduler?.state === "running" ? "scheduler-task" : supervisor?.state === "running" ? "task-or-gate" : scheduler?.state ?? supervisor?.state ?? "unknown";
|
|
const observed = new Date().toISOString();
|
|
return {
|
|
schemaVersion: 1,
|
|
observedAt: observed,
|
|
state,
|
|
phase,
|
|
message: state === "RUNNING"
|
|
? `RUNNING: ${queue.task ?? "unknown task"} is owned by ${autopilotLock.alive ? "Python autopilot" : "guardian/supervisor"}; wait for its handoff.`
|
|
: state === "BLOCKED" ? "BLOCKED: inspect the scheduler/guardian/supervisor checkpoint reason before any write or retry." : "IDLE: no live scheduler, guardian or supervisor lock was found.",
|
|
queue,
|
|
autopilot: { pid: autopilot?.pid ?? autopilotLock.pid, alive: autopilotLock.alive, state: autopilot?.state ?? null, phase: autopilot?.currentProcess?.phase ?? null, activeJob: autopilot?.activeJob ?? null, reason: autopilot?.reason ?? null, tokensUsed: autopilot?.tokensUsed ?? 0, providerFailures: autopilot?.providerFailures ?? 0, nextRetryAt: autopilot?.nextRetryAt ?? null },
|
|
scheduler: { pid: schedulerLock.alive ? schedulerLock.pid : scheduler?.pid ?? schedulerLock.pid, alive: schedulerLock.alive, state: pythonOwnsScheduler ? "python-owned" : scheduler?.state ?? null, updatedAt: pythonOwnsScheduler ? autopilot?.updatedAt : scheduler?.updatedAt ?? null, iteration: pythonOwnsScheduler ? null : scheduler?.iteration ?? null, before: pythonOwnsScheduler ? null : scheduler?.before ?? null, after: pythonOwnsScheduler ? null : scheduler?.after ?? null, latestBatch: pythonOwnsScheduler ? null : scheduler?.latestBatch ?? null },
|
|
guardian: { pid: guardian?.pid ?? guardianLock.pid, alive: guardianLock.alive, state: guardian?.state ?? null, updatedAt: guardian?.updatedAt ?? null, heartbeatAt: guardian?.heartbeatAt ?? null, restartCount: guardian?.restartCount ?? null },
|
|
supervisor: { pid: supervisor?.pid ?? supervisorLock.pid, alive: supervisorLock.alive, state: supervisor?.state ?? null, updatedAt: supervisor?.updatedAt ?? null, iteration: supervisor?.iteration ?? null, before: supervisor?.before ?? null, after: supervisor?.after ?? null },
|
|
latestHandoff: handoff,
|
|
latestPushVerification: pushVerification(handoff?.before ?? supervisor?.before?.task),
|
|
localHead: gitHead(),
|
|
checkpoints: { scheduler: schedulerCheckpointPath, guardian: guardianCheckpointPath, supervisor: supervisorCheckpointPath },
|
|
};
|
|
};
|
|
const format = (status) => [
|
|
`状态: ${status.state}${status.state === "RUNNING" ? "（正在执行）" : status.state === "BLOCKED" ? "（已阻塞）" : "（未运行）"}`,
|
|
`阶段: ${status.phase}`,
|
|
`当前任务: ${status.queue.task ?? "unknown"}`,
|
|
`parent manifest: ${status.queue.parentManifest ?? "unknown"}`,
|
|
`Python autopilot: pid=${status.autopilot.pid ?? "none"} alive=${status.autopilot.alive} state=${status.autopilot.state ?? "unknown"} job=${status.autopilot.activeJob ?? "none"} reason=${status.autopilot.reason ?? "none"}`,
|
|
`scheduler: pid=${status.scheduler.pid ?? "none"} alive=${status.scheduler.alive} state=${status.scheduler.state ?? "unknown"} iteration=${status.scheduler.iteration ?? "none"} batch=${status.scheduler.latestBatch?.batchId ?? "none"} completed=${status.scheduler.latestBatch?.completed ?? false}`,
|
|
`guardian: pid=${status.guardian.pid ?? "none"} alive=${status.guardian.alive} state=${status.guardian.state ?? "unknown"} heartbeat=${status.guardian.heartbeatAt ?? "none"}`,
|
|
`supervisor: pid=${status.supervisor.pid ?? "none"} alive=${status.supervisor.alive} state=${status.supervisor.state ?? "unknown"} iteration=${status.supervisor.iteration ?? "none"}`,
|
|
`最近交接: ${status.latestHandoff ? `${status.latestHandoff.before} -> ${status.latestHandoff.after} (#${status.latestHandoff.iteration})` : "none"}`,
|
|
`本地 HEAD: ${status.localHead ?? "unknown"}`,
|
|
`最近 push 校验: ${status.latestPushVerification?.equal ? "PASS" : status.latestPushVerification ? "NOT_EQUAL_OR_INCOMPLETE" : "none"}`,
|
|
`观察时间: ${status.observedAt}`,
|
|
status.message,
|
|
].join("\n");
|
|
const print = () => {
|
|
const status = collect();
|
|
if (jsonOnly) {
|
|
process.stdout.write(`${JSON.stringify(status)}\n`);
|
|
return;
|
|
}
|
|
if (watch) process.stdout.write("\x1b[2J\x1b[H");
|
|
process.stdout.write(`${format(status)}\n`);
|
|
};
|
|
print();
|
|
if (watch) {
|
|
const timer = setInterval(print, intervalMs);
|
|
const stop = () => { clearInterval(timer); process.exitCode = 0; };
|
|
process.once("SIGINT", stop);
|
|
process.once("SIGTERM", stop);
|
|
}
|