190 lines
7.9 KiB
JavaScript
190 lines
7.9 KiB
JavaScript
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { spawn } from "node:child_process";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const abs = (file) => path.join(root, file);
|
|
const controlRoot = "tests/golden/WBV2";
|
|
const logPath = `${controlRoot}/continuous-guardian.log`;
|
|
const checkpointPath = `${controlRoot}/continuous-guardian.json`;
|
|
const lockPath = `${controlRoot}/continuous-guardian.lock`;
|
|
|
|
const argv = process.argv.slice(2);
|
|
const numericArg = (name, fallback) => {
|
|
const index = argv.indexOf(name);
|
|
return Number(index >= 0 ? argv[index + 1] : fallback);
|
|
};
|
|
const restartDelayMs = numericArg("--restart-delay-ms", 5000);
|
|
const maxRestarts = numericArg("--max-restarts", 0);
|
|
const maxIterations = numericArg("--max-iterations", 0);
|
|
if (!Number.isSafeInteger(restartDelayMs) || restartDelayMs < 0) throw new Error("--restart-delay-ms must be a non-negative integer");
|
|
if (!Number.isSafeInteger(maxRestarts) || maxRestarts < 0) throw new Error("--max-restarts must be a non-negative integer; 0 means unlimited safe restarts");
|
|
if (!Number.isSafeInteger(maxIterations) || maxIterations < 0) throw new Error("--max-iterations must be a non-negative integer; 0 means unlimited");
|
|
if (argv.includes("--push") && !argv.includes("--commit")) throw new Error("--push requires --commit");
|
|
|
|
const supervisorArgs = [];
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
if (argv[index] === "--restart-delay-ms" || argv[index] === "--max-restarts") {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
supervisorArgs.push(argv[index]);
|
|
}
|
|
const supervisorCommand = ["tools/web/run-wbv2-supervisor.mjs", ...supervisorArgs].join(" ");
|
|
const supervisorNodeArgs = ["tools/web/run-wbv2-supervisor.mjs", ...supervisorArgs];
|
|
|
|
fs.mkdirSync(abs(controlRoot), { recursive: true });
|
|
let lockHandle;
|
|
const acquireLock = () => {
|
|
try {
|
|
return fs.openSync(abs(lockPath), "wx");
|
|
}
|
|
catch (error) {
|
|
if (error?.code !== "EEXIST") throw error;
|
|
try {
|
|
const lock = JSON.parse(fs.readFileSync(abs(lockPath), "utf8"));
|
|
if (Number.isSafeInteger(lock.pid)) {
|
|
try { process.kill(lock.pid, 0); }
|
|
catch (probeError) {
|
|
if (probeError?.code === "ESRCH") {
|
|
fs.unlinkSync(abs(lockPath));
|
|
return fs.openSync(abs(lockPath), "wx");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch {
|
|
// An unreadable lock is treated as owned to avoid concurrent guardians.
|
|
}
|
|
throw new Error(`another WBV2 guardian appears to be running; lock=${lockPath}`);
|
|
}
|
|
};
|
|
lockHandle = acquireLock();
|
|
fs.writeFileSync(lockHandle, `${JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() })}\n`);
|
|
|
|
let released = false;
|
|
const releaseLock = () => {
|
|
if (released || lockHandle === undefined) return;
|
|
released = true;
|
|
fs.closeSync(lockHandle);
|
|
try { fs.unlinkSync(abs(lockPath)); }
|
|
catch (error) { if (error?.code !== "ENOENT") throw error; }
|
|
};
|
|
const appendLog = (text) => fs.appendFileSync(abs(logPath), `[${new Date().toISOString()}] ${text}\n`);
|
|
const writeCheckpoint = (payload) => fs.writeFileSync(abs(checkpointPath), `${JSON.stringify({ schemaVersion: 1, pid: process.pid, updatedAt: new Date().toISOString(), ...payload }, null, 2)}\n`);
|
|
const readJson = (file) => {
|
|
try { return JSON.parse(fs.readFileSync(abs(file), "utf8")); }
|
|
catch { return null; }
|
|
};
|
|
const currentTask = () => {
|
|
const queue = fs.readFileSync(abs("docs/EXECUTION_QUEUE.md"), "utf8");
|
|
const task = queue.match(/\| 当前任务 \| `(WBV2-P\d+-\d+)` \|/u)?.[1];
|
|
const parentManifest = queue.match(/\| parent manifest \| `([^`]+)` \|/u)?.[1];
|
|
if (!task || !parentManifest) throw new Error("queue pointer is incomplete");
|
|
return { task, parentManifest };
|
|
};
|
|
const samePointer = (left, right) => left?.task === right?.task && left?.parentManifest === right?.parentManifest;
|
|
const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
|
|
let child;
|
|
let stopRequested = false;
|
|
let stopSignal;
|
|
const signalChild = (signal) => {
|
|
if (!child?.pid) return;
|
|
try { process.kill(-child.pid, signal); }
|
|
catch (error) {
|
|
if (error?.code !== "ESRCH") {
|
|
try { child.kill(signal); }
|
|
catch { /* The exit handler will record the final state. */ }
|
|
}
|
|
}
|
|
};
|
|
const requestStop = (signal) => {
|
|
stopRequested = true;
|
|
stopSignal = signal;
|
|
appendLog(`stop-requested signal=${signal}`);
|
|
signalChild(signal);
|
|
};
|
|
process.once("SIGINT", () => requestStop("SIGINT"));
|
|
process.once("SIGTERM", () => requestStop("SIGTERM"));
|
|
|
|
const runSupervisor = () => new Promise((resolve) => {
|
|
child = spawn(process.execPath, supervisorNodeArgs, {
|
|
cwd: root,
|
|
detached: true,
|
|
env: process.env,
|
|
stdio: "inherit",
|
|
});
|
|
child.once("error", (error) => resolve({ code: 1, signal: null, error: error.message }));
|
|
child.once("exit", (code, signal) => {
|
|
child = undefined;
|
|
resolve({ code: code ?? 1, signal });
|
|
});
|
|
});
|
|
|
|
function safeRestart(checkpoint, pointer) {
|
|
if (!checkpoint) return { ok: false, reason: "supervisor_checkpoint_missing" };
|
|
if (checkpoint.state === "blocked") return { ok: false, reason: checkpoint.reason ?? "supervisor_blocked" };
|
|
if (checkpoint.state === "stopped") return { ok: false, reason: checkpoint.reason ?? "supervisor_stopped" };
|
|
if (checkpoint.state !== "handed_off") return { ok: false, reason: `supervisor_state_${checkpoint.state ?? "unknown"}` };
|
|
if (!samePointer(checkpoint.after, pointer)) return { ok: false, reason: "handoff_pointer_mismatch" };
|
|
return { ok: true };
|
|
}
|
|
|
|
let guardianExit = 0;
|
|
let restartCount = 0;
|
|
try {
|
|
appendLog(`started pid=${process.pid} supervisor=${supervisorCommand} maxRestarts=${maxRestarts} restartDelayMs=${restartDelayMs}`);
|
|
writeCheckpoint({ state: "started", restartCount, supervisor: supervisorCommand, current: currentTask() });
|
|
while (true) {
|
|
if (stopRequested) {
|
|
writeCheckpoint({ state: "stopped", restartCount, reason: stopSignal, current: currentTask() });
|
|
guardianExit = stopSignal === "SIGINT" ? 130 : 143;
|
|
break;
|
|
}
|
|
const before = currentTask();
|
|
writeCheckpoint({ state: "running", restartCount, before, supervisor: supervisorCommand });
|
|
const result = await runSupervisor();
|
|
const checkpoint = readJson(checkpointPath);
|
|
const after = currentTask();
|
|
appendLog(`supervisor-exit code=${result.code} signal=${result.signal ?? "none"} task=${before.task} queue=${after.task} state=${checkpoint?.state ?? "missing"}`);
|
|
if (stopRequested) {
|
|
writeCheckpoint({ state: "stopped", restartCount, reason: stopSignal, before, after, supervisorExit: result.code });
|
|
guardianExit = stopSignal === "SIGINT" ? 130 : 143;
|
|
break;
|
|
}
|
|
if (checkpoint?.state === "complete") {
|
|
writeCheckpoint({ state: "complete", restartCount, current: after, supervisorExit: result.code });
|
|
break;
|
|
}
|
|
const decision = safeRestart(checkpoint, after);
|
|
if (!decision.ok) {
|
|
writeCheckpoint({ state: "blocked", restartCount, before, after, reason: decision.reason, supervisorExit: result.code });
|
|
appendLog(`blocked reason=${decision.reason}; no automatic replay`);
|
|
guardianExit = 2;
|
|
break;
|
|
}
|
|
restartCount += 1;
|
|
if (maxRestarts > 0 && restartCount > maxRestarts) {
|
|
writeCheckpoint({ state: "blocked", restartCount, before, after, reason: "max_safe_restarts_exceeded", supervisorExit: result.code });
|
|
appendLog("blocked reason=max_safe_restarts_exceeded");
|
|
guardianExit = 3;
|
|
break;
|
|
}
|
|
writeCheckpoint({ state: "restarting", restartCount, after, reason: "safe_handoff_checkpoint" });
|
|
appendLog(`restarting count=${restartCount} after=${after.task}`);
|
|
await sleep(restartDelayMs);
|
|
}
|
|
}
|
|
catch (error) {
|
|
guardianExit = 4;
|
|
writeCheckpoint({ state: "blocked", restartCount, reason: "guardian_error", error: error?.message ?? String(error) });
|
|
appendLog(`blocked reason=guardian_error error=${error?.message ?? String(error)}`);
|
|
}
|
|
finally {
|
|
signalChild("SIGTERM");
|
|
releaseLock();
|
|
}
|
|
process.exitCode = guardianExit;
|