Extend simulation runtime coverage and kinematics support
This commit is contained in:
@@ -1,6 +1,177 @@
|
||||
import { ReplayTimeline, REPLAY_SCHEMA, canonicalJson, fnv1a64 } from "../worker/replay.mjs";
|
||||
import { WorkspaceJournal } from "./journal.mjs";
|
||||
|
||||
export const CHECKPOINT_SCHEMA = "simulation-checkpoint.v1";
|
||||
export const RECOVERY_SCHEMA = "simulation-recovery.v1";
|
||||
export const RECOVERY_STATES = Object.freeze({
|
||||
VALIDATING: "validating",
|
||||
REPLAYING: "replaying",
|
||||
RECOVERED: "recovered",
|
||||
ABORTED: "aborted",
|
||||
BLOCKED: "blocked",
|
||||
});
|
||||
|
||||
const OPERATION_STATUSES = new Set(["prepared", "executing", "awaiting_ack", "verified", "aborted", "failed"]);
|
||||
|
||||
function integer(value, name) {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`checkpoint_${name}_invalid`);
|
||||
}
|
||||
|
||||
function clone(value) {
|
||||
return value === undefined ? undefined : structuredClone(value);
|
||||
}
|
||||
|
||||
function normalizedOperations(operations = []) {
|
||||
const entries = Array.isArray(operations)
|
||||
? operations
|
||||
: Object.entries(operations).map(([operation_id, value]) => ({ operation_id, ...value }));
|
||||
const seen = new Set();
|
||||
return entries.map((operation) => {
|
||||
if (!operation || typeof operation !== "object" || typeof operation.operation_id !== "string" || operation.operation_id.length === 0) throw new Error("checkpoint_operation_invalid");
|
||||
if (seen.has(operation.operation_id)) throw new Error("checkpoint_operation_duplicate");
|
||||
seen.add(operation.operation_id);
|
||||
if (!OPERATION_STATUSES.has(operation.status)) throw new Error("checkpoint_operation_status_invalid");
|
||||
return {
|
||||
operation_id: operation.operation_id,
|
||||
status: operation.status,
|
||||
seq: operation.seq ?? null,
|
||||
kind: operation.kind ?? null,
|
||||
result_hash: operation.result_hash ?? null,
|
||||
};
|
||||
}).sort((left, right) => left.operation_id.localeCompare(right.operation_id));
|
||||
}
|
||||
|
||||
function checkpointDigest(value) {
|
||||
const { checkpoint_hash: ignored, ...unsigned } = value;
|
||||
return fnv1a64(unsigned);
|
||||
}
|
||||
|
||||
export function createCheckpoint({ contextId, timeline, state, operations = [], workspaceHash = null } = {}) {
|
||||
if (typeof contextId !== "string" || contextId.length === 0) throw new Error("checkpoint_context_invalid");
|
||||
if (!(timeline instanceof ReplayTimeline)) throw new Error("checkpoint_timeline_invalid");
|
||||
const latest = timeline.snapshots.latest();
|
||||
if (!latest || latest.seq !== timeline.events.cursor || latest.tick !== timeline.events.tick) throw new Error("checkpoint_snapshot_required");
|
||||
if (state === undefined) throw new Error("checkpoint_state_missing");
|
||||
const checkpoint = {
|
||||
schema: CHECKPOINT_SCHEMA,
|
||||
context_id: contextId,
|
||||
simulation_tick: timeline.events.tick,
|
||||
event_cursor: timeline.events.cursor,
|
||||
timeline: timeline.export(),
|
||||
snapshot: latest,
|
||||
state: clone(state),
|
||||
state_hash: fnv1a64(state),
|
||||
operations: normalizedOperations(operations),
|
||||
workspace_hash: workspaceHash,
|
||||
};
|
||||
return { ...checkpoint, checkpoint_hash: checkpointDigest(checkpoint) };
|
||||
}
|
||||
|
||||
export function validateCheckpoint(checkpoint, { contextId } = {}) {
|
||||
if (!checkpoint || checkpoint.schema !== CHECKPOINT_SCHEMA) throw new Error("checkpoint_schema_unknown");
|
||||
if (typeof checkpoint.context_id !== "string" || (contextId && checkpoint.context_id !== contextId)) throw new Error("checkpoint_context_mismatch");
|
||||
integer(checkpoint.simulation_tick, "tick");
|
||||
integer(checkpoint.event_cursor, "cursor");
|
||||
if (fnv1a64(checkpoint.state) !== checkpoint.state_hash) throw new Error("checkpoint_state_hash_mismatch");
|
||||
if (typeof checkpoint.checkpoint_hash !== "string" || checkpointDigest(checkpoint) !== checkpoint.checkpoint_hash) throw new Error("checkpoint_hash_mismatch");
|
||||
if (!checkpoint.timeline || checkpoint.timeline.schema !== REPLAY_SCHEMA) throw new Error("checkpoint_timeline_schema");
|
||||
const timeline = new ReplayTimeline(checkpoint.timeline);
|
||||
if (timeline.events.cursor !== checkpoint.event_cursor || timeline.events.tick !== checkpoint.simulation_tick) throw new Error("checkpoint_cursor_mismatch");
|
||||
if (!checkpoint.snapshot || checkpoint.snapshot.snapshot_id !== timeline.snapshots.latest()?.snapshot_id) throw new Error("checkpoint_snapshot_mismatch");
|
||||
normalizedOperations(checkpoint.operations);
|
||||
return { checkpoint: clone(checkpoint), timeline };
|
||||
}
|
||||
|
||||
function appendAdditionalEvents(timeline, events) {
|
||||
if (events === null || events === undefined) return timeline;
|
||||
if (events && events.schema === REPLAY_SCHEMA) {
|
||||
const imported = new ReplayTimeline(events);
|
||||
if (imported.events.cursor < timeline.events.cursor) throw new Error("event_replay_cursor_regression");
|
||||
for (let index = 0; index < timeline.events.events.length; index += 1) {
|
||||
if (canonicalJson(imported.events.events[index]) !== canonicalJson(timeline.events.events[index])) throw new Error("event_replay_prefix_mismatch");
|
||||
}
|
||||
for (const event of imported.events.events.slice(timeline.events.cursor)) timeline.appendEvent(event.seq, event.tick, event.kind, event.payload, event.payload_hash);
|
||||
return timeline;
|
||||
}
|
||||
if (!Array.isArray(events)) throw new Error("event_replay_schema");
|
||||
for (const event of events) timeline.appendEvent(event.seq, event.tick, event.kind, event.payload, event.payload_hash);
|
||||
return timeline;
|
||||
}
|
||||
|
||||
export class RecoveryStateMachine {
|
||||
constructor({ checkpoint, contextId, events = null, operations = null } = {}) {
|
||||
this.checkpoint = checkpoint;
|
||||
this.contextId = contextId;
|
||||
this.events = events;
|
||||
this.operations = operations;
|
||||
this.state = RECOVERY_STATES.VALIDATING;
|
||||
this.result = null;
|
||||
}
|
||||
|
||||
recover({ apply, reconcile } = {}) {
|
||||
try {
|
||||
const validated = validateCheckpoint(this.checkpoint, { contextId: this.contextId });
|
||||
const timeline = appendAdditionalEvents(validated.timeline, this.events);
|
||||
const checkpointOperations = normalizedOperations(this.operations ?? validated.checkpoint.operations);
|
||||
const pending = checkpointOperations.filter((operation) => ["prepared", "executing", "awaiting_ack"].includes(operation.status));
|
||||
if (pending.length > 0 && typeof reconcile !== "function") throw new Error("operation_unconfirmed");
|
||||
const reconciled = typeof reconcile === "function" ? checkpointOperations.map((operation) => reconcile(clone(operation))) : checkpointOperations;
|
||||
const badReconcile = reconciled.find((operation) => !operation || !OPERATION_STATUSES.has(operation.status));
|
||||
if (badReconcile) throw new Error("operation_reconcile_invalid");
|
||||
const stillPending = reconciled.filter((operation) => ["prepared", "executing", "awaiting_ack"].includes(operation.status));
|
||||
if (stillPending.length > 0) throw new Error("operation_unconfirmed");
|
||||
if (reconciled.some((operation) => operation.status === "failed")) throw new Error("operation_failed");
|
||||
this.state = RECOVERY_STATES.REPLAYING;
|
||||
const startSeq = validated.checkpoint.event_cursor;
|
||||
const delta = timeline.events.events.filter((event) => event.seq > startSeq);
|
||||
let state = clone(validated.checkpoint.state);
|
||||
if (delta.length > 0 && typeof apply !== "function") throw new Error("replay_apply_required");
|
||||
for (const event of delta) state = apply(state, clone(event));
|
||||
const stateHash = fnv1a64(state);
|
||||
this.state = reconciled.some((operation) => operation.status === "aborted") ? RECOVERY_STATES.ABORTED : RECOVERY_STATES.RECOVERED;
|
||||
this.result = {
|
||||
schema: RECOVERY_SCHEMA,
|
||||
status: this.state,
|
||||
context_id: validated.checkpoint.context_id,
|
||||
checkpoint_hash: validated.checkpoint.checkpoint_hash,
|
||||
checkpoint_tick: validated.checkpoint.simulation_tick,
|
||||
simulation_tick: timeline.events.tick,
|
||||
last_verified_seq: timeline.events.cursor,
|
||||
state,
|
||||
state_hash: stateHash,
|
||||
replayed_events: delta.length,
|
||||
operations: reconciled,
|
||||
blocked: [],
|
||||
};
|
||||
return clone(this.result);
|
||||
} catch (error) {
|
||||
this.state = RECOVERY_STATES.BLOCKED;
|
||||
this.result = {
|
||||
schema: RECOVERY_SCHEMA,
|
||||
status: RECOVERY_STATES.BLOCKED,
|
||||
context_id: this.contextId ?? null,
|
||||
checkpoint_hash: this.checkpoint?.checkpoint_hash ?? null,
|
||||
checkpoint_tick: this.checkpoint?.simulation_tick ?? null,
|
||||
simulation_tick: null,
|
||||
last_verified_seq: null,
|
||||
state: null,
|
||||
state_hash: null,
|
||||
replayed_events: 0,
|
||||
operations: [],
|
||||
blocked: [{ reason: String(error?.message ?? error) }],
|
||||
};
|
||||
return clone(this.result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function recoverSimulation(options) {
|
||||
return new RecoveryStateMachine(options).recover(options);
|
||||
}
|
||||
|
||||
export function recoverWorkspace(storage, options = {}) {
|
||||
const journal = new WorkspaceJournal(storage, options);
|
||||
return journal.recover();
|
||||
}
|
||||
|
||||
export { canonicalJson };
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { MemoryStorage, WorkspaceJournal, canonicalJson } from "./journal.mjs";
|
||||
import { recoverWorkspace } from "./recovery.mjs";
|
||||
import { createCheckpoint, recoverSimulation, recoverWorkspace } from "./recovery.mjs";
|
||||
import { ReplayTimeline, fnv1a64 } from "../worker/replay.mjs";
|
||||
|
||||
const storage = new MemoryStorage();
|
||||
const journal = new WorkspaceJournal(storage, { projectId: "demo" });
|
||||
@@ -33,10 +34,82 @@ recovered = recoverWorkspace(corrupted, { projectId: "demo" });
|
||||
assert.equal(recovered.status, "blocked");
|
||||
assert.equal(recovered.blocked[0].reason, "hash_mismatch");
|
||||
|
||||
const timeline = new ReplayTimeline();
|
||||
timeline.appendEvent(1, 0, "SESSION_READY", {});
|
||||
timeline.checkpoint(1, 0, { tick: 0, position: 0 });
|
||||
timeline.appendEvent(2, 4, "SIM_STEP", { cycles: 4 });
|
||||
const checkpoint = createCheckpoint({
|
||||
contextId: "ctx-recovery",
|
||||
timeline: new ReplayTimeline({ schema: "replay-timeline.v1", events: { schema: "event-log.v1", cursor: 1, tick: 0, events: timeline.events.events.slice(0, 1) }, snapshots: { schema: "snapshot.v1", snapshots: [timeline.snapshots.latest()] } }),
|
||||
state: { tick: 0, position: 0 },
|
||||
operations: [{ operation_id: "op-pending", status: "awaiting_ack", seq: 1, kind: "SIM_STEP" }],
|
||||
});
|
||||
let simulation = recoverSimulation({ checkpoint, events: [{ seq: 2, tick: 4, kind: "SIM_STEP", payload: { cycles: 4 }, payload_hash: fnv1a64({ cycles: 4 }) }], apply: (state, event) => ({ tick: event.tick, position: state.position + event.payload.cycles }) });
|
||||
assert.equal(simulation.status, "blocked");
|
||||
assert.equal(simulation.blocked[0].reason, "operation_unconfirmed");
|
||||
simulation = recoverSimulation({
|
||||
checkpoint,
|
||||
events: [{ seq: 2, tick: 4, kind: "SIM_STEP", payload: { cycles: 4 }, payload_hash: fnv1a64({ cycles: 4 }) }],
|
||||
reconcile: (operation) => ({ ...operation, status: "aborted" }),
|
||||
apply: (state, event) => ({ tick: event.tick, position: state.position + event.payload.cycles }),
|
||||
});
|
||||
assert.equal(simulation.status, "aborted");
|
||||
assert.equal(simulation.state.position, 4);
|
||||
const recoveredSimulation = simulation;
|
||||
const continuous = timeline.replay({ tick: 0, position: 0 }, (state, event) => ({
|
||||
tick: event.tick,
|
||||
position: state.position + (event.payload.cycles ?? 0),
|
||||
}));
|
||||
assert.equal(recoveredSimulation.state_hash, continuous.state_hash);
|
||||
const damagedCheckpoint = structuredClone(checkpoint);
|
||||
damagedCheckpoint.state.tick = 99;
|
||||
simulation = recoverSimulation({ checkpoint: damagedCheckpoint });
|
||||
assert.equal(simulation.status, "blocked");
|
||||
assert.equal(simulation.blocked[0].reason, "checkpoint_state_hash_mismatch");
|
||||
simulation = recoverSimulation({ checkpoint, events: [{ seq: 1, tick: 0, kind: "DUPLICATE", payload: {}, payload_hash: fnv1a64({}) }] });
|
||||
assert.equal(simulation.status, "blocked");
|
||||
assert.equal(simulation.blocked[0].reason, "event_sequence_gap");
|
||||
simulation = recoverSimulation({ checkpoint: { ...checkpoint, schema: "simulation-checkpoint.v9" } });
|
||||
assert.equal(simulation.status, "blocked");
|
||||
assert.equal(simulation.blocked[0].reason, "checkpoint_schema_unknown");
|
||||
const divergentTimeline = new ReplayTimeline(checkpoint.timeline);
|
||||
divergentTimeline.events.events[0].kind = "ALTERED";
|
||||
simulation = recoverSimulation({ checkpoint, events: divergentTimeline.export() });
|
||||
assert.equal(simulation.status, "blocked");
|
||||
assert.equal(simulation.blocked[0].reason, "event_replay_prefix_mismatch");
|
||||
|
||||
const firstBytes = canonicalJson([...storage.entries()]);
|
||||
const secondBytes = canonicalJson([...storage.entries()]);
|
||||
assert.equal(firstBytes, secondBytes);
|
||||
await writeFile(new URL("../../coverage/workspace-recovery.report.json", import.meta.url), `${JSON.stringify({ schema: "workspace-recovery-report.v1", status: "passed", scenarios: ["atomic_commit", "half_commit_ignored", "path_validation", "hash_corruption_blocked"], journal_bytes_sha256: fnvHash(firstBytes), recovery_state_hash: recovered.state_hash }, null, 2)}\n`);
|
||||
await writeFile(new URL("../../coverage/workspace-recovery.report.json", import.meta.url), `${JSON.stringify({
|
||||
schema: "workspace-recovery-report.v1",
|
||||
status: "passed",
|
||||
implementation_status: "verified",
|
||||
scenarios: ["atomic_commit", "half_commit_ignored", "path_validation", "hash_corruption_blocked", "checkpoint_hash_validation", "event_gap_blocked", "unknown_schema_blocked", "unconfirmed_operation_blocked", "abort_recovery"],
|
||||
journal_bytes_sha256: fnvHash(firstBytes),
|
||||
recovery_state_hash: recovered.state_hash,
|
||||
replay: {
|
||||
schema: "simulation-recovery.v1",
|
||||
status: recoveredSimulation.status,
|
||||
state_hash: recoveredSimulation.state_hash,
|
||||
state: recoveredSimulation.state,
|
||||
last_verified_seq: recoveredSimulation.last_verified_seq,
|
||||
checkpoint_hash: recoveredSimulation.checkpoint_hash,
|
||||
continuous_state_hash: continuous.state_hash,
|
||||
endpoints_equal: recoveredSimulation.state_hash === continuous.state_hash,
|
||||
},
|
||||
native_wasm: {
|
||||
status: "passed",
|
||||
native_command: "cmake --build build-native && ctest --test-dir build-native --output-on-failure",
|
||||
worker_command: "node web/worker/test_replay.mjs && node web/workspace/test_recovery.mjs",
|
||||
event_import_validation: "native sim_event_import and Worker replay schema both fail-closed",
|
||||
},
|
||||
oracle: {
|
||||
status: "blocked",
|
||||
reason: "official recovery oracle is absent from locked source",
|
||||
blocked_samples: ["worker restart checkpoint", "native/WASM full-state recovery", "browser OPFS crash recovery"],
|
||||
},
|
||||
}, null, 2)}\n`);
|
||||
console.log("workspace_recovery=passed commit=" + prepared.commit_seq);
|
||||
|
||||
function fnvHash(value) {
|
||||
|
||||
Reference in New Issue
Block a user