import { ReplayTimeline, REPLAY_SCHEMA, canonicalJson, fnv1a64 } from "../worker/replay.mjs"; import { WorkspaceJournal } from "./journal.mjs"; export const CHECKPOINT_SCHEMA = "simulation-checkpoint.v1"; export const RECOVERY_SCHEMA = "simulation-recovery.v1"; export const RECOVERY_STATES = Object.freeze({ VALIDATING: "validating", REPLAYING: "replaying", RECOVERED: "recovered", ABORTED: "aborted", BLOCKED: "blocked", }); const OPERATION_STATUSES = new Set(["prepared", "executing", "awaiting_ack", "verified", "aborted", "failed"]); function integer(value, name) { if (!Number.isSafeInteger(value) || value < 0) throw new Error(`checkpoint_${name}_invalid`); } function clone(value) { return value === undefined ? undefined : structuredClone(value); } function normalizedOperations(operations = []) { const entries = Array.isArray(operations) ? operations : Object.entries(operations).map(([operation_id, value]) => ({ operation_id, ...value })); const seen = new Set(); return entries.map((operation) => { if (!operation || typeof operation !== "object" || typeof operation.operation_id !== "string" || operation.operation_id.length === 0) throw new Error("checkpoint_operation_invalid"); if (seen.has(operation.operation_id)) throw new Error("checkpoint_operation_duplicate"); seen.add(operation.operation_id); if (!OPERATION_STATUSES.has(operation.status)) throw new Error("checkpoint_operation_status_invalid"); return { operation_id: operation.operation_id, status: operation.status, seq: operation.seq ?? null, kind: operation.kind ?? null, result_hash: operation.result_hash ?? null, }; }).sort((left, right) => left.operation_id.localeCompare(right.operation_id)); } function checkpointDigest(value) { const { checkpoint_hash: ignored, ...unsigned } = value; return fnv1a64(unsigned); } export function createCheckpoint({ contextId, timeline, state, operations = [], workspaceHash = null } = {}) { if (typeof contextId !== "string" || contextId.length === 0) throw new Error("checkpoint_context_invalid"); if (!(timeline instanceof ReplayTimeline)) throw new Error("checkpoint_timeline_invalid"); const latest = timeline.snapshots.latest(); if (!latest || latest.seq !== timeline.events.cursor || latest.tick !== timeline.events.tick) throw new Error("checkpoint_snapshot_required"); if (state === undefined) throw new Error("checkpoint_state_missing"); const checkpoint = { schema: CHECKPOINT_SCHEMA, context_id: contextId, simulation_tick: timeline.events.tick, event_cursor: timeline.events.cursor, timeline: timeline.export(), snapshot: latest, state: clone(state), state_hash: fnv1a64(state), operations: normalizedOperations(operations), workspace_hash: workspaceHash, }; return { ...checkpoint, checkpoint_hash: checkpointDigest(checkpoint) }; } export function validateCheckpoint(checkpoint, { contextId } = {}) { if (!checkpoint || checkpoint.schema !== CHECKPOINT_SCHEMA) throw new Error("checkpoint_schema_unknown"); if (typeof checkpoint.context_id !== "string" || (contextId && checkpoint.context_id !== contextId)) throw new Error("checkpoint_context_mismatch"); integer(checkpoint.simulation_tick, "tick"); integer(checkpoint.event_cursor, "cursor"); if (fnv1a64(checkpoint.state) !== checkpoint.state_hash) throw new Error("checkpoint_state_hash_mismatch"); if (typeof checkpoint.checkpoint_hash !== "string" || checkpointDigest(checkpoint) !== checkpoint.checkpoint_hash) throw new Error("checkpoint_hash_mismatch"); if (!checkpoint.timeline || checkpoint.timeline.schema !== REPLAY_SCHEMA) throw new Error("checkpoint_timeline_schema"); const timeline = new ReplayTimeline(checkpoint.timeline); if (timeline.events.cursor !== checkpoint.event_cursor || timeline.events.tick !== checkpoint.simulation_tick) throw new Error("checkpoint_cursor_mismatch"); if (!checkpoint.snapshot || checkpoint.snapshot.snapshot_id !== timeline.snapshots.latest()?.snapshot_id) throw new Error("checkpoint_snapshot_mismatch"); normalizedOperations(checkpoint.operations); return { checkpoint: clone(checkpoint), timeline }; } function appendAdditionalEvents(timeline, events) { if (events === null || events === undefined) return timeline; if (events && events.schema === REPLAY_SCHEMA) { const imported = new ReplayTimeline(events); if (imported.events.cursor < timeline.events.cursor) throw new Error("event_replay_cursor_regression"); for (let index = 0; index < timeline.events.events.length; index += 1) { if (canonicalJson(imported.events.events[index]) !== canonicalJson(timeline.events.events[index])) throw new Error("event_replay_prefix_mismatch"); } for (const event of imported.events.events.slice(timeline.events.cursor)) timeline.appendEvent(event.seq, event.tick, event.kind, event.payload, event.payload_hash); return timeline; } if (!Array.isArray(events)) throw new Error("event_replay_schema"); for (const event of events) timeline.appendEvent(event.seq, event.tick, event.kind, event.payload, event.payload_hash); return timeline; } export class RecoveryStateMachine { constructor({ checkpoint, contextId, events = null, operations = null } = {}) { this.checkpoint = checkpoint; this.contextId = contextId; this.events = events; this.operations = operations; this.state = RECOVERY_STATES.VALIDATING; this.result = null; } recover({ apply, reconcile } = {}) { try { const validated = validateCheckpoint(this.checkpoint, { contextId: this.contextId }); const timeline = appendAdditionalEvents(validated.timeline, this.events); const checkpointOperations = normalizedOperations(this.operations ?? validated.checkpoint.operations); const pending = checkpointOperations.filter((operation) => ["prepared", "executing", "awaiting_ack"].includes(operation.status)); if (pending.length > 0 && typeof reconcile !== "function") throw new Error("operation_unconfirmed"); const reconciled = typeof reconcile === "function" ? checkpointOperations.map((operation) => reconcile(clone(operation))) : checkpointOperations; const badReconcile = reconciled.find((operation) => !operation || !OPERATION_STATUSES.has(operation.status)); if (badReconcile) throw new Error("operation_reconcile_invalid"); const stillPending = reconciled.filter((operation) => ["prepared", "executing", "awaiting_ack"].includes(operation.status)); if (stillPending.length > 0) throw new Error("operation_unconfirmed"); if (reconciled.some((operation) => operation.status === "failed")) throw new Error("operation_failed"); this.state = RECOVERY_STATES.REPLAYING; const startSeq = validated.checkpoint.event_cursor; const delta = timeline.events.events.filter((event) => event.seq > startSeq); let state = clone(validated.checkpoint.state); if (delta.length > 0 && typeof apply !== "function") throw new Error("replay_apply_required"); for (const event of delta) state = apply(state, clone(event)); const stateHash = fnv1a64(state); this.state = reconciled.some((operation) => operation.status === "aborted") ? RECOVERY_STATES.ABORTED : RECOVERY_STATES.RECOVERED; this.result = { schema: RECOVERY_SCHEMA, status: this.state, context_id: validated.checkpoint.context_id, checkpoint_hash: validated.checkpoint.checkpoint_hash, checkpoint_tick: validated.checkpoint.simulation_tick, simulation_tick: timeline.events.tick, last_verified_seq: timeline.events.cursor, state, state_hash: stateHash, replayed_events: delta.length, operations: reconciled, blocked: [], }; return clone(this.result); } catch (error) { this.state = RECOVERY_STATES.BLOCKED; this.result = { schema: RECOVERY_SCHEMA, status: RECOVERY_STATES.BLOCKED, context_id: this.contextId ?? null, checkpoint_hash: this.checkpoint?.checkpoint_hash ?? null, checkpoint_tick: this.checkpoint?.simulation_tick ?? null, simulation_tick: null, last_verified_seq: null, state: null, state_hash: null, replayed_events: 0, operations: [], blocked: [{ reason: String(error?.message ?? error) }], }; return clone(this.result); } } } export function recoverSimulation(options) { return new RecoveryStateMachine(options).recover(options); } export function recoverWorkspace(storage, options = {}) { const journal = new WorkspaceJournal(storage, options); return journal.recover(); } export { canonicalJson };