Extend simulation runtime coverage and kinematics support
This commit is contained in:
@@ -1,6 +1,177 @@
|
||||
import { ReplayTimeline, REPLAY_SCHEMA, canonicalJson, fnv1a64 } from "../worker/replay.mjs";
|
||||
import { WorkspaceJournal } from "./journal.mjs";
|
||||
|
||||
export const CHECKPOINT_SCHEMA = "simulation-checkpoint.v1";
|
||||
export const RECOVERY_SCHEMA = "simulation-recovery.v1";
|
||||
export const RECOVERY_STATES = Object.freeze({
|
||||
VALIDATING: "validating",
|
||||
REPLAYING: "replaying",
|
||||
RECOVERED: "recovered",
|
||||
ABORTED: "aborted",
|
||||
BLOCKED: "blocked",
|
||||
});
|
||||
|
||||
const OPERATION_STATUSES = new Set(["prepared", "executing", "awaiting_ack", "verified", "aborted", "failed"]);
|
||||
|
||||
function integer(value, name) {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`checkpoint_${name}_invalid`);
|
||||
}
|
||||
|
||||
function clone(value) {
|
||||
return value === undefined ? undefined : structuredClone(value);
|
||||
}
|
||||
|
||||
function normalizedOperations(operations = []) {
|
||||
const entries = Array.isArray(operations)
|
||||
? operations
|
||||
: Object.entries(operations).map(([operation_id, value]) => ({ operation_id, ...value }));
|
||||
const seen = new Set();
|
||||
return entries.map((operation) => {
|
||||
if (!operation || typeof operation !== "object" || typeof operation.operation_id !== "string" || operation.operation_id.length === 0) throw new Error("checkpoint_operation_invalid");
|
||||
if (seen.has(operation.operation_id)) throw new Error("checkpoint_operation_duplicate");
|
||||
seen.add(operation.operation_id);
|
||||
if (!OPERATION_STATUSES.has(operation.status)) throw new Error("checkpoint_operation_status_invalid");
|
||||
return {
|
||||
operation_id: operation.operation_id,
|
||||
status: operation.status,
|
||||
seq: operation.seq ?? null,
|
||||
kind: operation.kind ?? null,
|
||||
result_hash: operation.result_hash ?? null,
|
||||
};
|
||||
}).sort((left, right) => left.operation_id.localeCompare(right.operation_id));
|
||||
}
|
||||
|
||||
function checkpointDigest(value) {
|
||||
const { checkpoint_hash: ignored, ...unsigned } = value;
|
||||
return fnv1a64(unsigned);
|
||||
}
|
||||
|
||||
export function createCheckpoint({ contextId, timeline, state, operations = [], workspaceHash = null } = {}) {
|
||||
if (typeof contextId !== "string" || contextId.length === 0) throw new Error("checkpoint_context_invalid");
|
||||
if (!(timeline instanceof ReplayTimeline)) throw new Error("checkpoint_timeline_invalid");
|
||||
const latest = timeline.snapshots.latest();
|
||||
if (!latest || latest.seq !== timeline.events.cursor || latest.tick !== timeline.events.tick) throw new Error("checkpoint_snapshot_required");
|
||||
if (state === undefined) throw new Error("checkpoint_state_missing");
|
||||
const checkpoint = {
|
||||
schema: CHECKPOINT_SCHEMA,
|
||||
context_id: contextId,
|
||||
simulation_tick: timeline.events.tick,
|
||||
event_cursor: timeline.events.cursor,
|
||||
timeline: timeline.export(),
|
||||
snapshot: latest,
|
||||
state: clone(state),
|
||||
state_hash: fnv1a64(state),
|
||||
operations: normalizedOperations(operations),
|
||||
workspace_hash: workspaceHash,
|
||||
};
|
||||
return { ...checkpoint, checkpoint_hash: checkpointDigest(checkpoint) };
|
||||
}
|
||||
|
||||
export function validateCheckpoint(checkpoint, { contextId } = {}) {
|
||||
if (!checkpoint || checkpoint.schema !== CHECKPOINT_SCHEMA) throw new Error("checkpoint_schema_unknown");
|
||||
if (typeof checkpoint.context_id !== "string" || (contextId && checkpoint.context_id !== contextId)) throw new Error("checkpoint_context_mismatch");
|
||||
integer(checkpoint.simulation_tick, "tick");
|
||||
integer(checkpoint.event_cursor, "cursor");
|
||||
if (fnv1a64(checkpoint.state) !== checkpoint.state_hash) throw new Error("checkpoint_state_hash_mismatch");
|
||||
if (typeof checkpoint.checkpoint_hash !== "string" || checkpointDigest(checkpoint) !== checkpoint.checkpoint_hash) throw new Error("checkpoint_hash_mismatch");
|
||||
if (!checkpoint.timeline || checkpoint.timeline.schema !== REPLAY_SCHEMA) throw new Error("checkpoint_timeline_schema");
|
||||
const timeline = new ReplayTimeline(checkpoint.timeline);
|
||||
if (timeline.events.cursor !== checkpoint.event_cursor || timeline.events.tick !== checkpoint.simulation_tick) throw new Error("checkpoint_cursor_mismatch");
|
||||
if (!checkpoint.snapshot || checkpoint.snapshot.snapshot_id !== timeline.snapshots.latest()?.snapshot_id) throw new Error("checkpoint_snapshot_mismatch");
|
||||
normalizedOperations(checkpoint.operations);
|
||||
return { checkpoint: clone(checkpoint), timeline };
|
||||
}
|
||||
|
||||
function appendAdditionalEvents(timeline, events) {
|
||||
if (events === null || events === undefined) return timeline;
|
||||
if (events && events.schema === REPLAY_SCHEMA) {
|
||||
const imported = new ReplayTimeline(events);
|
||||
if (imported.events.cursor < timeline.events.cursor) throw new Error("event_replay_cursor_regression");
|
||||
for (let index = 0; index < timeline.events.events.length; index += 1) {
|
||||
if (canonicalJson(imported.events.events[index]) !== canonicalJson(timeline.events.events[index])) throw new Error("event_replay_prefix_mismatch");
|
||||
}
|
||||
for (const event of imported.events.events.slice(timeline.events.cursor)) timeline.appendEvent(event.seq, event.tick, event.kind, event.payload, event.payload_hash);
|
||||
return timeline;
|
||||
}
|
||||
if (!Array.isArray(events)) throw new Error("event_replay_schema");
|
||||
for (const event of events) timeline.appendEvent(event.seq, event.tick, event.kind, event.payload, event.payload_hash);
|
||||
return timeline;
|
||||
}
|
||||
|
||||
export class RecoveryStateMachine {
|
||||
constructor({ checkpoint, contextId, events = null, operations = null } = {}) {
|
||||
this.checkpoint = checkpoint;
|
||||
this.contextId = contextId;
|
||||
this.events = events;
|
||||
this.operations = operations;
|
||||
this.state = RECOVERY_STATES.VALIDATING;
|
||||
this.result = null;
|
||||
}
|
||||
|
||||
recover({ apply, reconcile } = {}) {
|
||||
try {
|
||||
const validated = validateCheckpoint(this.checkpoint, { contextId: this.contextId });
|
||||
const timeline = appendAdditionalEvents(validated.timeline, this.events);
|
||||
const checkpointOperations = normalizedOperations(this.operations ?? validated.checkpoint.operations);
|
||||
const pending = checkpointOperations.filter((operation) => ["prepared", "executing", "awaiting_ack"].includes(operation.status));
|
||||
if (pending.length > 0 && typeof reconcile !== "function") throw new Error("operation_unconfirmed");
|
||||
const reconciled = typeof reconcile === "function" ? checkpointOperations.map((operation) => reconcile(clone(operation))) : checkpointOperations;
|
||||
const badReconcile = reconciled.find((operation) => !operation || !OPERATION_STATUSES.has(operation.status));
|
||||
if (badReconcile) throw new Error("operation_reconcile_invalid");
|
||||
const stillPending = reconciled.filter((operation) => ["prepared", "executing", "awaiting_ack"].includes(operation.status));
|
||||
if (stillPending.length > 0) throw new Error("operation_unconfirmed");
|
||||
if (reconciled.some((operation) => operation.status === "failed")) throw new Error("operation_failed");
|
||||
this.state = RECOVERY_STATES.REPLAYING;
|
||||
const startSeq = validated.checkpoint.event_cursor;
|
||||
const delta = timeline.events.events.filter((event) => event.seq > startSeq);
|
||||
let state = clone(validated.checkpoint.state);
|
||||
if (delta.length > 0 && typeof apply !== "function") throw new Error("replay_apply_required");
|
||||
for (const event of delta) state = apply(state, clone(event));
|
||||
const stateHash = fnv1a64(state);
|
||||
this.state = reconciled.some((operation) => operation.status === "aborted") ? RECOVERY_STATES.ABORTED : RECOVERY_STATES.RECOVERED;
|
||||
this.result = {
|
||||
schema: RECOVERY_SCHEMA,
|
||||
status: this.state,
|
||||
context_id: validated.checkpoint.context_id,
|
||||
checkpoint_hash: validated.checkpoint.checkpoint_hash,
|
||||
checkpoint_tick: validated.checkpoint.simulation_tick,
|
||||
simulation_tick: timeline.events.tick,
|
||||
last_verified_seq: timeline.events.cursor,
|
||||
state,
|
||||
state_hash: stateHash,
|
||||
replayed_events: delta.length,
|
||||
operations: reconciled,
|
||||
blocked: [],
|
||||
};
|
||||
return clone(this.result);
|
||||
} catch (error) {
|
||||
this.state = RECOVERY_STATES.BLOCKED;
|
||||
this.result = {
|
||||
schema: RECOVERY_SCHEMA,
|
||||
status: RECOVERY_STATES.BLOCKED,
|
||||
context_id: this.contextId ?? null,
|
||||
checkpoint_hash: this.checkpoint?.checkpoint_hash ?? null,
|
||||
checkpoint_tick: this.checkpoint?.simulation_tick ?? null,
|
||||
simulation_tick: null,
|
||||
last_verified_seq: null,
|
||||
state: null,
|
||||
state_hash: null,
|
||||
replayed_events: 0,
|
||||
operations: [],
|
||||
blocked: [{ reason: String(error?.message ?? error) }],
|
||||
};
|
||||
return clone(this.result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function recoverSimulation(options) {
|
||||
return new RecoveryStateMachine(options).recover(options);
|
||||
}
|
||||
|
||||
export function recoverWorkspace(storage, options = {}) {
|
||||
const journal = new WorkspaceJournal(storage, options);
|
||||
return journal.recover();
|
||||
}
|
||||
|
||||
export { canonicalJson };
|
||||
|
||||
Reference in New Issue
Block a user