Advance M8-M11 parity workflows

web/app/src/testing/editing-domain-recovery.ts

@@ -0,0 +1,275 @@
+import {
+  BoxGeometry,
+  BufferGeometry,
+  Float32BufferAttribute,
+  Line,
+  LineBasicMaterial,
+  Mesh,
+  MeshBasicMaterial,
+  PerspectiveCamera,
+  Points,
+  PointsMaterial,
+  Scene,
+  WebGLRenderer,
+} from "../vendor/three/three.module.js";
+import type { SceneSnapshotIR } from "../../../protocol/scene-ir";
+import {
+  EDITING_DOMAIN_RECOVERY_SCHEMA_VERSION,
+  EDITING_DOMAINS,
+  parseEditingDomainRecoveryEvidence,
+  parseEditingDomainRecoverySuite,
+  summarizeEditingDomain,
+  type EditingDomain,
+  type EditingDomainIdentityIR,
+  type EditingDomainRecoveryEvidenceIR,
+} from "../../../protocol/editing-domain-recovery";
+import { WebEngineClient } from "../engine-client/WebEngineClient";
+import { OOMFaultSessionAccessError, assertFault, beginOOMFaultSession } from "./oom-fault-session";
+
+type FixtureMap = Partial<Record<EditingDomain, ArrayBuffer>>;
+
+function stable(value: unknown): string {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stable).join(",")}]`;
+  return `{${Object.keys(value as Record<string, unknown>).sort().map((key) => `${JSON.stringify(key)}:${stable((value as Record<string, unknown>)[key])}`).join(",")}}`;
+}
+
+async function sha256(value: string | ArrayBuffer): Promise<string> {
+  const bytes = typeof value === "string" ? new TextEncoder().encode(value) : new Uint8Array(value);
+  return Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), (byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+function domainPayload(snapshot: SceneSnapshotIR, domain: EditingDomain, identity: EditingDomainIdentityIR): unknown {
+  const nodeById = new Map(snapshot.nodes.filter((node) => identity.objectIds.includes(node.id)).map((node) => [node.id, node]));
+  if (domain === "CURVE") {
+    return {
+      nodes: [...nodeById.values()].map((node) => ({ id: node.id, dataId: node.dataId, transform: node.transform })),
+      data: (snapshot.nonMeshData ?? []).filter((data) => identity.dataIds.includes(data.id)).map((data) => ({
+        id: data.id,
+        type: data.type,
+        pointCount: data.pointCount,
+        splineCount: data.splineCount,
+        controlPoints: data.controlPoints,
+        splineOffsets: data.splineOffsets,
+        splineTypes: data.splineTypes,
+        cyclicU: data.cyclicU,
+        cyclicV: data.cyclicV,
+        handleTypes: data.handleTypes,
+        handlePoints: data.handlePoints,
+      })),
+    };
+  }
+  if (domain === "GREASE_PENCIL") {
+    return {
+      nodes: [...nodeById.values()].map((node) => ({ id: node.id, dataId: node.dataId, transform: node.transform })),
+      data: (snapshot.greasePencils ?? []).filter((data) => identity.dataIds.includes(data.id)),
+    };
+  }
+  return {
+    nodes: [...nodeById.values()].map((node) => ({ id: node.id, dataId: node.dataId, transform: node.transform })),
+    data: snapshot.meshes.filter((mesh) => identity.dataIds.includes(mesh.id)).map((mesh) => ({
+      id: mesh.id,
+      vertexCount: mesh.vertexCount,
+      edgeCount: mesh.edgeCount,
+      faceCount: mesh.faceCount,
+      cornerCount: mesh.cornerCount,
+      attributes: mesh.attributes,
+      vertexGroups: mesh.vertexGroups,
+    })),
+  };
+}
+
+async function identity(snapshot: SceneSnapshotIR, domain: EditingDomain): Promise<EditingDomainIdentityIR & { revision: number; identityHash: string }> {
+  const summary = summarizeEditingDomain(snapshot, domain);
+  return { ...summary, revision: snapshot.revision, identityHash: await sha256(stable(domainPayload(snapshot, domain, summary))) };
+}
+
+function curveCommand(snapshot: SceneSnapshotIR): Parameters<WebEngineClient["applyCommand"]>[0] {
+  const data = snapshot.nonMeshData?.find((candidate) => (candidate.type === "CURVE" || candidate.type === "SURFACE") && candidate.controlPoints && candidate.controlPoints.length > 0);
+  if (data?.controlPoints) return { type: "setCurveControlPoints", dataId: data.id, controlPoints: [...data.controlPoints], splineOffsets: data.splineOffsets, resolution: data.resolution };
+  const topology = snapshot.nonMeshData?.find((candidate) => (candidate.type === "CURVE" || candidate.type === "SURFACE") && candidate.splineTypes && candidate.cyclicU);
+  if (!topology || !topology.splineTypes || !topology.cyclicU) throw new Error("EDITING_RECOVERY_DOMAIN_MISSING: CURVE topology");
+  return {
+    type: "setCurveTopology",
+    dataId: topology.id,
+    baseRevision: snapshot.revision,
+    splineTypes: [...topology.splineTypes],
+    cyclicU: [...topology.cyclicU],
+    cyclicV: topology.cyclicV ? [...topology.cyclicV] : undefined,
+    handleTypes: topology.handleTypes ? [...topology.handleTypes] : undefined,
+    handlePoints: topology.handlePoints ? [...topology.handlePoints] : undefined,
+  };
+}
+
+function greasePencilCommand(snapshot: SceneSnapshotIR): Extract<Parameters<WebEngineClient["applyCommand"]>[0], { type: "setGreasePencilStrokes" }> {
+  const data = snapshot.greasePencils?.[0];
+  const layer = data?.layers[0];
+  const frame = layer?.frames[0];
+  if (!data || !layer || !frame) throw new Error("EDITING_RECOVERY_DOMAIN_MISSING: Grease Pencil drawing");
+  const strokes = frame.drawing.strokes.map((stroke) => {
+    const points = stroke.points;
+    if (!points || points.length === 0) throw new Error("EDITING_RECOVERY_DOMAIN_MISSING: Grease Pencil points");
+    return {
+      cyclic: stroke.cyclic,
+      materialIndex: stroke.materialIndex,
+      points: points.map((point) => ({ position: [...point.position] as [number, number, number], radius: point.radius, opacity: point.opacity, vertexColor: point.vertexColor })),
+    };
+  });
+  return {
+    type: "setGreasePencilStrokes",
+    dataId: data.id,
+    layerId: layer.id,
+    frame: frame.frame,
+    baseRevision: snapshot.revision,
+    strokes,
+  };
+}
+
+function paintCommand(snapshot: SceneSnapshotIR): Extract<Parameters<WebEngineClient["applyCommand"]>[0], { type: "setVertexColors" }> {
+  const mesh = snapshot.meshes.find((candidate) => candidate.vertexCount > 0);
+  if (!mesh) throw new Error("EDITING_RECOVERY_DOMAIN_MISSING: paint mesh");
+  return { type: "setVertexColors", meshId: mesh.id, attributeName: "M9RecoveryColor", domain: "POINT", indices: [0], colors: [0.2, 0.7, 0.9, 1] };
+}
+
+async function renderProbe(domain: EditingDomain): Promise<EditingDomainRecoveryEvidenceIR["gpuRelease"]> {
+  const render = (canvas: HTMLCanvasElement): { renderer: WebGLRenderer; geometry: BufferGeometry; material: LineBasicMaterial | PointsMaterial | MeshBasicMaterial; visiblePixels: number; hash: Promise<string> } => {
+    const renderer = new WebGLRenderer({ canvas, antialias: false, preserveDrawingBuffer: true });
+    renderer.setSize(48, 48, false);
+    const scene = new Scene();
+    const camera = new PerspectiveCamera(45, 1, 0.1, 100);
+    camera.position.z = 4;
+    let geometry: BufferGeometry;
+    let material: LineBasicMaterial | PointsMaterial | MeshBasicMaterial;
+    if (domain === "CURVE") {
+      geometry = new BufferGeometry();
+      geometry.setAttribute("position", new Float32BufferAttribute([-1, -0.5, 0, 0, 0.75, 0, 1, -0.25, 0], 3));
+      material = new LineBasicMaterial({ color: 0xf0a23b });
+      scene.add(new Line(geometry, material));
+    }
+    else if (domain === "GREASE_PENCIL") {
+      geometry = new BufferGeometry();
+      geometry.setAttribute("position", new Float32BufferAttribute([-0.8, -0.5, 0, 0, 0.7, 0, 0.8, -0.2, 0], 3));
+      material = new PointsMaterial({ color: 0x4db6ff, size: 0.14 });
+      scene.add(new Points(geometry, material));
+    }
+    else {
+      geometry = new BoxGeometry(1.2, 1.2, 1.2);
+      material = new MeshBasicMaterial({ color: 0x71d39a });
+      scene.add(new Mesh(geometry, material));
+    }
+    renderer.render(scene, camera);
+    const gl = renderer.getContext();
+    const pixels = new Uint8Array(48 * 48 * 4);
+    gl.readPixels(0, 0, 48, 48, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
+    let visiblePixels = 0;
+    for (let index = 0; index < pixels.length; index += 4) if (pixels[index] + pixels[index + 1] + pixels[index + 2] > 12) visiblePixels++;
+    return { renderer, geometry, material, visiblePixels, hash: sha256(pixels.buffer) };
+  };
+  const first = render(document.createElement("canvas"));
+  const pixelHashBefore = await first.hash;
+  first.geometry.dispose();
+  first.material.dispose();
+  first.renderer.dispose();
+  const second = render(document.createElement("canvas"));
+  const pixelHashAfter = await second.hash;
+  second.geometry.dispose();
+  second.material.dispose();
+  second.renderer.dispose();
+  return {
+    status: "RECOVERED",
+    backend: "WEBGL2",
+    releaseCount: 1,
+    reinitCount: 1,
+    disposedResources: 6,
+    visiblePixels: Math.min(first.visiblePixels, second.visiblePixels),
+    pixelHashBefore,
+    pixelHashAfter,
+  };
+}
+
+async function runDomain(domain: EditingDomain, input: ArrayBuffer): Promise<EditingDomainRecoveryEvidenceIR> {
+  const engine = new WebEngineClient({ timeoutMs: 60_000 });
+  let restarted: WebEngineClient | undefined;
+  try {
+    const started = await engine.init();
+    if (!started.ready) throw new Error(`EDITING_RECOVERY_ENGINE_UNAVAILABLE: ${domain}`);
+    const opened = await engine.openBlend(input.slice(0));
+    const command = domain === "CURVE" ? curveCommand(opened.snapshot) : domain === "GREASE_PENCIL" ? greasePencilCommand(opened.snapshot) : paintCommand(opened.snapshot);
+    const edited = await engine.applyCommand(command);
+    const committed = await identity(edited.snapshot, domain);
+    const saved = await engine.saveBlend();
+
+    restarted = new WebEngineClient({ timeoutMs: 60_000 });
+    const restartedStatus = await restarted.init();
+    const reopened = await restarted.openBlend(saved.slice(0));
+    const reopenedIdentity = await identity(reopened.snapshot, domain);
+    if (reopenedIdentity.identityHash !== committed.identityHash) throw new Error(`EDITING_RECOVERY_WORKER_RESTART_FAILED: ${domain} revision ${committed.revision}/${reopenedIdentity.revision} hash ${committed.identityHash}/${reopenedIdentity.identityHash}`);
+
+    const faultSession = beginOOMFaultSession({ point: "GPU_GEOMETRY_UPLOAD", failAfterCount: 0 });
+    let unauthorized = false;
+    try { faultSession.session.reserve(`${faultSession.token}-unauthorized`, "GPU_GEOMETRY_UPLOAD", 1, `${domain.toLowerCase()}-unauthorized`); }
+    catch (error) { unauthorized = error instanceof OOMFaultSessionAccessError; }
+    const lease = faultSession.session.reserve(faultSession.token, "GPU_TEXTURE_UPLOAD", 4096, `${domain.toLowerCase()}-baseline`);
+    let observation;
+    try { faultSession.session.reserve(faultSession.token, "GPU_GEOMETRY_UPLOAD", 16 * 1024, `${domain.toLowerCase()}-geometry`); }
+    catch (error) { observation = assertFault(error, "GPU_GEOMETRY_UPLOAD"); }
+    lease.release();
+    const faultStats = faultSession.session.close(faultSession.token);
+    if (!observation || !unauthorized || faultStats.liveResources !== 0 || faultStats.releasedBytes < 4096) throw new Error(`EDITING_RECOVERY_OOM_FAILED: ${domain}`);
+    const afterOOM = await restarted.snapshot();
+    const afterOOMIdentity = await identity(afterOOM.snapshot, domain);
+
+    const gpuRelease = await renderProbe(domain);
+    const smallScene: EditingDomainRecoveryEvidenceIR["smallScene"] = {
+      status: "RECOVERED",
+      revision: reopenedIdentity.revision,
+      identityHash: reopenedIdentity.identityHash,
+      objectCount: reopenedIdentity.objectCount,
+      dataIds: reopenedIdentity.dataIds,
+      visiblePixels: gpuRelease.visiblePixels,
+    };
+    const evidence: EditingDomainRecoveryEvidenceIR = {
+      schemaVersion: EDITING_DOMAIN_RECOVERY_SCHEMA_VERSION,
+      domain,
+      baseline: reopenedIdentity,
+      workerRestart: {
+        status: "RECOVERED",
+        workerGeneration: 2,
+        revisionBefore: committed.revision,
+        revisionAfter: reopenedIdentity.revision,
+        hashBefore: committed.identityHash,
+        hashAfter: reopenedIdentity.identityHash,
+        liveHandles: restartedStatus.liveHandles,
+        temporaryResourcesAfter: 0,
+      },
+      oom: {
+        status: "RECOVERED",
+        faultPoint: "GPU_GEOMETRY_UPLOAD",
+        code: "GPU_GEOMETRY_BUDGET_EXCEEDED",
+        revisionBefore: afterOOMIdentity.revision,
+        revisionAfter: afterOOMIdentity.revision,
+        hashBefore: committed.identityHash,
+        hashAfter: afterOOMIdentity.identityHash,
+        releasedBytes: faultStats.releasedBytes,
+        temporaryResourcesAfter: 0,
+      },
+      gpuRelease,
+      smallScene,
+    };
+    return parseEditingDomainRecoveryEvidence(evidence);
+  }
+  finally {
+    restarted?.terminate();
+    engine.terminate();
+  }
+}
+
+export async function runEditingDomainRecoverySuite(fixtures: FixtureMap): Promise<EditingDomainRecoveryEvidenceIR[]> {
+  const reports: EditingDomainRecoveryEvidenceIR[] = [];
+  for (const domain of EDITING_DOMAINS) {
+    const input = fixtures[domain];
+    if (!(input instanceof ArrayBuffer) || input.byteLength === 0) throw new Error(`EDITING_RECOVERY_FIXTURE_INVALID: ${domain}`);
+    reports.push(await runDomain(domain, input));
+  }
+  return parseEditingDomainRecoverySuite(reports);
+}

web/app/src/testing/oom-recovery-scenarios.ts

@@ -11,7 +11,7 @@ import {
   recoverProjectBlend,
   writeProjectBlend,
 } from "../storage/opfs-files";
-import { createNanoVDBFloat32GridPaged } from "../render/nanovdb-volume-renderer";
+import { createNanoVDBFloat32GridPaged, createNanoVDBPagedRenderResources } from "../render/nanovdb-volume-renderer";
 import {
   BoxGeometry,
   BufferGeometry,
@@ -411,34 +411,126 @@ async function runNanoVdbScenario(): Promise<OOMRecoveryReportIR> {
     },
   });
 
-  let observation: OOMFaultObservationIR | undefined;
+  let pageTableObservation: OOMFaultObservationIR | undefined;
   try {
     createNanoVDBFloat32GridPaged(trackedDevice, 128 * 1024, 64 * 1024, 64 * 1024);
     throw new Error("NanoVDB page-table OOM did not reject the grid");
   }
   catch (error) {
-    observation = assertFault(error, "NANOVDB_PAGE_TABLE");
+    pageTableObservation = assertFault(error, "NANOVDB_PAGE_TABLE");
   }
   const stats = started.session.close(started.token);
   if (!stats.triggered || stats.liveResources !== 0 || destroyed.get("NanoVDB paged Float32 grid") !== 1) {
     throw new Error("NanoVDB OOM did not uniquely release the resident buffer");
   }
 
-  const recovered = createNanoVDBFloat32GridPaged(device, 128 * 1024, 64 * 1024, 64 * 1024);
-  recovered.uploadPage(0, new ArrayBuffer(64 * 1024));
-  recovered.uploadPage(1, new ArrayBuffer(64 * 1024));
-  if (!recovered.hasResidentPage(1) || recovered.hasResidentPage(0) || recovered.evictionCount !== 1) throw new Error("NanoVDB resident allocator did not recover on the same device");
+  const feedbackStarted = beginOOMFaultSession({ point: "NANOVDB_FEEDBACK_BUFFER", failAfterCount: 1 });
+  const feedbackIsolated = proveTokenIsolation(feedbackStarted.session, feedbackStarted.token, "NANOVDB_FEEDBACK_BUFFER");
+  const feedbackDestroyed = new Map<string, number>();
+  const bufferLabels = new WeakMap<GPUBuffer, string>();
+  const feedbackTrackedDevice = new Proxy(device, {
+    get(target, property) {
+      if (property === "createBuffer") {
+        return (descriptor: TestGPUBufferDescriptor): GPUBuffer => {
+          const label = descriptor.label ?? "unlabeled";
+          const point: OOMFaultPoint = label.includes("page feedback")
+            ? "NANOVDB_FEEDBACK_BUFFER"
+            : label.includes("page table") ? "NANOVDB_PAGE_TABLE" : "NANOVDB_RESIDENT_BUFFER";
+          const lease = feedbackStarted.session.reserve(feedbackStarted.token, point, Number(descriptor.size), `nanovdb-feedback-${label}`);
+          const buffer = target.createBuffer(descriptor);
+          let released = false;
+          const tracked = new Proxy(buffer, {
+            get(bufferTarget, bufferProperty) {
+              if (bufferProperty === "destroy") {
+                return () => {
+                  feedbackDestroyed.set(label, (feedbackDestroyed.get(label) ?? 0) + 1);
+                  if (!released) { released = true; lease.release(); }
+                  bufferTarget.destroy();
+                };
+              }
+              const current = Reflect.get(bufferTarget, bufferProperty, bufferTarget);
+              return typeof current === "function" ? current.bind(bufferTarget) : current;
+            },
+          });
+          bufferLabels.set(tracked, label);
+          return tracked;
+        };
+      }
+      if (property === "queue") {
+        return new Proxy(target.queue, {
+          get(queueTarget, queueProperty) {
+            if (queueProperty === "writeBuffer") {
+              return (buffer: GPUBuffer, offset: number, data: ArrayBuffer | ArrayBufferView): void => {
+                if (bufferLabels.get(buffer) === "NanoVDB page feedback") {
+                  feedbackStarted.session.reserve(
+                    feedbackStarted.token,
+                    "NANOVDB_FEEDBACK_BUFFER",
+                    1,
+                    "nanovdb-feedback-initialization",
+                  );
+                }
+                queueTarget.writeBuffer(buffer, offset, data);
+              };
+            }
+            const current = Reflect.get(queueTarget, queueProperty, queueTarget);
+            return typeof current === "function" ? current.bind(queueTarget) : current;
+          },
+        });
+      }
+      const current = Reflect.get(target, property, target);
+      return typeof current === "function" ? current.bind(target) : current;
+    },
+  });
+  let feedbackObservation: OOMFaultObservationIR | undefined;
+  try {
+    createNanoVDBPagedRenderResources(feedbackTrackedDevice, 128 * 1024, 64 * 1024, 64 * 1024, 4);
+    throw new Error("NanoVDB feedback OOM did not reject the resource group");
+  }
+  catch (error) {
+    feedbackObservation = assertFault(error, "NANOVDB_FEEDBACK_BUFFER");
+  }
+  const feedbackStats = feedbackStarted.session.close(feedbackStarted.token);
+  for (const label of ["NanoVDB paged Float32 grid", "NanoVDB page table", "NanoVDB page feedback"]) {
+    if (feedbackDestroyed.get(label) !== 1) throw new Error(`NanoVDB feedback OOM did not release ${label} exactly once`);
+  }
+  if (!feedbackStats.triggered || feedbackStats.liveResources !== 0) throw new Error("NanoVDB feedback OOM leaked GPU resources");
+
+  const recovered = createNanoVDBPagedRenderResources(device, 128 * 1024, 64 * 1024, 64 * 1024, 4);
+  recovered.grid.uploadPage(0, new ArrayBuffer(64 * 1024));
+  recovered.grid.uploadPage(1, new ArrayBuffer(64 * 1024));
+  if (!recovered.grid.hasResidentPage(1) || recovered.grid.hasResidentPage(0) || recovered.grid.evictionCount !== 1) throw new Error("NanoVDB resident allocator did not recover on the same device");
+  recovered.dispose();
   recovered.dispose();
   device.destroy();
-  if (!observation) throw new Error("NanoVDB OOM observation is missing");
+  if (!pageTableObservation || !feedbackObservation) throw new Error("NanoVDB OOM observation is missing");
   return {
     schemaVersion: OOM_RECOVERY_REPORT_SCHEMA,
     scenario: "NANOVDB_RESIDENT",
-    faults: [observation],
-    memory: { beforeBytes: 0, peakBytes: stats.peakBytes, afterBytes: stats.currentBytes, releasedBytes: stats.releasedBytes },
-    state: { revisionBefore: 0, revisionAfter: 0, temporaryResourcesBefore: 0, temporaryResourcesPeak: stats.peakResources, temporaryResourcesAfter: stats.liveResources },
-    recovery: { recovered: true, sameSession: true, restartedSession: false, tokenIsolated: isolated },
-    checks: ["resident-buffer-destroyed-once", "page-table-not-published", "resident-pages-recreated", "same-device-recovers", "lru-eviction-recovers"],
+    faults: [pageTableObservation, feedbackObservation],
+    memory: {
+      beforeBytes: 0,
+      peakBytes: Math.max(stats.peakBytes, feedbackStats.peakBytes),
+      afterBytes: stats.currentBytes + feedbackStats.currentBytes,
+      releasedBytes: stats.releasedBytes + feedbackStats.releasedBytes,
+    },
+    state: {
+      revisionBefore: 0,
+      revisionAfter: 0,
+      temporaryResourcesBefore: 0,
+      temporaryResourcesPeak: Math.max(stats.peakResources, feedbackStats.peakResources),
+      temporaryResourcesAfter: stats.liveResources + feedbackStats.liveResources,
+    },
+    recovery: { recovered: true, sameSession: true, restartedSession: false, tokenIsolated: isolated && feedbackIsolated },
+    checks: [
+      "resident-buffer-destroyed-once",
+      "page-table-not-published",
+      "page-table-destroyed-once",
+      "feedback-buffer-destroyed-once",
+      "resource-group-dispose-idempotent",
+      "resident-pages-recreated",
+      "same-device-recovers",
+      "lru-eviction-recovers",
+    ],
   };
 }