Advance M8-M11 parity workflows

web/app/src/three-adapter/grease-pencil.ts

@@ -1,5 +1,6 @@
 import {
   BufferGeometry,
+  type Camera,
   Color,
   Float32BufferAttribute,
   Group,
@@ -7,9 +8,15 @@ import {
   LineBasicMaterial,
   Points,
   PointsMaterial,
+  Vector3,
   type Object3D,
 } from "../vendor/three/three.module.js";
 import type { GreasePencilDataIR, GreasePencilDrawingIR, GreasePencilFrameIR, GreasePencilLayerIR } from "../../../protocol/grease-pencil";
+import type {
+  GreasePencilDrawingScopeIR,
+  GreasePencilMarqueeCandidateIR,
+  GreasePencilStablePointRefIR,
+} from "../../../protocol/grease-pencil-marquee";
 import type { SceneNodeIR } from "../../../protocol/scene-ir";
 
 interface DrawingPreview {
@@ -18,13 +25,7 @@ interface DrawingPreview {
   onion: "NONE" | "PREVIOUS" | "NEXT";
 }
 
-export interface GreasePencilPointRef {
-  dataId: string;
-  layerId: string;
-  frame: number;
-  strokeIndex: number;
-  pointIndex: number;
-}
+export type GreasePencilPointRef = GreasePencilStablePointRefIR;
 
 export interface GreasePencilPointPreview extends GreasePencilPointRef {
   position: [number, number, number];
@@ -55,7 +56,7 @@ function layerDrawings(layer: GreasePencilLayerIR, frame: number): DrawingPrevie
 }
 
 function pointKey(point: GreasePencilPointRef): string {
-  return `${point.dataId}\u0000${point.layerId}\u0000${point.frame}\u0000${point.strokeIndex}\u0000${point.pointIndex}`;
+  return `${point.drawingId}\u0000${point.strokeId}\u0000${point.pointId}`;
 }
 
 function addDrawing(group: Group, dataId: string, layer: GreasePencilLayerIR, preview: DrawingPreview): number {
@@ -101,6 +102,8 @@ function addDrawing(group: Group, dataId: string, layer: GreasePencilLayerIR, pr
       line.userData.greasePencilPreviewDataId = dataId;
       line.userData.greasePencilPreviewLayerId = layer.id;
       line.userData.greasePencilPreviewFrame = preview.frame;
+      line.userData.greasePencilPreviewDrawingId = preview.drawing.id;
+      line.userData.greasePencilPreviewStrokeId = stroke.id;
       line.userData.greasePencilPreviewStrokeIndex = strokeIndex;
       line.userData.greasePencilPointIndexMap = Array.from({ length: linePointCount }, (_, index) => index % stroke.points!.length);
       line.userData.greasePencilPointBasePositions = new Float32Array(positions.subarray(0, linePointCount * 3));
@@ -114,8 +117,11 @@ function addDrawing(group: Group, dataId: string, layer: GreasePencilLayerIR, pr
       points.userData.greasePencilPointDataId = dataId;
       points.userData.greasePencilPointLayerId = layer.id;
       points.userData.greasePencilPointFrame = preview.frame;
+      points.userData.greasePencilPointDrawingId = preview.drawing.id;
+      points.userData.greasePencilPointStrokeId = stroke.id;
       points.userData.greasePencilPointStrokeIndex = strokeIndex;
       points.userData.greasePencilPointIndexMap = Array.from({ length: stroke.points.length }, (_, index) => index);
+      points.userData.greasePencilPointIdMap = stroke.points.map((point) => point.id);
       points.userData.greasePencilPointBasePositions = new Float32Array(positions.subarray(0, stroke.points.length * 3));
       points.visible = false;
       (strokeObject ?? group).add(points);
@@ -147,9 +153,49 @@ export function greasePencilPointRef(object: Object3D, pointIndex: number): Grea
   const dataId = object.userData.greasePencilPointDataId;
   const layerId = object.userData.greasePencilPointLayerId;
   const frame = object.userData.greasePencilPointFrame;
+  const drawingId = object.userData.greasePencilPointDrawingId;
+  const strokeId = object.userData.greasePencilPointStrokeId;
   const strokeIndex = object.userData.greasePencilPointStrokeIndex;
-  if (typeof dataId !== "string" || typeof layerId !== "string" || !Number.isSafeInteger(frame) || !Number.isSafeInteger(strokeIndex) || !Number.isSafeInteger(pointIndex) || pointIndex < 0) return null;
-  return { dataId, layerId, frame, strokeIndex, pointIndex };
+  const pointIndexMap = object.userData.greasePencilPointIndexMap as number[] | undefined;
+  const pointIdMap = object.userData.greasePencilPointIdMap as string[] | undefined;
+  const stablePointIndex = pointIndexMap?.[pointIndex];
+  const pointId = pointIdMap?.[pointIndex];
+  if (typeof dataId !== "string" || typeof layerId !== "string" || typeof drawingId !== "string" ||
+      typeof strokeId !== "string" || typeof pointId !== "string" || !Number.isSafeInteger(frame) ||
+      !Number.isSafeInteger(strokeIndex) || !Number.isSafeInteger(stablePointIndex) ||
+      (stablePointIndex ?? -1) < 0) return null;
+  return { dataId, layerId, frame, drawingId, strokeId, pointId, strokeIndex, pointIndex: stablePointIndex! };
+}
+
+export function greasePencilMarqueeCandidates(
+  root: Object3D,
+  drawing: GreasePencilDrawingScopeIR,
+  camera: Camera,
+): GreasePencilMarqueeCandidateIR[] {
+  const candidates: GreasePencilMarqueeCandidateIR[] = [];
+  const worldPosition = new Vector3();
+  root.updateMatrixWorld(true);
+  camera.updateMatrixWorld(true);
+  root.traverse((object) => {
+    if (!(object instanceof Points) || object.userData.greasePencilPointDrawingId !== drawing.drawingId ||
+        object.userData.greasePencilPointDataId !== drawing.dataId ||
+        object.userData.greasePencilPointLayerId !== drawing.layerId ||
+        object.userData.greasePencilPointFrame !== drawing.frame) return;
+    const positions = object.geometry.getAttribute("position");
+    for (let index = 0; index < positions.count; index++) {
+      const point = greasePencilPointRef(object, index);
+      if (!point) continue;
+      worldPosition.fromBufferAttribute(positions, index);
+      object.localToWorld(worldPosition);
+      worldPosition.project(camera);
+      if (worldPosition.z < -1 || worldPosition.z > 1) continue;
+      candidates.push({
+        ...point,
+        viewportPosition: [(worldPosition.x + 1) / 2, (1 - worldPosition.y) / 2],
+      });
+    }
+  });
+  return candidates;
 }
 
 export function applyGreasePencilPointSelection(root: Object3D, selection: readonly GreasePencilPointRef[]): void {
@@ -157,11 +203,10 @@ export function applyGreasePencilPointSelection(root: Object3D, selection: reado
   root.traverse((object) => {
     if (!(object instanceof Points) || !(object.material instanceof PointsMaterial)) return;
     const count = object.geometry.getAttribute("position")?.count ?? 0;
-    const first = greasePencilPointRef(object, 0);
-    if (!first) return;
     const colors = new Float32Array(count * 3);
     for (let pointIndex = 0; pointIndex < count; pointIndex++) {
-      const active = selected.has(pointKey({ ...first, pointIndex }));
+      const point = greasePencilPointRef(object, pointIndex);
+      const active = point ? selected.has(pointKey(point)) : false;
       colors.set(active ? [1, 0.38, 0.08] : [0.46, 0.73, 1], pointIndex * 3);
     }
     object.geometry.setAttribute("color", new Float32BufferAttribute(colors, 3));

web/app/src/three-adapter/offscreen-viewport-protocol.ts

@@ -2,10 +2,14 @@ import type { SceneSnapshotIR } from "../../../protocol/scene-ir";
 import type { MeshElementMode, MeshGeometryBuffer } from "../../../protocol/web-engine";
 import type { NonMeshGeometryChunk } from "../../../protocol/nonmesh-binary";
 import type { GPUTextureAsset } from "../../../protocol/render-assets";
+import type { PBRLightingBudgetReport, PBRTextureBudgetReport } from "../../../protocol/render-budget";
 import type { NonMeshElementKind } from "./nonmesh";
 import type { GreasePencilPointPreview, GreasePencilPointRef } from "./grease-pencil";
 import type { CurveGizmoFrameIR, CurveGizmoHandleIR, CurveGizmoScreenFrameIR } from "../../../protocol/nonmesh-interaction";
 import type { NanoVDBViewportAssetIR } from "../volume/nanovdb-viewport";
+import type { ViewportCameraState } from "../../../protocol/viewport-camera";
+import type { GreasePencilDrawingScopeIR, GreasePencilMarqueeBoxIR, GreasePencilMarqueeResultIR } from "../../../protocol/grease-pencil-marquee";
+import type { PaintDepthVisibilityRequestIR, PaintDepthVisibilityResultIR } from "../../../protocol/paint-depth-visibility";
 
 export type OffscreenViewportRequest =
   | { type: "init"; canvas: OffscreenCanvas; width: number; height: number; pixelRatio: number }
@@ -13,23 +17,29 @@ export type OffscreenViewportRequest =
   | { type: "textureAssets"; assets: GPUTextureAsset[] }
   | { type: "volumeAssets"; assets: NanoVDBViewportAssetIR[] }
   | { type: "resize"; width: number; height: number; pixelRatio: number }
-  | { type: "selection"; objectIds: string[]; elements: Array<{ dataId: string; kind: NonMeshElementKind; index: number }>; greasePencilPoints: GreasePencilPointRef[] }
+  | { type: "selection"; objectIds: string[]; elements: Array<{ dataId: string; kind: NonMeshElementKind; index: number }>; greasePencilPoints: GreasePencilPointRef[]; greasePencilSelectionRevision: number }
   | { type: "interaction"; editMode: boolean; selectionMode: MeshElementMode }
   | { type: "curveHandlePreview"; dataId: string; handles: CurveGizmoHandleIR[] | null }
   | { type: "greasePencilPointPreview"; dataId: string; layerId: string; frame: number; points: GreasePencilPointPreview[] | null }
   | { type: "curveGizmoFrame"; dataId: string | null; frame: CurveGizmoFrameIR | null }
   | { type: "orbit"; deltaX: number; deltaY: number; zoom: number }
-  | { type: "pick"; x: number; y: number; additive: boolean }
+  | { type: "pick"; x: number; y: number; additive: boolean; baseSelectionRevision: number }
+  | { type: "greasePencilMarquee"; drawing: GreasePencilDrawingScopeIR; box: GreasePencilMarqueeBoxIR; baseRevision: number; baseSelectionRevision: number; additive: boolean }
+  | { type: "paintDepthVisibility"; requestId: string; request: PaintDepthVisibilityRequestIR }
   | { type: "dispose" };
 
 export type OffscreenViewportResponse =
   | { type: "ready" }
-  | { type: "frame"; visiblePixels: number }
-  | { type: "snapshotStatus"; nonMeshCount: number; nonMeshBlockedCount: number; greasePencilCount: number; greasePencilBlockedCount: number; greasePencilOnionStrokeCount: number }
-  | { type: "textureStatus"; loaded: number; rejected: number; bytes: number; errors: string[]; errorCodes: string[] }
+  | { type: "frame"; visiblePixels: number; camera?: ViewportCameraState }
+  | { type: "snapshotStatus"; nonMeshCount: number; nonMeshBlockedCount: number; greasePencilCount: number; greasePencilBlockedCount: number; greasePencilOnionStrokeCount: number; renderBudget: PBRLightingBudgetReport }
+  | { type: "textureStatus"; loaded: number; rejected: number; bytes: number; errors: string[]; errorCodes: string[]; budget: PBRTextureBudgetReport }
   | { type: "volumeStatus"; status: "none" | "loading" | "ready" | "blocked"; count: number; errorCode?: string }
   | { type: "selected"; objectId: string; additive: boolean }
   | { type: "elementSelected"; meshId: string; mode: MeshElementMode; index: number; additive: boolean; nonMeshKind?: NonMeshElementKind }
-  | { type: "greasePencilPointSelected"; point: GreasePencilPointRef; additive: boolean }
+  | { type: "greasePencilPointSelected"; point: GreasePencilPointRef; additive: boolean; baseSelectionRevision: number }
+  | { type: "greasePencilSelectionStatus"; selectionRevision: number; pointIds: string[] }
+  | { type: "greasePencilMarqueeSelected"; result: GreasePencilMarqueeResultIR; additive: boolean }
+  | { type: "paintDepthVisibilityResult"; requestId: string; result: PaintDepthVisibilityResultIR }
+  | { type: "paintDepthVisibilityError"; requestId: string; message: string }
   | { type: "curveGizmoScreenFrame"; frame: CurveGizmoScreenFrameIR | null }
   | { type: "error"; message: string };

web/app/src/three-adapter/offscreen-viewport.ts

@@ -11,13 +11,22 @@ import type { NonMeshElementKind } from "./nonmesh";
 import type { GreasePencilPointPreview, GreasePencilPointRef } from "./grease-pencil";
 import type { CurveGizmoFrameIR, CurveGizmoHandleIR } from "../../../protocol/nonmesh-interaction";
 import { cloneNanoVDBViewportAssets, nanoVDBViewportAssetTransferables, type NanoVDBViewportAssetIR } from "../volume/nanovdb-viewport";
+import type { GreasePencilDrawingScopeIR, GreasePencilMarqueeBoxIR, GreasePencilMarqueeResultIR } from "../../../protocol/grease-pencil-marquee";
+import {
+  validatePaintDepthVisibilityRequest,
+  validatePaintDepthVisibilityResult,
+  type PaintDepthVisibilityRequestIR,
+  type PaintDepthVisibilityResultIR,
+} from "../../../protocol/paint-depth-visibility";
 
 export interface ViewportBackend {
   setSnapshot(snapshot: SceneSnapshotIR, geometryBuffers?: MeshGeometryBuffer[], nonMeshGeometryBuffers?: NonMeshGeometryChunk[]): void;
   setTextureAssets(assets: readonly GPUTextureAsset[]): void;
   setVolumeAssets(assets: readonly NanoVDBViewportAssetIR[]): void;
-  setSelection(objectIds: ReadonlySet<string>, elementSelection?: ReadonlyMap<string, ReadonlyMap<NonMeshElementKind, ReadonlySet<number>>>, greasePencilPoints?: readonly GreasePencilPointRef[]): void;
+  setSelection(objectIds: ReadonlySet<string>, elementSelection?: ReadonlyMap<string, ReadonlyMap<NonMeshElementKind, ReadonlySet<number>>>, greasePencilPoints?: readonly GreasePencilPointRef[], greasePencilSelectionRevision?: number): void;
   setInteractionMode(editMode: boolean, selectionMode: MeshElementMode): void;
+  samplePaintVisibility(request: PaintDepthVisibilityRequestIR): Promise<PaintDepthVisibilityResultIR>;
+  selectGreasePencilMarquee(drawing: GreasePencilDrawingScopeIR, box: GreasePencilMarqueeBoxIR, baseRevision: number, baseSelectionRevision: number, additive: boolean): void;
   setCurveHandlePreview(dataId: string, handles: readonly CurveGizmoHandleIR[] | null): void;
   setGreasePencilPointPreview(dataId: string, layerId: string, frame: number, points: readonly GreasePencilPointPreview[] | null): void;
   setCurveGizmoFrame(dataId: string | null, frame: CurveGizmoFrameIR | null): void;
@@ -41,7 +50,8 @@ export function acquireOffscreenViewportRenderer(
   canvas: HTMLCanvasElement,
   onSelect?: (objectId: string, additive: boolean) => void,
   onElementSelect?: (meshId: string, mode: MeshElementMode, index: number, additive: boolean, nonMeshKind?: NonMeshElementKind) => void,
-  onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean) => void,
+  onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void,
+  onGreasePencilMarqueeSelect?: (result: GreasePencilMarqueeResultIR, additive: boolean) => void,
 ): OffscreenViewportRenderer {
   const existing = sharedBackends.get(canvas);
   if (existing) {
@@ -50,7 +60,7 @@ export function acquireOffscreenViewportRenderer(
     existing.references += 1;
     return existing.renderer;
   }
-  const renderer = new OffscreenViewportRenderer(canvas, onSelect, onElementSelect, onGreasePencilPointSelect);
+  const renderer = new OffscreenViewportRenderer(canvas, onSelect, onElementSelect, onGreasePencilPointSelect, onGreasePencilMarqueeSelect);
   sharedBackends.set(canvas, { renderer, references: 1 });
   return renderer;
 }
@@ -73,23 +83,33 @@ export class OffscreenViewportRenderer implements ViewportBackend {
   private readonly resizeObserver: ResizeObserver;
   private readonly onSelect?: (objectId: string, additive: boolean) => void;
   private readonly onElementSelect?: (meshId: string, mode: MeshElementMode, index: number, additive: boolean, nonMeshKind?: NonMeshElementKind) => void;
-  private readonly onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean) => void;
+  private readonly onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void;
+  private readonly onGreasePencilMarqueeSelect?: (result: GreasePencilMarqueeResultIR, additive: boolean) => void;
   private pointer: { id: number; x: number; y: number; moved: boolean } | null = null;
   private lastSnapshot: SceneSnapshotIR | null = null;
   private lastGeometryBuffers: MeshGeometryBuffer[] | null = null;
   private lastNonMeshGeometryBuffers: NonMeshGeometryChunk[] | null = null;
+  private greasePencilSelectionRevision = 0;
+  private paintDepthRequestSequence = 0;
+  private readonly pendingPaintDepth = new Map<string, {
+    request: PaintDepthVisibilityRequestIR;
+    resolve: (result: PaintDepthVisibilityResultIR) => void;
+    reject: (error: Error) => void;
+  }>();
 
   constructor(
     canvas: HTMLCanvasElement,
     onSelect?: (objectId: string, additive: boolean) => void,
     onElementSelect?: (meshId: string, mode: MeshElementMode, index: number, additive: boolean, nonMeshKind?: NonMeshElementKind) => void,
-    onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean) => void,
+    onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void,
+    onGreasePencilMarqueeSelect?: (result: GreasePencilMarqueeResultIR, additive: boolean) => void,
   ) {
     if (!supportsOffscreenViewport(canvas)) throw new Error("OffscreenCanvas viewport is unavailable");
     this.canvas = canvas;
     this.onSelect = onSelect;
     this.onElementSelect = onElementSelect;
     this.onGreasePencilPointSelect = onGreasePencilPointSelect;
+    this.onGreasePencilMarqueeSelect = onGreasePencilMarqueeSelect;
     this.worker = new Worker(new URL("../workers/viewport-render.worker.ts", import.meta.url), { type: "module" });
     this.worker.onmessage = (event: MessageEvent<OffscreenViewportResponse>) => this.handleMessage(event.data);
     const offscreen = canvas.transferControlToOffscreen();
@@ -151,6 +171,11 @@ export class OffscreenViewportRenderer implements ViewportBackend {
       this.canvas.dataset.textureStatus = "none";
       this.canvas.dataset.textureLoaded = "0";
       this.canvas.dataset.textureBytes = "0";
+      this.canvas.dataset.textureBudgetStatus = "ready";
+      this.canvas.dataset.textureBudgetCode = "";
+      this.canvas.dataset.textureBudgetAssets = "0";
+      this.canvas.dataset.textureBudgetPayloadBytes = "0";
+      this.canvas.dataset.textureBudgetGpuBytes = "0";
       return;
     }
     const cloned = assets.map((asset) => ({ ...asset, data: asset.data.slice(0) }));
@@ -163,15 +188,35 @@ export class OffscreenViewportRenderer implements ViewportBackend {
     this.worker.postMessage({ type: "volumeAssets", assets: cloned } satisfies OffscreenViewportRequest, nanoVDBViewportAssetTransferables(cloned));
   }
 
-  setSelection(objectIds: ReadonlySet<string>, elementSelection?: ReadonlyMap<string, ReadonlyMap<NonMeshElementKind, ReadonlySet<number>>>, greasePencilPoints: readonly GreasePencilPointRef[] = []): void {
+  setSelection(objectIds: ReadonlySet<string>, elementSelection?: ReadonlyMap<string, ReadonlyMap<NonMeshElementKind, ReadonlySet<number>>>, greasePencilPoints: readonly GreasePencilPointRef[] = [], greasePencilSelectionRevision = 0): void {
     const elements = [...(elementSelection ?? new Map())].flatMap(([dataId, kinds]) => [...kinds].flatMap(([kind, indices]) => [...indices].map((index) => ({ dataId, kind, index }))));
-    this.worker.postMessage({ type: "selection", objectIds: [...objectIds], elements, greasePencilPoints: [...greasePencilPoints] } satisfies OffscreenViewportRequest);
+    this.greasePencilSelectionRevision = greasePencilSelectionRevision;
+    this.worker.postMessage({ type: "selection", objectIds: [...objectIds], elements, greasePencilPoints: [...greasePencilPoints], greasePencilSelectionRevision } satisfies OffscreenViewportRequest);
   }
 
   setInteractionMode(editMode: boolean, selectionMode: MeshElementMode): void {
     this.worker.postMessage({ type: "interaction", editMode, selectionMode } satisfies OffscreenViewportRequest);
   }
 
+  samplePaintVisibility(requestValue: PaintDepthVisibilityRequestIR): Promise<PaintDepthVisibilityResultIR> {
+    const request = validatePaintDepthVisibilityRequest(requestValue, this.lastSnapshot?.revision ?? -1);
+    const requestId = `paint-depth-${++this.paintDepthRequestSequence}`;
+    return new Promise((resolve, reject) => {
+      this.pendingPaintDepth.set(requestId, { request, resolve, reject });
+      this.worker.postMessage({ type: "paintDepthVisibility", requestId, request } satisfies OffscreenViewportRequest);
+    });
+  }
+
+  selectGreasePencilMarquee(
+    drawing: GreasePencilDrawingScopeIR,
+    box: GreasePencilMarqueeBoxIR,
+    baseRevision: number,
+    baseSelectionRevision: number,
+    additive: boolean,
+  ): void {
+    this.worker.postMessage({ type: "greasePencilMarquee", drawing, box, baseRevision, baseSelectionRevision, additive } satisfies OffscreenViewportRequest);
+  }
+
   setCurveHandlePreview(dataId: string, handles: readonly CurveGizmoHandleIR[] | null): void {
     this.canvas.dataset.curveGizmoPreview = handles ? String(handles.length) : "0";
     this.worker.postMessage({ type: "curveHandlePreview", dataId, handles: handles ? handles.map((handle) => ({ ...handle, position: [...handle.position] as [number, number, number] })) : null } satisfies OffscreenViewportRequest);
@@ -202,7 +247,12 @@ export class OffscreenViewportRenderer implements ViewportBackend {
 
   private pointerDown = (event: PointerEvent): void => {
     this.pointer = { id: event.pointerId, x: event.clientX, y: event.clientY, moved: false };
-    this.canvas.setPointerCapture(event.pointerId);
+    try {
+      this.canvas.setPointerCapture(event.pointerId);
+    }
+    catch {
+      // Synthetic test events and browsers without pointer capture still support picking.
+    }
   };
 
   private pointerMove = (event: PointerEvent): void => {
@@ -221,7 +271,7 @@ export class OffscreenViewportRenderer implements ViewportBackend {
       const bounds = this.canvas.getBoundingClientRect();
       const x = ((event.clientX - bounds.left) / Math.max(1, bounds.width)) * 2 - 1;
       const y = -((event.clientY - bounds.top) / Math.max(1, bounds.height)) * 2 + 1;
-      this.worker.postMessage({ type: "pick", x, y, additive: event.shiftKey || event.ctrlKey || event.metaKey } satisfies OffscreenViewportRequest);
+      this.worker.postMessage({ type: "pick", x, y, additive: event.shiftKey || event.ctrlKey || event.metaKey, baseSelectionRevision: this.greasePencilSelectionRevision } satisfies OffscreenViewportRequest);
     }
     this.pointer = null;
   };
@@ -234,20 +284,69 @@ export class OffscreenViewportRenderer implements ViewportBackend {
   private handleMessage(message: OffscreenViewportResponse): void {
     if (message.type === "selected") this.onSelect?.(message.objectId, message.additive);
     else if (message.type === "elementSelected") this.onElementSelect?.(message.meshId, message.mode, message.index, message.additive, message.nonMeshKind);
-    else if (message.type === "greasePencilPointSelected") this.onGreasePencilPointSelect?.(message.point, message.additive);
-    else if (message.type === "frame") this.canvas.dataset.rendererPixels = String(message.visiblePixels);
+    else if (message.type === "greasePencilPointSelected") this.onGreasePencilPointSelect?.(message.point, message.additive, message.baseSelectionRevision);
+    else if (message.type === "greasePencilSelectionStatus") {
+      this.canvas.dataset.greasePencilSelectionRevision = String(message.selectionRevision);
+      this.canvas.dataset.greasePencilSelectionPointIds = message.pointIds.join(",");
+    }
+    else if (message.type === "greasePencilMarqueeSelected") {
+      this.canvas.dataset.greasePencilMarqueeSelectionRevision = String(message.result.baseSelectionRevision);
+      this.canvas.dataset.greasePencilMarqueeDrawingId = message.result.drawing.drawingId;
+      this.canvas.dataset.greasePencilMarqueePointIds = message.result.selectedPoints.map((point) => point.pointId).join(",");
+      this.canvas.dataset.greasePencilMarqueeStrokeIds = message.result.selectedStrokeIds.join(",");
+      this.canvas.dataset.greasePencilMarqueeCount = String(message.result.selectedPoints.length);
+      this.onGreasePencilMarqueeSelect?.(message.result, message.additive);
+    }
+    else if (message.type === "paintDepthVisibilityResult") {
+      const pending = this.pendingPaintDepth.get(message.requestId);
+      if (!pending) return;
+      this.pendingPaintDepth.delete(message.requestId);
+      try { pending.resolve(validatePaintDepthVisibilityResult(message.result, pending.request)); }
+      catch (error) { pending.reject(error instanceof Error ? error : new Error(String(error))); }
+    }
+    else if (message.type === "paintDepthVisibilityError") {
+      const pending = this.pendingPaintDepth.get(message.requestId);
+      if (!pending) return;
+      this.pendingPaintDepth.delete(message.requestId);
+      pending.reject(new Error(message.message));
+    }
+    else if (message.type === "frame") {
+      this.canvas.dataset.rendererPixels = String(message.visiblePixels);
+      if (message.camera) {
+        this.canvas.dataset.cameraPosition = message.camera.position.map((value) => Number(value.toFixed(6))).join(",");
+        this.canvas.dataset.cameraTarget = message.camera.target.map((value) => Number(value.toFixed(6))).join(",");
+        this.canvas.dataset.cameraYaw = message.camera.yaw.toFixed(6);
+        this.canvas.dataset.cameraPitch = message.camera.pitch.toFixed(6);
+        this.canvas.dataset.cameraDistance = message.camera.distance.toFixed(6);
+      }
+    }
     else if (message.type === "snapshotStatus") {
       this.canvas.dataset.nonMeshCount = String(message.nonMeshCount);
       this.canvas.dataset.nonMeshBlockedCount = String(message.nonMeshBlockedCount);
       this.canvas.dataset.greasePencilCount = String(message.greasePencilCount);
       this.canvas.dataset.greasePencilBlockedCount = String(message.greasePencilBlockedCount);
       this.canvas.dataset.greasePencilOnionStrokeCount = String(message.greasePencilOnionStrokeCount);
+      this.canvas.dataset.renderBudgetBackend = message.renderBudget.backend;
+      this.canvas.dataset.renderBudgetStatus = message.renderBudget.status.toLowerCase();
+      this.canvas.dataset.renderBudgetCode = message.renderBudget.issues[0]?.code ?? "";
+      this.canvas.dataset.renderBudgetLights = String(message.renderBudget.requestedLights);
+      this.canvas.dataset.renderBudgetRenderedLights = String(message.renderBudget.renderedLightNodeIds.length);
+      this.canvas.dataset.renderBudgetDroppedLights = String(message.renderBudget.droppedLightNodeIds.length);
+      this.canvas.dataset.renderBudgetShadows = String(message.renderBudget.requestedShadowMaps);
+      this.canvas.dataset.renderBudgetRenderedShadows = String(message.renderBudget.shadowLightNodeIds.length);
+      this.canvas.dataset.renderBudgetBlockedShadows = String(message.renderBudget.shadowBlockedLightNodeIds.length);
+      this.canvas.dataset.renderBudgetShadowMapDimension = String(message.renderBudget.budget.shadowMapDimension);
     }
     else if (message.type === "textureStatus") {
       this.canvas.dataset.textureStatus = message.rejected > 0 ? "blocked" : "ready";
       this.canvas.dataset.textureLoaded = String(message.loaded);
       this.canvas.dataset.textureBytes = String(message.bytes);
       this.canvas.dataset.textureErrorCode = message.errorCodes[0] ?? "";
+      this.canvas.dataset.textureBudgetStatus = message.budget.status.toLowerCase();
+      this.canvas.dataset.textureBudgetCode = message.budget.issues[0]?.code ?? "";
+      this.canvas.dataset.textureBudgetAssets = String(message.budget.requestedAssets);
+      this.canvas.dataset.textureBudgetPayloadBytes = String(message.budget.payloadBytes);
+      this.canvas.dataset.textureBudgetGpuBytes = String(message.budget.decodedGPUBytes);
     }
     else if (message.type === "volumeStatus") {
       this.canvas.dataset.volumeStatus = message.status;
@@ -269,5 +368,7 @@ export class OffscreenViewportRenderer implements ViewportBackend {
     this.canvas.removeEventListener("wheel", this.wheel);
     this.worker.postMessage({ type: "dispose" } satisfies OffscreenViewportRequest);
     this.worker.terminate();
+    for (const pending of this.pendingPaintDepth.values()) pending.reject(new Error("PAINT_DEPTH_UNAVAILABLE: Offscreen viewport disposed"));
+    this.pendingPaintDepth.clear();
   }
 }

web/app/src/three-adapter/paint-depth-visibility.ts

@@ -0,0 +1,171 @@
+import {
+  Color,
+  InstancedMesh,
+  Matrix4,
+  MeshDepthMaterial,
+  RGBADepthPacking,
+  Vector2,
+  Vector3,
+  WebGLRenderTarget,
+  type Object3D,
+  type PerspectiveCamera,
+  type Scene,
+  type WebGLRenderer,
+} from "../vendor/three/three.module.js";
+import {
+  PAINT_DEPTH_VISIBILITY_BUDGET,
+  PaintDepthVisibilityError,
+  validatePaintDepthVisibilityResult,
+  type PaintDepthVisibilityBackend,
+  type PaintDepthVisibilityRequestIR,
+  type PaintDepthVisibilityResultIR,
+} from "../../../protocol/paint-depth-visibility";
+
+const PACK_DOWNSCALE = 255 / 256;
+const DEPTH_TOLERANCE_RATIO = 0.002;
+
+function unpackRGBADepth(pixels: Uint8Array, offset: number): number {
+  return (pixels[offset] / 255) * PACK_DOWNSCALE +
+    (pixels[offset + 1] / 255) * (PACK_DOWNSCALE / 256) +
+    (pixels[offset + 2] / 255) * (PACK_DOWNSCALE / 65_536) +
+    (pixels[offset + 3] / 255) / 16_777_216;
+}
+
+function perspectiveDistance(depth: number, near: number, far: number): number {
+  const viewZ = (near * far) / ((far - near) * depth - far);
+  return -viewZ;
+}
+
+function depthTargetSize(renderer: WebGLRenderer): { width: number; height: number } {
+  const size = renderer.getDrawingBufferSize(new Vector2());
+  const sourceWidth = Math.max(1, Math.floor(size.x));
+  const sourceHeight = Math.max(1, Math.floor(size.y));
+  const maxPixels = Math.floor(PAINT_DEPTH_VISIBILITY_BUDGET.maxReadbackBytes / 4);
+  const scale = Math.min(
+    1,
+    PAINT_DEPTH_VISIBILITY_BUDGET.maxDimension / sourceWidth,
+    PAINT_DEPTH_VISIBILITY_BUDGET.maxDimension / sourceHeight,
+    Math.sqrt(maxPixels / (sourceWidth * sourceHeight)),
+  );
+  return {
+    width: Math.max(1, Math.floor(sourceWidth * scale)),
+    height: Math.max(1, Math.floor(sourceHeight * scale)),
+  };
+}
+
+function candidateMatrix(object: Object3D, objectId: string): Matrix4 {
+  object.updateWorldMatrix(true, false);
+  if (!(object instanceof InstancedMesh)) return object.matrixWorld.clone();
+  const instanceIds = object.userData.instanceNodeIds as string[] | undefined;
+  const instanceIndex = instanceIds?.indexOf(objectId) ?? -1;
+  if (instanceIndex < 0) throw new PaintDepthVisibilityError("PAINT_DEPTH_UNAVAILABLE", "Paint object instance is unavailable");
+  const instance = new Matrix4();
+  object.getMatrixAt(instanceIndex, instance);
+  return new Matrix4().multiplyMatrices(object.matrixWorld, instance);
+}
+
+export function samplePaintDepthVisibilityGPU({
+  renderer,
+  scene,
+  camera,
+  object,
+  request,
+  backend,
+}: {
+  renderer: WebGLRenderer;
+  scene: Scene;
+  camera: PerspectiveCamera;
+  object: Object3D;
+  request: PaintDepthVisibilityRequestIR;
+  backend: PaintDepthVisibilityBackend;
+}): PaintDepthVisibilityResultIR {
+  if (!renderer.capabilities.isWebGL2) throw new PaintDepthVisibilityError("PAINT_DEPTH_UNAVAILABLE", "Paint visibility requires a WebGL2 depth pass");
+  const meshId = object.userData.meshId;
+  const sourcePositions = object.userData.sourcePositions as number[] | undefined;
+  if (meshId !== request.meshId || !sourcePositions || sourcePositions.length % 3 !== 0) {
+    throw new PaintDepthVisibilityError("PAINT_DEPTH_UNAVAILABLE", "Paint source geometry is unavailable");
+  }
+  const vertexCount = sourcePositions.length / 3;
+  if (request.vertexIndices.some((index) => index >= vertexCount)) {
+    throw new PaintDepthVisibilityError("PAINT_SCHEMA_INVALID", "Paint depth request references an unknown vertex");
+  }
+
+  const { width, height } = depthTargetSize(renderer);
+  const target = new WebGLRenderTarget(width, height, { depthBuffer: true, stencilBuffer: false });
+  const depthMaterial = new MeshDepthMaterial({ depthPacking: RGBADepthPacking });
+  const pixels = new Uint8Array(width * height * 4);
+  const previousTarget = renderer.getRenderTarget();
+  const previousOverride = scene.overrideMaterial;
+  const previousBackground = scene.background;
+  const previousClearColor = renderer.getClearColor(new Color()).clone();
+  const previousClearAlpha = renderer.getClearAlpha();
+  const hidden: Object3D[] = [];
+
+  try {
+    scene.traverse((candidate) => {
+      const renderable = candidate as Object3D & { isLine?: boolean; isPoints?: boolean; isSprite?: boolean };
+      if (candidate.visible && (renderable.isLine || renderable.isPoints || renderable.isSprite || candidate.userData.nanoVDBVolume)) {
+        hidden.push(candidate);
+        candidate.visible = false;
+      }
+    });
+    scene.background = null;
+    scene.overrideMaterial = depthMaterial;
+    renderer.setClearColor(0xffffff, 1);
+    renderer.setRenderTarget(target);
+    renderer.clear(true, true, true);
+    camera.updateMatrixWorld(true);
+    scene.updateMatrixWorld(true);
+    renderer.render(scene, camera);
+    renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
+  }
+  catch (error) {
+    throw new PaintDepthVisibilityError("PAINT_DEPTH_UNAVAILABLE", error instanceof Error ? error.message : "GPU depth readback failed");
+  }
+  finally {
+    renderer.setRenderTarget(previousTarget);
+    renderer.setClearColor(previousClearColor, previousClearAlpha);
+    scene.overrideMaterial = previousOverride;
+    scene.background = previousBackground;
+    for (const candidate of hidden) candidate.visible = true;
+    depthMaterial.dispose();
+    target.dispose();
+  }
+
+  let occluderPixelCount = 0;
+  for (let offset = 0; offset < pixels.length; offset += 4) {
+    if (unpackRGBADepth(pixels, offset) < 1 - 1e-7) occluderPixelCount++;
+  }
+  if (occluderPixelCount === 0) throw new PaintDepthVisibilityError("PAINT_DEPTH_UNAVAILABLE", "GPU depth pass produced no mesh coverage");
+
+  const modelMatrix = candidateMatrix(object, request.objectId);
+  const visibleVertexIndices: number[] = [];
+  for (const index of request.vertexIndices) {
+    const sourceOffset = index * 3;
+    const world = new Vector3(
+      sourcePositions[sourceOffset],
+      sourcePositions[sourceOffset + 2],
+      -sourcePositions[sourceOffset + 1],
+    ).applyMatrix4(modelMatrix);
+    const viewDistance = -world.clone().applyMatrix4(camera.matrixWorldInverse).z;
+    const projected = world.project(camera);
+    if (projected.x < -1 || projected.x > 1 || projected.y < -1 || projected.y > 1 || projected.z < -1 || projected.z > 1 || viewDistance <= 0) continue;
+    const x = Math.max(0, Math.min(width - 1, Math.round((projected.x * 0.5 + 0.5) * (width - 1))));
+    const y = Math.max(0, Math.min(height - 1, Math.round((projected.y * 0.5 + 0.5) * (height - 1))));
+    const depth = unpackRGBADepth(pixels, (y * width + x) * 4);
+    const sampledDistance = perspectiveDistance(depth, camera.near, camera.far);
+    const tolerance = Math.max(1e-4, viewDistance * DEPTH_TOLERANCE_RATIO);
+    if (viewDistance <= sampledDistance + tolerance) visibleVertexIndices.push(index);
+  }
+
+  return validatePaintDepthVisibilityResult({
+    ...request,
+    backend,
+    source: "GPU_RGBA_DEPTH_READBACK",
+    width,
+    height,
+    depthReadbackBytes: pixels.byteLength,
+    occluderPixelCount,
+    visibleVertexIndices,
+  }, request);
+}

web/app/src/three-adapter/pbr.ts

@@ -6,6 +6,7 @@ import {
   MeshPhysicalMaterial,
   Object3D,
   PCFShadowMap,
+  PerspectiveCamera,
   PointLight,
   RectAreaLight,
   SRGBColorSpace,
@@ -14,11 +15,14 @@ import {
   type Light,
   type WebGLRenderer,
 } from "../vendor/three/three.module.js";
-import type { LightIR, MaterialIR, SceneNodeIR } from "../../../protocol/scene-ir";
+import type { CameraIR, LightIR, MaterialIR, SceneNodeIR } from "../../../protocol/scene-ir";
+import { compileMaterialGraph, type ShaderCompileContext, type ShaderCompileReport } from "../../../protocol/shader-compiler";
+import { PBR_RENDER_BUDGETS } from "../../../protocol/render-budget";
 
 export const PBR_PROFILE = "physical-v1";
 export const PBR_TONE_MAPPING = "aces";
 export const PBR_SHADOW_PROFILE = "pcf-1024";
+export const PBR_SHADOW_MAP_DIMENSION = PBR_RENDER_BUDGETS.THREE_WEBGL2.shadowMapDimension;
 
 function clamp(value: number | undefined, minimum: number, maximum: number, fallback: number): number {
   return Number.isFinite(value) ? Math.min(maximum, Math.max(minimum, value as number)) : fallback;
@@ -32,23 +36,36 @@ export function configurePBRRenderer(renderer: WebGLRenderer, exposure = 0): voi
   renderer.shadowMap.type = PCFShadowMap;
 }
 
-export function createPBRMaterial(definition?: MaterialIR, active = false): MeshPhysicalMaterial {
-  const baseColor = definition?.baseColor ?? (active ? [0.83, 0.48, 0.29, 1] : [0.55, 0.62, 0.69, 1]);
-  const emission = definition?.emissionColor ?? [0, 0, 0, 1];
-  const alpha = clamp(definition?.alpha ?? baseColor[3], 0, 1, 1);
-  const transmission = clamp(definition?.transmissionWeight, 0, 1, 0);
+export function configurePBRCamera(camera: PerspectiveCamera, definition: CameraIR): void {
+  const sensor = definition.sensorFit === 2 ? definition.sensorHeightMm : definition.sensorWidthMm;
+  const fov = (2 * Math.atan((sensor / Math.max(0.001, definition.lensMm)) / 2) * 180) / Math.PI;
+  camera.fov = definition.projection === "ORTHOGRAPHIC" ? 45 : fov;
+  camera.near = Math.max(0.0001, definition.near);
+  camera.far = Math.max(camera.near + 0.001, definition.far);
+  camera.filmGauge = sensor;
+  camera.filmOffset = definition.shift[0] * sensor;
+  camera.updateProjectionMatrix();
+}
+
+export function createPBRMaterial(definition?: MaterialIR, active = false, shaderContext: ShaderCompileContext = {}): MeshPhysicalMaterial {
+  const compileReport = definition?.nodes?.length ? compileMaterialGraph(definition, shaderContext) : undefined;
+  const compiled = compileReport?.status === "COMPILED" ? compileReport.material : undefined;
+  const baseColor = compiled?.baseColor ?? definition?.baseColor ?? (active ? [0.83, 0.48, 0.29, 1] : [0.55, 0.62, 0.69, 1]);
+  const emission = compiled?.emissionColor ?? definition?.emissionColor ?? [0, 0, 0, 1];
+  const alpha = clamp(compiled?.alpha ?? definition?.alpha ?? baseColor[3], 0, 1, 1);
+  const transmission = clamp(compiled?.transmissionWeight ?? definition?.transmissionWeight, 0, 1, 0);
   const material = new MeshPhysicalMaterial({
     color: new Color().setRGB(baseColor[0], baseColor[1], baseColor[2]),
-    roughness: clamp(definition?.roughness, 0, 1, 0.45),
-    metalness: clamp(definition?.metallic, 0, 1, 0.05),
-    ior: clamp(definition?.ior, 1, 2.333, 1.45),
+    roughness: clamp(compiled?.roughness ?? definition?.roughness, 0, 1, 0.45),
+    metalness: clamp(compiled?.metallic ?? definition?.metallic, 0, 1, 0.05),
+    ior: clamp(compiled?.ior ?? definition?.ior, 1, 2.333, 1.45),
     // Blender's neutral Specular IOR Level is 0.5; Three's neutral multiplier is 1.0.
-    specularIntensity: clamp((definition?.specularIORLevel ?? 0.5) * 2, 0, 1, 1),
-    clearcoat: clamp(definition?.coatWeight, 0, 1, 0),
-    clearcoatRoughness: clamp(definition?.coatRoughness, 0, 1, 0.03),
+    specularIntensity: clamp((compiled?.specularIORLevel ?? definition?.specularIORLevel ?? 0.5) * 2, 0, 1, 1),
+    clearcoat: clamp(compiled?.coatWeight ?? definition?.coatWeight, 0, 1, 0),
+    clearcoatRoughness: clamp(compiled?.coatRoughness ?? definition?.coatRoughness, 0, 1, 0.03),
     transmission,
     emissive: new Color().setRGB(emission[0], emission[1], emission[2]),
-    emissiveIntensity: clamp(definition?.emissionStrength, 0, 1_000_000, 1),
+    emissiveIntensity: clamp(compiled?.emissionStrength ?? definition?.emissionStrength, 0, 1_000_000, 1),
     opacity: alpha,
     transparent: alpha < 0.999,
     depthWrite: alpha >= 0.999,
@@ -58,9 +75,47 @@ export function createPBRMaterial(definition?: MaterialIR, active = false): Mesh
   material.userData.baseEmissive = material.emissive.getHex();
   material.userData.baseEmissiveIntensity = material.emissiveIntensity;
   material.userData.pbrProfile = PBR_PROFILE;
+  if (compileReport) {
+    material.userData.shaderCompile = compileReport as ShaderCompileReport;
+    material.userData.shaderCompileTextures = compileReport.status === "COMPILED" ? compileReport.textureBindings : [];
+  }
   return material;
 }
 
+export interface PBRMaterialPipelineUpdate {
+  material: MeshPhysicalMaterial;
+  report?: ShaderCompileReport;
+  replaced: boolean;
+}
+
+/** Keeps the last compiled material alive when a later graph fails closed. */
+export class PBRMaterialPipeline {
+  private current: MeshPhysicalMaterial | null = null;
+
+  get material(): MeshPhysicalMaterial | null {
+    return this.current;
+  }
+
+  update(definition?: MaterialIR, active = false, shaderContext: ShaderCompileContext = {}): PBRMaterialPipelineUpdate {
+    const candidate = createPBRMaterial(definition, active, shaderContext);
+    const report = candidate.userData.shaderCompile as ShaderCompileReport | undefined;
+    if (report?.status === "BLOCKED" && this.current) {
+      this.current.userData.shaderCompileFailure = report;
+      candidate.dispose();
+      return { material: this.current, report, replaced: false };
+    }
+    const previous = this.current;
+    this.current = candidate;
+    previous?.dispose();
+    return { material: candidate, report, replaced: previous !== null };
+  }
+
+  dispose(): void {
+    this.current?.dispose();
+    this.current = null;
+  }
+}
+
 export function setPBRMaterialSelected(material: MeshPhysicalMaterial, selected: boolean): void {
   const baseEmissive = typeof material.userData.baseEmissive === "number" ? material.userData.baseEmissive : 0;
   const baseIntensity = typeof material.userData.baseEmissiveIntensity === "number" ? material.userData.baseEmissiveIntensity : 1;
@@ -108,7 +163,12 @@ export function createPBRLight(definition: LightIR): Light {
   return light;
 }
 
-export function configurePBRLight(light: Light, node: SceneNodeIR, parent: Object3D): void {
+export function configurePBRLight(
+  light: Light,
+  node: SceneNodeIR,
+  parent: Object3D,
+  options: { shadowEnabled?: boolean; shadowMapDimension?: number } = {},
+): void {
   const [x, y, z] = node.transform.translation;
   light.position.set(x, z, -y);
   light.rotation.set(node.transform.rotationEuler[0], node.transform.rotationEuler[2], -node.transform.rotationEuler[1]);
@@ -117,13 +177,15 @@ export function configurePBRLight(light: Light, node: SceneNodeIR, parent: Objec
     light.decay = 2;
   }
   light.userData.blenderCastsShadow = definitionCastsShadow(light);
-  if ((light instanceof DirectionalLight || light instanceof SpotLight) && light.userData.blenderCastsShadow) {
+  const shadowEnabled = options.shadowEnabled ?? light.userData.blenderCastsShadow;
+  light.userData.pbrShadowBudgetBlocked = light.userData.blenderCastsShadow && !shadowEnabled;
+  if ((light instanceof DirectionalLight || light instanceof SpotLight) && shadowEnabled) {
     const target = new Object3D();
     const forward = new Vector3(0, -1, 0).applyEuler(light.rotation);
     target.position.copy(light.position).add(forward);
     light.target = target;
     light.castShadow = true;
-    light.shadow.mapSize.set(1024, 1024);
+    light.shadow.mapSize.set(options.shadowMapDimension ?? PBR_SHADOW_MAP_DIMENSION, options.shadowMapDimension ?? PBR_SHADOW_MAP_DIMENSION);
     light.shadow.bias = -0.0005;
     light.shadow.normalBias = 0.03;
     light.shadow.camera.near = 0.05;
@@ -136,9 +198,9 @@ export function configurePBRLight(light: Light, node: SceneNodeIR, parent: Objec
     }
     parent.add(target);
   }
-  else if (light instanceof PointLight && light.userData.blenderCastsShadow) {
+  else if (light instanceof PointLight && shadowEnabled) {
     light.castShadow = true;
-    light.shadow.mapSize.set(1024, 1024);
+    light.shadow.mapSize.set(options.shadowMapDimension ?? PBR_SHADOW_MAP_DIMENSION, options.shadowMapDimension ?? PBR_SHADOW_MAP_DIMENSION);
     light.shadow.bias = -0.0005;
     light.shadow.normalBias = 0.03;
     light.shadow.camera.near = 0.05;

web/app/src/three-adapter/render-image-comparison.ts

@@ -0,0 +1,9 @@
+export {
+  compareRenderImages,
+  MAX_RENDER_COMPARISON_DIMENSION,
+  MAX_RENDER_COMPARISON_PIXELS,
+  RENDER_IMAGE_COMPARISON_SCHEMA_VERSION,
+  RENDER_REFERENCE_MISMATCH_CODE,
+  type RenderImageComparisonIR,
+  type RenderImageComparisonThresholdsIR,
+} from "../../../protocol/render-image-comparison";

web/app/src/three-adapter/render-routing.ts

@@ -0,0 +1,10 @@
+export {
+  routeRenderExecution,
+  RENDER_ROUTING_SCHEMA_VERSION,
+  type RenderRoutingBackend,
+  type RenderRoutingContextIR,
+  type RenderRoutingEngine,
+  type RenderRoutingRequestIR,
+  type RenderRoutingResultIR,
+  type RenderRoutingTarget,
+} from "../../../protocol/render-routing";

web/app/src/three-adapter/texture-assets.ts

@@ -11,6 +11,7 @@ import {
 } from "../vendor/three/three.module.js";
 import type { MaterialIR, SceneSnapshotIR, WorldIR } from "../../../protocol/scene-ir";
 import { RenderAssetValidationError, validateGPUTextureAsset, type GPUTextureAsset, type GPUTextureColorSpace, type GPUTextureUsage } from "../../../protocol/render-assets";
+import { planPBRTextureBudget, type PBRDeviceLimits, type PBRRenderBackend, type PBRTextureBudgetReport } from "../../../protocol/render-budget";
 
 export interface TextureUploadStatus {
   loaded: number;
@@ -18,6 +19,7 @@ export interface TextureUploadStatus {
   bytes: number;
   errors: string[];
   errorCodes: string[];
+  budget: PBRTextureBudgetReport;
 }
 
 function key(imageId: string, usage: GPUTextureUsage): string {
@@ -48,6 +50,11 @@ export class GPUTextureStore {
   private readonly udimTileCounts = new Map<string, number>();
   private revision = 0;
 
+  constructor(
+    private readonly backend: PBRRenderBackend = "THREE_WEBGL2",
+    private readonly deviceLimits: PBRDeviceLimits = {},
+  ) {}
+
   getRevision(): number {
     return this.revision;
   }
@@ -61,7 +68,20 @@ export class GPUTextureStore {
   }
 
   async upload(assets: readonly GPUTextureAsset[]): Promise<TextureUploadStatus> {
-    const status: TextureUploadStatus = { loaded: 0, rejected: 0, bytes: 0, errors: [], errorCodes: [] };
+    const candidate = new Map(this.assets);
+    for (const asset of assets) {
+      const lookupUsage = asset.usage === "UDIM_TILE" ? "BASE_COLOR" : asset.usage;
+      candidate.set(key(asset.imageId, lookupUsage), asset);
+    }
+    const budget = planPBRTextureBudget([...candidate.values()], this.backend, this.deviceLimits);
+    const status: TextureUploadStatus = { loaded: 0, rejected: 0, bytes: 0, errors: [], errorCodes: [], budget };
+    if (budget.status === "BLOCKED") {
+      status.rejected = assets.length;
+      status.errors.push(...budget.issues.map((issue) => issue.message));
+      status.errorCodes.push(...budget.issues.map((issue) => issue.code));
+      this.revision += 1;
+      return status;
+    }
     const incomingUDIMCounts = new Map<string, number>();
     for (const asset of assets) if (asset.usage === "UDIM_TILE") incomingUDIMCounts.set(asset.imageId, (incomingUDIMCounts.get(asset.imageId) ?? 0) + 1);
     for (const asset of assets) {
@@ -92,12 +112,17 @@ export class GPUTextureStore {
 
   applyMaterial(material: MeshPhysicalMaterial, definition?: MaterialIR): void {
     if (!definition) return;
-    const baseImageId = definition.imageIds?.find((imageId) => imageId !== definition.normalImageId) ?? definition.nodes?.find((node) => node.type === "IMAGE_TEXTURE" && node.imageId && node.imageId !== definition.normalImageId)?.imageId;
+    const compile = material.userData.shaderCompile as { status?: string; textureBindings?: Array<{ imageId: string; usage: "BASE_COLOR" | "NORMAL" }> } | undefined;
+    const baseImageId = compile?.status === "COMPILED"
+      ? compile.textureBindings?.find((binding) => binding.usage === "BASE_COLOR")?.imageId
+      : definition.imageIds?.find((imageId) => imageId !== definition.normalImageId) ?? definition.nodes?.find((node) => node.type === "IMAGE_TEXTURE" && node.imageId && node.imageId !== definition.normalImageId)?.imageId;
     if (baseImageId) {
       const texture = this.get(baseImageId, "BASE_COLOR");
       if (texture) material.map = texture;
     }
-    const normalImageId = definition.normalImageId ?? definition.nodes?.find((node) => node.type === "IMAGE_TEXTURE" && node.imageId)?.imageId;
+    const normalImageId = compile?.status === "COMPILED"
+      ? compile.textureBindings?.find((binding) => binding.usage === "NORMAL")?.imageId
+      : definition.normalImageId ?? definition.nodes?.find((node) => node.type === "IMAGE_TEXTURE" && node.imageId)?.imageId;
     if (normalImageId) {
       const texture = this.get(normalImageId, "NORMAL");
       if (texture) material.normalMap = texture;

web/app/src/three-adapter/viewport.ts

@@ -30,10 +30,12 @@ import type { MeshElementMode, MeshGeometryBuffer, WebEngineLODLevelResult } fro
 import { ThreeLODAdapter, type ThreeLODLevel, type LODSelectionResult } from "./lod";
 import {
   configurePBRLight,
+  configurePBRCamera,
   configurePBRRenderer,
   createPBRLight,
   createPBRMaterial,
   PBR_PROFILE,
+  PBR_SHADOW_MAP_DIMENSION,
   PBR_SHADOW_PROFILE,
   PBR_TONE_MAPPING,
   setPBRMaterialSelected,
@@ -41,6 +43,7 @@ import {
 import { GPUTextureStore } from "./texture-assets";
 import type { GPUTextureAsset } from "../../../protocol/render-assets";
 import { gateEnvironmentImage, gateUDIMImage } from "../../../protocol/render-assets";
+import { planPBRLightingBudget, type PBRLightingBudgetReport } from "../../../protocol/render-budget";
 import { applyCurveHandlePreview, applyNonMeshElementSelection, applyNonMeshTransform, createNonMeshObject, type NonMeshElementKind } from "./nonmesh";
 import type { NonMeshGeometryChunk } from "../../../protocol/nonmesh-binary";
 import type { CurveGizmoFrameIR, CurveGizmoHandleIR, CurveGizmoScreenFrameIR } from "../../../protocol/nonmesh-interaction";
@@ -52,10 +55,20 @@ import {
   applyGreasePencilPointPreview,
   applyGreasePencilTransform,
   createGreasePencilObject,
+  greasePencilMarqueeCandidates,
   greasePencilPointRef,
   type GreasePencilPointRef,
   type GreasePencilPointPreview,
 } from "./grease-pencil";
+import {
+  selectGreasePencilMarquee,
+  type GreasePencilDrawingScopeIR,
+  type GreasePencilMarqueeBoxIR,
+  type GreasePencilMarqueeResultIR,
+} from "../../../protocol/grease-pencil-marquee";
+import { VIEWPORT_DEFAULT_ORBIT, VIEWPORT_ORBIT_MAX_DISTANCE, VIEWPORT_ORBIT_MIN_DISTANCE, VIEWPORT_ORBIT_ROTATE_SENSITIVITY, VIEWPORT_ORBIT_ZOOM_SENSITIVITY, orbitPosition, orbitStateFromPosition } from "../../../protocol/viewport-camera";
+import { validatePaintDepthVisibilityRequest, type PaintDepthVisibilityRequestIR, type PaintDepthVisibilityResultIR } from "../../../protocol/paint-depth-visibility";
+import { samplePaintDepthVisibilityGPU } from "./paint-depth-visibility";
 
 export function collectMeshInstanceGroups(snapshot: SceneSnapshotIR, minimumSize = 2): Map<string, string[]> {
   const groups = new Map<string, string[]>();
@@ -86,14 +99,16 @@ export class ViewportRenderer {
   private readonly objectByBlenderId = new Map<string, Object3D>();
   private readonly instanceIndexByBlenderId = new Map<string, number>();
   private readonly lodAdapter = new ThreeLODAdapter();
-  private readonly textureStore = new GPUTextureStore();
+  private readonly textureStore = new GPUTextureStore("THREE_WEBGL2");
   private readonly raycaster = new Raycaster();
   private readonly pointer = new Vector2();
   private readonly onSelect?: (objectId: string, additive: boolean) => void;
   private readonly onElementSelect?: (meshId: string, mode: MeshElementMode, index: number, additive: boolean, nonMeshKind?: NonMeshElementKind) => void;
-  private readonly onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean) => void;
+  private readonly onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void;
+  private readonly onGreasePencilMarqueeSelect?: (result: GreasePencilMarqueeResultIR, additive: boolean) => void;
   private editMode = false;
   private selectionMode: MeshElementMode = "FACE";
+  private greasePencilSelectionRevision = 0;
   private curveGizmoFrame: { dataId: string; frame: CurveGizmoFrameIR } | null = null;
   private curveGizmoScreenFrame = "";
   private volumeAssets: NanoVDBViewportAssetIR[] = [];
@@ -105,12 +120,14 @@ export class ViewportRenderer {
     canvas: HTMLCanvasElement,
     onSelect?: (objectId: string, additive: boolean) => void,
     onElementSelect?: (meshId: string, mode: MeshElementMode, index: number, additive: boolean, nonMeshKind?: NonMeshElementKind) => void,
-    onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean) => void,
+    onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void,
+    onGreasePencilMarqueeSelect?: (result: GreasePencilMarqueeResultIR, additive: boolean) => void,
   ) {
     this.canvas = canvas;
     this.onSelect = onSelect;
     this.onElementSelect = onElementSelect;
     this.onGreasePencilPointSelect = onGreasePencilPointSelect;
+    this.onGreasePencilMarqueeSelect = onGreasePencilMarqueeSelect;
     this.volumeRenderSession = new NanoVDBViewportRenderSession(() => {
       if (this.disposed) return;
       this.volumeRenderCache.clear();
@@ -129,16 +146,20 @@ export class ViewportRenderer {
     this.canvas.dataset.deviceStatus = "ready";
     this.scene = new Scene();
     this.camera = new PerspectiveCamera(45, 1, 0.01, 1000);
-    this.camera.position.set(4.5, -4.5, 3.5);
+    this.camera.position.set(...orbitPosition(VIEWPORT_DEFAULT_ORBIT));
     this.controls = new OrbitControls(this.camera, canvas);
-    this.controls.target.set(0, 0, 0);
-    this.controls.enableDamping = true;
+    this.controls.target.set(...VIEWPORT_DEFAULT_ORBIT.target);
+    this.controls.enableDamping = false;
+    this.controls.enablePan = false;
+    this.controls.minDistance = VIEWPORT_ORBIT_MIN_DISTANCE;
+    this.controls.maxDistance = VIEWPORT_ORBIT_MAX_DISTANCE;
+    this.controls.zoomSpeed = VIEWPORT_ORBIT_ZOOM_SENSITIVITY / (0.01 * -Math.log(0.95));
 
     this.scene.add(new HemisphereLight(0xf2f5ff, 0x3a4149, 0.55));
     const keyLight = new DirectionalLight(0xffffff, 2.5);
     keyLight.position.set(4, -5, 8);
     keyLight.castShadow = true;
-    keyLight.shadow.mapSize.set(1024, 1024);
+    keyLight.shadow.mapSize.set(PBR_SHADOW_MAP_DIMENSION, PBR_SHADOW_MAP_DIMENSION);
     keyLight.shadow.bias = -0.0005;
     keyLight.shadow.normalBias = 0.03;
     this.scene.add(keyLight, keyLight.target);
@@ -152,6 +173,8 @@ export class ViewportRenderer {
     this.canvas.addEventListener("webglcontextlost", this.handleContextLost);
     this.canvas.addEventListener("webglcontextrestored", this.handleContextRestored);
     this.resize();
+    this.controls.update();
+    this.publishCameraState();
     this.renderLoop();
   }
 
@@ -176,7 +199,9 @@ export class ViewportRenderer {
     this.clearImportedScene();
     this.applyWorld(snapshot);
     this.applyCamera(snapshot);
-    this.populateLights(snapshot);
+    const lightingBudget = planPBRLightingBudget(snapshot, "THREE_WEBGL2");
+    this.publishLightingBudget(lightingBudget);
+    this.populateLights(snapshot, lightingBudget);
     this.populateNonMesh(snapshot, nonMeshGeometryBuffers);
     this.populateGreasePencils(snapshot);
     const meshes = new Map(snapshot.meshes.map((mesh) => [mesh.id, mesh]));
@@ -340,6 +365,11 @@ export class ViewportRenderer {
       this.canvas.dataset.textureStatus = "none";
       this.canvas.dataset.textureLoaded = "0";
       this.canvas.dataset.textureBytes = "0";
+      this.canvas.dataset.textureBudgetStatus = "ready";
+      this.canvas.dataset.textureBudgetCode = "";
+      this.canvas.dataset.textureBudgetAssets = "0";
+      this.canvas.dataset.textureBudgetPayloadBytes = "0";
+      this.canvas.dataset.textureBudgetGpuBytes = "0";
       return;
     }
     void this.textureStore.upload(assets).then((status) => {
@@ -347,6 +377,11 @@ export class ViewportRenderer {
       this.canvas.dataset.textureLoaded = String(status.loaded);
       this.canvas.dataset.textureBytes = String(status.bytes);
       this.canvas.dataset.textureErrorCode = status.errorCodes[0] ?? "";
+      this.canvas.dataset.textureBudgetStatus = status.budget.status.toLowerCase();
+      this.canvas.dataset.textureBudgetCode = status.budget.issues[0]?.code ?? "";
+      this.canvas.dataset.textureBudgetAssets = String(status.budget.requestedAssets);
+      this.canvas.dataset.textureBudgetPayloadBytes = String(status.budget.payloadBytes);
+      this.canvas.dataset.textureBudgetGpuBytes = String(status.budget.decodedGPUBytes);
       if (!this.currentSnapshot) return;
       this.textureStore.applySnapshotMaterials(this.importedRoot, this.currentSnapshot);
       const worldId = this.currentSnapshot.scenes[0]?.worldId;
@@ -449,6 +484,7 @@ export class ViewportRenderer {
     objectIds: ReadonlySet<string>,
     elementSelection?: ReadonlyMap<string, ReadonlyMap<NonMeshElementKind, ReadonlySet<number>>>,
     greasePencilPoints: readonly GreasePencilPointRef[] = [],
+    greasePencilSelectionRevision = 0,
   ): void {
     const visitedInstances = new Set<InstancedMesh>();
     for (const [objectId, object] of this.objectByBlenderId) {
@@ -471,6 +507,9 @@ export class ViewportRenderer {
     }
     applyNonMeshElementSelection(this.importedRoot, elementSelection ?? new Map());
     applyGreasePencilPointSelection(this.importedRoot, greasePencilPoints);
+    this.greasePencilSelectionRevision = greasePencilSelectionRevision;
+    this.canvas.dataset.greasePencilSelectionRevision = String(greasePencilSelectionRevision);
+    this.canvas.dataset.greasePencilSelectionPointIds = greasePencilPoints.map((point) => point.pointId).join(",");
   }
 
   setInteractionMode(editMode: boolean, selectionMode: MeshElementMode): void {
@@ -481,6 +520,44 @@ export class ViewportRenderer {
     });
   }
 
+  async samplePaintVisibility(requestValue: PaintDepthVisibilityRequestIR): Promise<PaintDepthVisibilityResultIR> {
+    const request = validatePaintDepthVisibilityRequest(requestValue, this.currentSnapshot?.revision ?? -1);
+    const node = this.currentSnapshot?.nodes.find((candidate) => candidate.id === request.objectId && candidate.dataId === request.meshId && candidate.visible);
+    const object = node ? this.objectByBlenderId.get(node.id) : undefined;
+    if (!object) throw new Error("PAINT_DEPTH_UNAVAILABLE: Paint object is not available in the current viewport");
+    return samplePaintDepthVisibilityGPU({
+      renderer: this.renderer,
+      scene: this.scene,
+      camera: this.camera,
+      object,
+      request,
+      backend: "MAIN_THREAD_WEBGL2",
+    });
+  }
+
+  selectGreasePencilMarquee(
+    drawing: GreasePencilDrawingScopeIR,
+    box: GreasePencilMarqueeBoxIR,
+    baseRevision: number,
+    baseSelectionRevision: number,
+    additive: boolean,
+  ): void {
+    const result = selectGreasePencilMarquee({
+      schemaVersion: 1,
+      baseRevision,
+      baseSelectionRevision,
+      drawing,
+      box,
+      candidates: greasePencilMarqueeCandidates(this.importedRoot, drawing, this.camera),
+    }, this.currentSnapshot?.revision ?? -1);
+    this.canvas.dataset.greasePencilMarqueeSelectionRevision = String(result.baseSelectionRevision);
+    this.canvas.dataset.greasePencilMarqueeDrawingId = result.drawing.drawingId;
+    this.canvas.dataset.greasePencilMarqueePointIds = result.selectedPoints.map((point) => point.pointId).join(",");
+    this.canvas.dataset.greasePencilMarqueeStrokeIds = result.selectedStrokeIds.join(",");
+    this.canvas.dataset.greasePencilMarqueeCount = String(result.selectedPoints.length);
+    this.onGreasePencilMarqueeSelect?.(result, additive);
+  }
+
   setCurveHandlePreview(dataId: string, handles: readonly CurveGizmoHandleIR[] | null): void {
     applyCurveHandlePreview(this.importedRoot, dataId, handles);
     this.canvas.dataset.curveGizmoPreview = handles ? String(handles.length) : "0";
@@ -675,24 +752,35 @@ export class ViewportRenderer {
     const cameraNode = snapshot.nodes.find((node) => node.id === cameraObjectId && node.type === "CAMERA");
     const definition = snapshot.cameras.find((camera) => camera.id === cameraNode?.dataId);
     if (!cameraNode || !definition) return;
-    const sensor = definition.sensorFit === 2 ? definition.sensorHeightMm : definition.sensorWidthMm;
-    const fov = (2 * Math.atan((sensor / Math.max(0.001, definition.lensMm)) / 2) * 180) / Math.PI;
-    this.camera.fov = definition.projection === "ORTHOGRAPHIC" ? 45 : fov;
-    this.camera.near = Math.max(0.0001, definition.near);
-    this.camera.far = Math.max(this.camera.near + 0.001, definition.far);
-    this.camera.filmGauge = sensor;
-    this.camera.filmOffset = definition.shift[0] * sensor;
-    this.camera.updateProjectionMatrix();
+    configurePBRCamera(this.camera, definition);
   }
 
-  private populateLights(snapshot: SceneSnapshotIR): void {
+  private publishLightingBudget(report: PBRLightingBudgetReport): void {
+    this.canvas.dataset.renderBudgetBackend = report.backend;
+    this.canvas.dataset.renderBudgetStatus = report.status.toLowerCase();
+    this.canvas.dataset.renderBudgetCode = report.issues[0]?.code ?? "";
+    this.canvas.dataset.renderBudgetLights = String(report.requestedLights);
+    this.canvas.dataset.renderBudgetRenderedLights = String(report.renderedLightNodeIds.length);
+    this.canvas.dataset.renderBudgetDroppedLights = String(report.droppedLightNodeIds.length);
+    this.canvas.dataset.renderBudgetShadows = String(report.requestedShadowMaps);
+    this.canvas.dataset.renderBudgetRenderedShadows = String(report.shadowLightNodeIds.length);
+    this.canvas.dataset.renderBudgetBlockedShadows = String(report.shadowBlockedLightNodeIds.length);
+    this.canvas.dataset.renderBudgetShadowMapDimension = String(report.budget.shadowMapDimension);
+  }
+
+  private populateLights(snapshot: SceneSnapshotIR, budget = planPBRLightingBudget(snapshot, "THREE_WEBGL2")): void {
     const lights = new Map(snapshot.lights.map((light) => [light.id, light]));
+    const rendered = new Set(budget.renderedLightNodeIds);
+    const shadowed = new Set(budget.shadowLightNodeIds);
     for (const node of snapshot.nodes) {
-      if (node.type !== "LIGHT" || !node.visible || !node.dataId) continue;
+      if (node.type !== "LIGHT" || !node.visible || !node.dataId || !rendered.has(node.id)) continue;
       const definition = lights.get(node.dataId);
       if (!definition) continue;
       const light = createPBRLight(definition);
-      configurePBRLight(light, node, this.importedLights);
+      configurePBRLight(light, node, this.importedLights, {
+        shadowEnabled: shadowed.has(node.id),
+        shadowMapDimension: budget.budget.shadowMapDimension,
+      });
       light.name = node.name;
       light.userData.sceneNodeId = node.id;
       light.userData.blenderId = node.id;
@@ -707,7 +795,18 @@ export class ViewportRenderer {
     const height = Math.max(1, this.canvas.clientHeight);
     this.camera.aspect = width / height;
     this.camera.updateProjectionMatrix();
+    this.controls.rotateSpeed = VIEWPORT_ORBIT_ROTATE_SENSITIVITY * height / (2 * Math.PI);
     this.renderer.setSize(width, height, false);
+    this.publishCameraState();
+  }
+
+  private publishCameraState(): void {
+    const orbit = orbitStateFromPosition(this.camera.position.toArray(), this.controls.target.toArray());
+    this.canvas.dataset.cameraPosition = this.camera.position.toArray().map((value) => Number(value.toFixed(6))).join(",");
+    this.canvas.dataset.cameraTarget = orbit.target.map((value) => Number(value.toFixed(6))).join(",");
+    this.canvas.dataset.cameraYaw = orbit.yaw.toFixed(6);
+    this.canvas.dataset.cameraPitch = orbit.pitch.toFixed(6);
+    this.canvas.dataset.cameraDistance = orbit.distance.toFixed(6);
   }
 
   private handleClick = (event: MouseEvent): void => {
@@ -724,7 +823,7 @@ export class ViewportRenderer {
       : undefined;
     if (greasePencilHit?.index !== undefined) {
       const point = greasePencilPointRef(greasePencilHit.object, greasePencilHit.index);
-      if (point) this.onGreasePencilPointSelect?.(point, event.shiftKey || event.ctrlKey || event.metaKey);
+      if (point) this.onGreasePencilPointSelect?.(point, event.shiftKey || event.ctrlKey || event.metaKey, this.greasePencilSelectionRevision);
       return;
     }
     const preferredNonMeshHit = this.editMode ? hits.find((intersection) => intersection.index !== undefined && Array.isArray(intersection.object.userData.nonMeshPointKindMap)) : undefined;
@@ -791,6 +890,7 @@ export class ViewportRenderer {
     if (this.disposed) return;
     if (!this.contextLost) {
       this.controls.update();
+      this.publishCameraState();
       this.lodAdapter.update(this.camera, Math.max(1, this.canvas.clientHeight));
       this.renderer.render(this.scene, this.camera);
       this.publishCurveGizmoFrame();