Govern task context and advance execution pointer

web/app/src/three-adapter/offscreen-viewport.ts

@@ -9,6 +9,7 @@ import type { OffscreenViewportRequest, OffscreenViewportResponse } from "./offs
 import { PBR_PROFILE, PBR_SHADOW_PROFILE, PBR_TONE_MAPPING } from "./pbr";
 import { resolveViewportPixelMetrics, viewportNDC } from "../../../protocol/viewport-dpr";
 import { observePointerEvent } from "../../../protocol/pointer-contract";
+import { createInputModalState, reduceInputModal, type InputModalState } from "../../../protocol/input-modal";
 import type { NonMeshElementKind } from "./nonmesh";
 import type { GreasePencilPointPreview, GreasePencilPointRef } from "./grease-pencil";
 import type { CurveGizmoFrameIR, CurveGizmoHandleIR } from "../../../protocol/nonmesh-interaction";
@@ -88,6 +89,8 @@ export class OffscreenViewportRenderer implements ViewportBackend {
   private readonly onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void;
   private readonly onGreasePencilMarqueeSelect?: (result: GreasePencilMarqueeResultIR, additive: boolean) => void;
   private pointer: { id: number; x: number; y: number; moved: boolean } | null = null;
+  private inputModal: InputModalState = createInputModalState();
+  private touchNavigationActive = false;
   private lastSnapshot: SceneSnapshotIR | null = null;
   private lastGeometryBuffers: MeshGeometryBuffer[] | null = null;
   private lastNonMeshGeometryBuffers: NonMeshGeometryChunk[] | null = null;
@@ -134,6 +137,7 @@ export class OffscreenViewportRenderer implements ViewportBackend {
     canvas.dataset.pbrProfile = PBR_PROFILE;
     canvas.dataset.toneMapping = PBR_TONE_MAPPING;
     canvas.dataset.shadowMap = PBR_SHADOW_PROFILE;
+    this.publishInputModal();
   }
 
   setSnapshot(snapshot: SceneSnapshotIR, geometryBuffers: MeshGeometryBuffer[] = [], nonMeshGeometryBuffers: NonMeshGeometryChunk[] = []): void {
@@ -252,8 +256,8 @@ export class OffscreenViewportRenderer implements ViewportBackend {
   }
 
   private pointerDown = (event: PointerEvent): void => {
-    try { this.canvas.dataset.lastPointer = JSON.stringify(observePointerEvent(event)); }
-    catch { this.canvas.dataset.lastPointer = "BLOCKED"; }
+    const observation = this.recordInputModalEvent(event);
+    if (observation?.pointerType === "touch" && this.inputModal.activePointerIds.length >= 2) this.touchNavigationActive = true;
     this.pointer = { id: event.pointerId, x: event.clientX, y: event.clientY, moved: false };
     try {
       this.canvas.setPointerCapture(event.pointerId);
@@ -274,8 +278,17 @@ export class OffscreenViewportRenderer implements ViewportBackend {
   };
 
   private pointerUp = (event: PointerEvent): void => {
-    try { this.canvas.dataset.lastPointer = JSON.stringify(observePointerEvent(event)); }
-    catch { this.canvas.dataset.lastPointer = "BLOCKED"; }
+    const observation = this.recordInputModalEvent(event);
+    const wasTouchNavigation = this.touchNavigationActive;
+    if (observation?.pointerType === "touch" && this.inputModal.activePointerIds.length === 0) this.touchNavigationActive = false;
+    if (!observation) {
+      this.pointer = null;
+      return;
+    }
+    if (event.type === "pointercancel" || observation.pointerType === "pen" || wasTouchNavigation) {
+      this.pointer = null;
+      return;
+    }
     if (!this.pointer || this.pointer.id !== event.pointerId) return;
     if (!this.pointer.moved) {
       const bounds = this.canvas.getBoundingClientRect();
@@ -285,6 +298,37 @@ export class OffscreenViewportRenderer implements ViewportBackend {
     this.pointer = null;
   };
 
+  private recordInputModalEvent(event: PointerEvent): ReturnType<typeof observePointerEvent> | null {
+    try {
+      const observation = observePointerEvent(event);
+      this.canvas.dataset.lastPointer = JSON.stringify(observation);
+      if (observation.pointerType === "touch" || observation.pointerType === "pen") {
+        this.inputModal = reduceInputModal(this.inputModal, {
+          type: event.type as "pointerdown" | "pointerup" | "pointercancel",
+          pointerType: observation.pointerType,
+          pointerId: observation.pointerId,
+        });
+        this.publishInputModal();
+      }
+      return observation;
+    }
+    catch {
+      this.canvas.dataset.lastPointer = "BLOCKED";
+      this.canvas.dataset.inputModalError = "INPUT_MODAL_EVENT_BLOCKED";
+      return null;
+    }
+  }
+
+  private publishInputModal(): void {
+    this.canvas.dataset.inputModalSchemaVersion = String(this.inputModal.schemaVersion);
+    this.canvas.dataset.inputModalKind = this.inputModal.kind;
+    this.canvas.dataset.inputModalActivePointerIds = this.inputModal.activePointerIds.join(",");
+    this.canvas.dataset.inputModalCancelled = this.inputModal.cancelled ? "1" : "0";
+    this.canvas.dataset.inputModalNavigationRevision = String(this.inputModal.navigationRevision);
+    this.canvas.dataset.inputModalMainCommitCount = String(this.inputModal.mainCommitCount);
+    this.canvas.dataset.inputModalError = "";
+  }
+
   private wheel = (event: WheelEvent): void => {
     event.preventDefault();
     this.worker.postMessage({ type: "orbit", deltaX: 0, deltaY: 0, zoom: event.deltaY } satisfies OffscreenViewportRequest);

web/app/src/three-adapter/viewport.ts

@@ -71,6 +71,7 @@ import { validatePaintDepthVisibilityRequest, type PaintDepthVisibilityRequestIR
 import { samplePaintDepthVisibilityGPU } from "./paint-depth-visibility";
 import { resolveViewportPixelMetrics, viewportNDC } from "../../../protocol/viewport-dpr";
 import { observePointerEvent } from "../../../protocol/pointer-contract";
+import { createInputModalState, reduceInputModal, type InputModalState } from "../../../protocol/input-modal";
 
 export function collectMeshInstanceGroups(snapshot: SceneSnapshotIR, minimumSize = 2): Map<string, string[]> {
   const groups = new Map<string, string[]>();
@@ -104,6 +105,7 @@ export class ViewportRenderer {
   private readonly textureStore = new GPUTextureStore("THREE_WEBGL2");
   private readonly raycaster = new Raycaster();
   private readonly pointer = new Vector2();
+  private inputModal: InputModalState = createInputModalState();
   private readonly onSelect?: (objectId: string, additive: boolean) => void;
   private readonly onElementSelect?: (meshId: string, mode: MeshElementMode, index: number, additive: boolean, nonMeshKind?: NonMeshElementKind) => void;
   private readonly onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void;
@@ -173,12 +175,14 @@ export class ViewportRenderer {
     this.resizeObserver.observe(canvas);
     this.canvas.addEventListener("click", this.handleClick);
     this.canvas.addEventListener("pointerdown", this.handlePointerObservation);
+    this.canvas.addEventListener("pointerup", this.handlePointerObservation);
     this.canvas.addEventListener("pointercancel", this.handlePointerObservation);
     this.canvas.addEventListener("webglcontextlost", this.handleContextLost);
     this.canvas.addEventListener("webglcontextrestored", this.handleContextRestored);
     this.resize();
     this.controls.update();
     this.publishCameraState();
+    this.publishInputModal();
     this.renderLoop();
   }
 
@@ -821,6 +825,8 @@ export class ViewportRenderer {
     if ("pointerType" in event) {
       try { this.canvas.dataset.lastPointer = JSON.stringify(observePointerEvent(event as MouseEvent & { pointerType?: string; pointerId?: number; pressure?: number; tiltX?: number; tiltY?: number; buttons?: number; type?: string })); }
       catch { this.canvas.dataset.lastPointer = "BLOCKED"; }
+      const pointerType = (event as MouseEvent & { pointerType?: string }).pointerType;
+      if ((pointerType === "touch" || pointerType === "pen") && this.inputModal.cancelled) return;
     }
     const bounds = this.canvas.getBoundingClientRect();
     if (bounds.width <= 0 || bounds.height <= 0) return;
@@ -897,10 +903,34 @@ export class ViewportRenderer {
   };
 
   private handlePointerObservation = (event: PointerEvent): void => {
-    try { this.canvas.dataset.lastPointer = JSON.stringify(observePointerEvent(event)); }
-    catch { this.canvas.dataset.lastPointer = "BLOCKED"; }
+    try {
+      const observation = observePointerEvent(event);
+      this.canvas.dataset.lastPointer = JSON.stringify(observation);
+      if (observation.pointerType === "touch" || observation.pointerType === "pen") {
+        this.inputModal = reduceInputModal(this.inputModal, {
+          type: event.type as "pointerdown" | "pointerup" | "pointercancel",
+          pointerType: observation.pointerType,
+          pointerId: observation.pointerId,
+        });
+        this.publishInputModal();
+      }
+    }
+    catch {
+      this.canvas.dataset.lastPointer = "BLOCKED";
+      this.canvas.dataset.inputModalError = "INPUT_MODAL_EVENT_BLOCKED";
+    }
   };
 
+  private publishInputModal(): void {
+    this.canvas.dataset.inputModalSchemaVersion = String(this.inputModal.schemaVersion);
+    this.canvas.dataset.inputModalKind = this.inputModal.kind;
+    this.canvas.dataset.inputModalActivePointerIds = this.inputModal.activePointerIds.join(",");
+    this.canvas.dataset.inputModalCancelled = this.inputModal.cancelled ? "1" : "0";
+    this.canvas.dataset.inputModalNavigationRevision = String(this.inputModal.navigationRevision);
+    this.canvas.dataset.inputModalMainCommitCount = String(this.inputModal.mainCommitCount);
+    this.canvas.dataset.inputModalError = "";
+  }
+
   private renderLoop = (): void => {
     if (this.disposed) return;
     if (!this.contextLost) {
@@ -965,6 +995,7 @@ export class ViewportRenderer {
     this.resizeObserver.disconnect();
     this.canvas.removeEventListener("click", this.handleClick);
     this.canvas.removeEventListener("pointerdown", this.handlePointerObservation);
+    this.canvas.removeEventListener("pointerup", this.handlePointerObservation);
     this.canvas.removeEventListener("pointercancel", this.handlePointerObservation);
     this.canvas.removeEventListener("webglcontextlost", this.handleContextLost);
     this.canvas.removeEventListener("webglcontextrestored", this.handleContextRestored);