import { BufferGeometry, BoxGeometry, Color, DirectionalLight, GridHelper, Group, InstancedMesh, Matrix4, Mesh, MeshBasicMaterial, HemisphereLight, MeshPhysicalMaterial, Raycaster, PerspectiveCamera, Scene, type Object3D, WebGLRenderer, Float32BufferAttribute, Uint32BufferAttribute, Euler, Quaternion, Vector2, Vector3, } from "../vendor/three/three.module.js"; import { OrbitControls } from "../vendor/three/addons/controls/OrbitControls.js"; import type { MaterialIR, SceneSnapshotIR } from "../../../protocol/scene-ir"; import { applySceneDelta, sceneDeltaRequiresRendererRebuild, type SceneDelta } from "../../../protocol/scene-delta"; import type { MeshElementMode, MeshGeometryBuffer, WebEngineLODLevelResult } from "../../../protocol/web-engine"; import { ThreeLODAdapter, type ThreeLODLevel, type LODSelectionResult } from "./lod"; import { configurePBRLight, configurePBRCamera, configurePBRRenderer, createPBRLight, createPBRMaterial, PBR_PROFILE, PBR_SHADOW_MAP_DIMENSION, PBR_SHADOW_PROFILE, PBR_TONE_MAPPING, setPBRMaterialSelected, } from "./pbr"; import { GPUTextureStore } from "./texture-assets"; import type { GPUTextureAsset } from "../../../protocol/render-assets"; import { gateEnvironmentImage, gateUDIMImage } from "../../../protocol/render-assets"; import { planPBRLightingBudget, type PBRLightingBudgetReport } from "../../../protocol/render-budget"; import { applyCurveHandlePreview, applyNonMeshElementSelection, applyNonMeshTransform, createNonMeshObject, type NonMeshElementKind } from "./nonmesh"; import type { NonMeshGeometryChunk } from "../../../protocol/nonmesh-binary"; import type { CurveGizmoFrameIR, CurveGizmoHandleIR, CurveGizmoScreenFrameIR } from "../../../protocol/nonmesh-interaction"; import type { NanoVDBViewportAssetIR, NanoVDBViewportRenderResultIR } from "../volume/nanovdb-viewport"; import { NANOVDB_VIEWPORT_PREVIEW_SIZE, NanoVDBViewportRenderSession, renderNanoVDBViewportAsset } from "../volume/nanovdb-viewport"; import { createNanoVDBViewportObject } from "./volume"; import { applyGreasePencilPointSelection, applyGreasePencilPointPreview, applyGreasePencilTransform, createGreasePencilObject, greasePencilMarqueeCandidates, greasePencilPointRef, type GreasePencilPointRef, type GreasePencilPointPreview, } from "./grease-pencil"; import { selectGreasePencilMarquee, type GreasePencilDrawingScopeIR, type GreasePencilMarqueeBoxIR, type GreasePencilMarqueeResultIR, } from "../../../protocol/grease-pencil-marquee"; import { VIEWPORT_DEFAULT_ORBIT, VIEWPORT_ORBIT_MAX_DISTANCE, VIEWPORT_ORBIT_MIN_DISTANCE, VIEWPORT_ORBIT_ROTATE_SENSITIVITY, VIEWPORT_ORBIT_ZOOM_SENSITIVITY, orbitPosition, orbitStateFromPosition } from "../../../protocol/viewport-camera"; import { validatePaintDepthVisibilityRequest, type PaintDepthVisibilityRequestIR, type PaintDepthVisibilityResultIR } from "../../../protocol/paint-depth-visibility"; import { samplePaintDepthVisibilityGPU } from "./paint-depth-visibility"; export function collectMeshInstanceGroups(snapshot: SceneSnapshotIR, minimumSize = 2): Map { const groups = new Map(); for (const node of snapshot.nodes) { if (node.type !== "MESH" || !node.visible || !node.dataId) continue; const ids = groups.get(node.dataId) ?? []; ids.push(node.id); groups.set(node.dataId, ids); } for (const [meshId, ids] of groups) if (ids.length < minimumSize) groups.delete(meshId); return groups; } /** Three.js owns the browser viewport; Blender's Z-up coordinates are adapted at the boundary. */ export class ViewportRenderer { readonly renderer: WebGLRenderer; readonly scene: Scene; readonly camera: PerspectiveCamera; readonly controls: OrbitControls; private readonly canvas: HTMLCanvasElement; private readonly importedRoot = new Group(); private readonly importedLights = new Group(); private readonly resizeObserver: ResizeObserver; private animationFrame = 0; private disposed = false; private contextLost = false; private currentSnapshot: SceneSnapshotIR | null = null; private readonly objectByBlenderId = new Map(); private readonly instanceIndexByBlenderId = new Map(); private readonly lodAdapter = new ThreeLODAdapter(); private readonly textureStore = new GPUTextureStore("THREE_WEBGL2"); private readonly raycaster = new Raycaster(); private readonly pointer = new Vector2(); private readonly onSelect?: (objectId: string, additive: boolean) => void; private readonly onElementSelect?: (meshId: string, mode: MeshElementMode, index: number, additive: boolean, nonMeshKind?: NonMeshElementKind) => void; private readonly onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void; private readonly onGreasePencilMarqueeSelect?: (result: GreasePencilMarqueeResultIR, additive: boolean) => void; private editMode = false; private selectionMode: MeshElementMode = "FACE"; private greasePencilSelectionRevision = 0; private curveGizmoFrame: { dataId: string; frame: CurveGizmoFrameIR } | null = null; private curveGizmoScreenFrame = ""; private volumeAssets: NanoVDBViewportAssetIR[] = []; private readonly volumeRenderCache = new Map(); private volumeRenderGeneration = 0; private readonly volumeRenderSession: NanoVDBViewportRenderSession; constructor( canvas: HTMLCanvasElement, onSelect?: (objectId: string, additive: boolean) => void, onElementSelect?: (meshId: string, mode: MeshElementMode, index: number, additive: boolean, nonMeshKind?: NonMeshElementKind) => void, onGreasePencilPointSelect?: (point: GreasePencilPointRef, additive: boolean, baseSelectionRevision: number) => void, onGreasePencilMarqueeSelect?: (result: GreasePencilMarqueeResultIR, additive: boolean) => void, ) { this.canvas = canvas; this.onSelect = onSelect; this.onElementSelect = onElementSelect; this.onGreasePencilPointSelect = onGreasePencilPointSelect; this.onGreasePencilMarqueeSelect = onGreasePencilMarqueeSelect; this.volumeRenderSession = new NanoVDBViewportRenderSession(() => { if (this.disposed) return; this.volumeRenderCache.clear(); this.canvas.dataset.volumeStatus = "loading"; void this.refreshVolumes(); }); this.raycaster.params.Points.threshold = 0.14; this.renderer = new WebGLRenderer({ canvas, antialias: true, alpha: false, preserveDrawingBuffer: true }); configurePBRRenderer(this.renderer); this.renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); this.renderer.setClearColor(new Color("#25272b")); this.canvas.dataset.rendererBackend = "webgl-pbr"; this.canvas.dataset.pbrProfile = PBR_PROFILE; this.canvas.dataset.toneMapping = PBR_TONE_MAPPING; this.canvas.dataset.shadowMap = PBR_SHADOW_PROFILE; this.canvas.dataset.deviceStatus = "ready"; this.scene = new Scene(); this.camera = new PerspectiveCamera(45, 1, 0.01, 1000); this.camera.position.set(...orbitPosition(VIEWPORT_DEFAULT_ORBIT)); this.controls = new OrbitControls(this.camera, canvas); this.controls.target.set(...VIEWPORT_DEFAULT_ORBIT.target); this.controls.enableDamping = false; this.controls.enablePan = false; this.controls.minDistance = VIEWPORT_ORBIT_MIN_DISTANCE; this.controls.maxDistance = VIEWPORT_ORBIT_MAX_DISTANCE; this.controls.zoomSpeed = VIEWPORT_ORBIT_ZOOM_SENSITIVITY / (0.01 * -Math.log(0.95)); this.scene.add(new HemisphereLight(0xf2f5ff, 0x3a4149, 0.55)); const keyLight = new DirectionalLight(0xffffff, 2.5); keyLight.position.set(4, -5, 8); keyLight.castShadow = true; keyLight.shadow.mapSize.set(PBR_SHADOW_MAP_DIMENSION, PBR_SHADOW_MAP_DIMENSION); keyLight.shadow.bias = -0.0005; keyLight.shadow.normalBias = 0.03; this.scene.add(keyLight, keyLight.target); this.scene.add(new GridHelper(20, 20, 0x60656e, 0x383b42)); this.scene.add(this.importedRoot); this.scene.add(this.importedLights); this.resizeObserver = new ResizeObserver(() => this.resize()); this.resizeObserver.observe(canvas); this.canvas.addEventListener("click", this.handleClick); this.canvas.addEventListener("webglcontextlost", this.handleContextLost); this.canvas.addEventListener("webglcontextrestored", this.handleContextRestored); this.resize(); this.controls.update(); this.publishCameraState(); this.renderLoop(); } setSnapshot(snapshot: SceneSnapshotIR, geometryBuffers: MeshGeometryBuffer[] = [], nonMeshGeometryBuffers: NonMeshGeometryChunk[] = []): void { this.currentSnapshot = snapshot; const udim = snapshot.images.find((image) => image.tiles && image.tiles.length > 0); if (udim) { const gate = gateUDIMImage(udim); this.canvas.dataset.udimGate = gate.status.toLowerCase(); this.canvas.dataset.udimGateCode = gate.issues[0]?.code ?? ""; } const worldId = snapshot.scenes[0]?.worldId; const world = snapshot.worlds.find((candidate) => candidate.id === worldId) ?? snapshot.worlds[0]; if (world?.environmentImageId) { const gate = gateEnvironmentImage(snapshot.images.find((image) => image.id === world.environmentImageId)); this.canvas.dataset.iblGate = gate.status.toLowerCase(); this.canvas.dataset.iblGateCode = gate.issues[0]?.code ?? ""; } this.objectByBlenderId.clear(); this.instanceIndexByBlenderId.clear(); this.lodAdapter.clear(); this.clearImportedScene(); this.applyWorld(snapshot); this.applyCamera(snapshot); const lightingBudget = planPBRLightingBudget(snapshot, "THREE_WEBGL2"); this.publishLightingBudget(lightingBudget); this.populateLights(snapshot, lightingBudget); this.populateNonMesh(snapshot, nonMeshGeometryBuffers); this.populateGreasePencils(snapshot); const meshes = new Map(snapshot.meshes.map((mesh) => [mesh.id, mesh])); const binaryGeometry = new Map(geometryBuffers.map((payload) => [payload.meshId, payload])); for (const node of snapshot.nodes) { if (node.type !== "MESH" || !node.visible || !node.dataId) continue; const summary = meshes.get(node.dataId); if (!summary) continue; const materialById = new Map(snapshot.materials.map((material) => [material.id, material])); const fallbackMaterial = this.createMaterial(undefined, node.id === snapshot.activeObjectId); const materials = (summary.materialSlotIds ?? []) .map((id) => materialById.get(id)) .map((material) => this.createMaterial(material, node.id === snapshot.activeObjectId)); if (materials.length === 0) materials.push(fallbackMaterial); let geometry: BufferGeometry = new BoxGeometry(2, 2, 2); const payload = binaryGeometry.get(summary.id); const positionsData = payload ? new Float32Array(payload.positions) : summary.positions; const indicesData = payload ? new Uint32Array(payload.indices) : summary.indices; const normalsData = payload?.normals ? new Float32Array(payload.normals) : summary.normals; const triangleCornerData = payload?.triangleCornerIndices ? new Uint32Array(payload.triangleCornerIndices) : summary.triangleCornerIndices; const triangleFaceData = payload?.triangleFaceIndices ? new Uint32Array(payload.triangleFaceIndices) : summary.triangleFaceIndices; const uvData = payload?.uvs ? new Float32Array(payload.uvs) : summary.uvs; const colorData = payload?.colors ? new Float32Array(payload.colors) : summary.colors; const triangleMaterialData = payload?.triangleMaterialIndices ? new Uint32Array(payload.triangleMaterialIndices) : summary.triangleMaterialIndices; if ((summary.geometryStatus === "available" || summary.geometryStatus === "binary") && positionsData && indicesData) { geometry = new BufferGeometry(); const hasCornerAttributes = Boolean(triangleCornerData && (uvData || colorData)); if (hasCornerAttributes && triangleCornerData) { const positions: number[] = []; const normals: number[] = []; const uvs: number[] = []; const colors: number[] = []; for (let index = 0; index < indicesData.length; index++) { const vertex = indicesData[index] * 3; positions.push(positionsData[vertex], positionsData[vertex + 2], -positionsData[vertex + 1]); if (normalsData) { normals.push(normalsData[vertex], normalsData[vertex + 2], -normalsData[vertex + 1]); } const corner = triangleCornerData[index]; if (uvData) uvs.push(uvData[corner * 2], uvData[corner * 2 + 1]); if (colorData) colors.push(colorData[corner * 4], colorData[corner * 4 + 1], colorData[corner * 4 + 2], colorData[corner * 4 + 3]); } geometry.setAttribute("position", new Float32BufferAttribute(positions, 3)); if (normals.length > 0) geometry.setAttribute("normal", new Float32BufferAttribute(normals, 3)); if (uvs.length > 0) geometry.setAttribute("uv", new Float32BufferAttribute(uvs, 2)); if (colors.length > 0) geometry.setAttribute("color", new Float32BufferAttribute(colors, 4)); } else { const positions = new Array(positionsData.length); for (let index = 0; index < positionsData.length; index += 3) { // Blender Z-up/right-handed -> Three.js Y-up/right-handed. positions[index] = positionsData[index]; positions[index + 1] = positionsData[index + 2]; positions[index + 2] = -positionsData[index + 1]; } geometry.setAttribute("position", new Float32BufferAttribute(positions, 3)); if (normalsData) { const normals = new Array(normalsData.length); for (let index = 0; index < normalsData.length; index += 3) { normals[index] = normalsData[index]; normals[index + 1] = normalsData[index + 2]; normals[index + 2] = -normalsData[index + 1]; } geometry.setAttribute("normal", new Float32BufferAttribute(normals, 3)); } geometry.setIndex(new Uint32BufferAttribute(indicesData, 1)); if (!normalsData) geometry.computeVertexNormals(); } if (triangleMaterialData && materials.length > 1) { for (let triangle = 0; triangle < triangleMaterialData.length; triangle++) { const materialIndex = Math.min(triangleMaterialData[triangle], materials.length - 1); geometry.addGroup(triangle * 3, 3, materialIndex); } } } const mesh = new Mesh(geometry, materials.length === 1 ? materials[0] : materials); mesh.castShadow = true; mesh.receiveShadow = true; mesh.name = node.name; // Summary-only meshes use a bounded proxy until their transferable geometry is available. const [x, y, z] = node.transform.translation; mesh.position.set(x, z, -y); mesh.rotation.set(node.transform.rotationEuler[0], node.transform.rotationEuler[2], -node.transform.rotationEuler[1]); mesh.scale.set(...node.transform.scale); mesh.userData.sceneNodeId = node.id; mesh.userData.blenderId = node.id; mesh.userData.meshId = summary.id; mesh.userData.materialSlotIds = summary.materialSlotIds ?? []; mesh.userData.revision = snapshot.revision; mesh.userData.vertexCount = summary.vertexCount; mesh.userData.sourcePositions = positionsData ? Array.from(positionsData) : undefined; mesh.userData.sourceIndices = indicesData ? Array.from(indicesData) : undefined; mesh.userData.triangleFaceIndices = triangleFaceData ? Array.from(triangleFaceData) : undefined; mesh.userData.edgeVertexIndices = payload?.edgeVertexIndices ? Array.from(new Uint32Array(payload.edgeVertexIndices)) : summary.edgeVertexIndices; this.importedRoot.add(mesh); this.objectByBlenderId.set(node.id, mesh); } this.coalesceMeshInstances(snapshot); void this.refreshVolumes(); if (this.importedRoot.children.length > 0) { this.controls.target.set(0, 0, 0); } } private populateNonMesh(snapshot: SceneSnapshotIR, nonMeshGeometryBuffers: readonly NonMeshGeometryChunk[]): void { const dataById = new Map((snapshot.nonMeshData ?? []).map((data) => [data.id, data])); let previewCount = 0; let blockedCount = 0; for (const node of snapshot.nodes) { if (!node.visible || !node.dataId || node.type === "MESH" || node.type === "LIGHT" || node.type === "CAMERA") continue; const data = dataById.get(node.dataId); if (!data) continue; if (data.type === "VOLUME") continue; const object = createNonMeshObject(data, nonMeshGeometryBuffers); if (!object) { blockedCount++; continue; } applyNonMeshTransform(object, node); object.traverse((child) => { child.userData.sceneNodeId = node.id; child.userData.blenderId = node.id; child.userData.nonMeshDataId = data.id; }); this.importedRoot.add(object); this.objectByBlenderId.set(node.id, object); previewCount++; } this.canvas.dataset.nonMeshCount = String(previewCount); this.canvas.dataset.nonMeshBlockedCount = String(blockedCount); } private populateGreasePencils(snapshot: SceneSnapshotIR): void { const dataById = new Map((snapshot.greasePencils ?? []).map((data) => [data.id, data])); let previewCount = 0; let blockedCount = 0; let onionStrokeCount = 0; for (const node of snapshot.nodes) { if (node.type !== "GREASE_PENCIL" || !node.visible || !node.dataId) continue; const data = dataById.get(node.dataId); if (!data) continue; const object = createGreasePencilObject(data, snapshot.frame.current); if (!object) { blockedCount++; continue; } applyGreasePencilTransform(object, node); this.importedRoot.add(object); this.objectByBlenderId.set(node.id, object); onionStrokeCount += Number(object.userData.greasePencilOnionStrokeCount ?? 0); previewCount++; } this.canvas.dataset.greasePencilCount = String(previewCount); this.canvas.dataset.greasePencilBlockedCount = String(blockedCount); this.canvas.dataset.greasePencilOnionStrokeCount = String(onionStrokeCount); } setTextureAssets(assets: readonly GPUTextureAsset[]): void { if (assets.length === 0) { this.canvas.dataset.textureStatus = "none"; this.canvas.dataset.textureLoaded = "0"; this.canvas.dataset.textureBytes = "0"; this.canvas.dataset.textureBudgetStatus = "ready"; this.canvas.dataset.textureBudgetCode = ""; this.canvas.dataset.textureBudgetAssets = "0"; this.canvas.dataset.textureBudgetPayloadBytes = "0"; this.canvas.dataset.textureBudgetGpuBytes = "0"; return; } void this.textureStore.upload(assets).then((status) => { this.canvas.dataset.textureStatus = status.rejected > 0 ? "blocked" : "ready"; this.canvas.dataset.textureLoaded = String(status.loaded); this.canvas.dataset.textureBytes = String(status.bytes); this.canvas.dataset.textureErrorCode = status.errorCodes[0] ?? ""; this.canvas.dataset.textureBudgetStatus = status.budget.status.toLowerCase(); this.canvas.dataset.textureBudgetCode = status.budget.issues[0]?.code ?? ""; this.canvas.dataset.textureBudgetAssets = String(status.budget.requestedAssets); this.canvas.dataset.textureBudgetPayloadBytes = String(status.budget.payloadBytes); this.canvas.dataset.textureBudgetGpuBytes = String(status.budget.decodedGPUBytes); if (!this.currentSnapshot) return; this.textureStore.applySnapshotMaterials(this.importedRoot, this.currentSnapshot); const worldId = this.currentSnapshot.scenes[0]?.worldId; const world = this.currentSnapshot.worlds.find((candidate) => candidate.id === worldId) ?? this.currentSnapshot.worlds[0]; void this.textureStore.applyEnvironment(this.scene, this.renderer, world, true).then((ready) => { this.canvas.dataset.iblStatus = ready ? "ready" : world?.environmentImageId ? "blocked" : "none"; }); }); } setVolumeAssets(assets: readonly NanoVDBViewportAssetIR[]): void { this.volumeAssets = [...assets]; void this.refreshVolumes(); } private async refreshVolumes(): Promise { const generation = ++this.volumeRenderGeneration; for (const child of [...this.importedRoot.children]) { if (!child.userData.nanoVDBVolume) continue; this.importedRoot.remove(child); child.traverse((object) => { const mesh = object as Mesh; mesh.geometry?.dispose?.(); const materials = Array.isArray(mesh.material) ? mesh.material : [mesh.material]; for (const material of materials) { if (material instanceof MeshBasicMaterial) material.map?.dispose(); material?.dispose?.(); } }); } const snapshot = this.currentSnapshot; const volumeNodes = snapshot?.nodes.filter((node) => node.visible && node.type === "VOLUME" && node.dataId) ?? []; if (!snapshot || volumeNodes.length === 0) { this.canvas.dataset.volumeStatus = "none"; this.canvas.dataset.volumeCount = "0"; return; } this.canvas.dataset.volumeStatus = "loading"; this.canvas.dataset.volumeCount = "0"; try { let rendered = 0; for (const node of volumeNodes) { const asset = this.volumeAssets.find((candidate) => candidate.dataId === node.dataId); if (!asset) continue; const cacheKey = `${asset.dataId}:${asset.manifest.bundleSha256}:${JSON.stringify(asset.material ?? asset.manifest.material)}`; let result = this.volumeRenderCache.get(cacheKey); if (!result) { result = await renderNanoVDBViewportAsset(asset, NANOVDB_VIEWPORT_PREVIEW_SIZE, NANOVDB_VIEWPORT_PREVIEW_SIZE, this.volumeRenderSession); this.volumeRenderCache.set(cacheKey, result); } if (generation !== this.volumeRenderGeneration || this.currentSnapshot !== snapshot) return; const object = createNanoVDBViewportObject(result, node); this.importedRoot.add(object); this.objectByBlenderId.set(node.id, object); rendered++; } if (generation !== this.volumeRenderGeneration) return; this.canvas.dataset.volumeCount = String(rendered); this.canvas.dataset.volumeStatus = rendered === volumeNodes.length ? "ready" : "blocked"; this.canvas.dataset.volumeErrorCode = rendered === volumeNodes.length ? "" : "NON_MESH_RESOURCE_MISSING"; } catch (error) { if (generation !== this.volumeRenderGeneration) return; this.canvas.dataset.volumeStatus = "blocked"; this.canvas.dataset.volumeErrorCode = error instanceof Error ? error.message.split(":", 1)[0] : "VOLUME_SHADER_UNAVAILABLE"; } } applyDelta(delta: SceneDelta, geometryBuffers: MeshGeometryBuffer[] = [], nonMeshGeometryBuffers: NonMeshGeometryChunk[] = []): void { if (!this.currentSnapshot) throw new Error("Cannot apply a SceneDelta before a snapshot"); const next = applySceneDelta(this.currentSnapshot, delta); const hasLifecycleChanges = sceneDeltaRequiresRendererRebuild(delta); if (hasLifecycleChanges) { this.setSnapshot(next, geometryBuffers, nonMeshGeometryBuffers); return; } for (const change of delta.nodes?.updated ?? []) { const object = this.objectByBlenderId.get(change.id); if (!object) continue; const instanceIndex = this.instanceIndexByBlenderId.get(change.id); if (object instanceof InstancedMesh && instanceIndex !== undefined) { const node = next.nodes.find((candidate) => candidate.id === change.id); if (node) object.setMatrixAt(instanceIndex, this.instanceMatrix(node.transform, node.visible)); object.instanceMatrix.needsUpdate = true; } else if (change.visible !== undefined) object.visible = change.visible; if (change.visible !== undefined && typeof object.userData.meshId === "string") this.lodAdapter.setEnabled(object.userData.meshId, change.visible); if (change.transform && !(object instanceof InstancedMesh)) { const [x, y, z] = change.transform.translation; object.position.set(x, z, -y); object.rotation.set(change.transform.rotationEuler[0], change.transform.rotationEuler[2], -change.transform.rotationEuler[1]); object.scale.set(...change.transform.scale); } object.userData.revision = next.revision; } this.currentSnapshot = next; } setSelection( objectIds: ReadonlySet, elementSelection?: ReadonlyMap>>, greasePencilPoints: readonly GreasePencilPointRef[] = [], greasePencilSelectionRevision = 0, ): void { const visitedInstances = new Set(); for (const [objectId, object] of this.objectByBlenderId) { if (object instanceof InstancedMesh) { if (visitedInstances.has(object)) continue; visitedInstances.add(object); const ids = object.userData.instanceNodeIds as string[]; for (let index = 0; index < ids.length; index++) { object.setColorAt(index, new Color(objectIds.has(ids[index]) ? 0xf08a45 : 0xffffff)); } if (object.instanceColor) object.instanceColor.needsUpdate = true; continue; } if (!(object instanceof Mesh)) continue; const materials = Array.isArray(object.material) ? object.material : [object.material]; for (const material of materials) { if (!(material instanceof MeshPhysicalMaterial)) continue; setPBRMaterialSelected(material, objectIds.has(objectId)); } } applyNonMeshElementSelection(this.importedRoot, elementSelection ?? new Map()); applyGreasePencilPointSelection(this.importedRoot, greasePencilPoints); this.greasePencilSelectionRevision = greasePencilSelectionRevision; this.canvas.dataset.greasePencilSelectionRevision = String(greasePencilSelectionRevision); this.canvas.dataset.greasePencilSelectionPointIds = greasePencilPoints.map((point) => point.pointId).join(","); } setInteractionMode(editMode: boolean, selectionMode: MeshElementMode): void { this.editMode = editMode; this.selectionMode = selectionMode; this.importedRoot.traverse((object) => { if (typeof object.userData.greasePencilPointDataId === "string") object.visible = editMode; }); } async samplePaintVisibility(requestValue: PaintDepthVisibilityRequestIR): Promise { const request = validatePaintDepthVisibilityRequest(requestValue, this.currentSnapshot?.revision ?? -1); const node = this.currentSnapshot?.nodes.find((candidate) => candidate.id === request.objectId && candidate.dataId === request.meshId && candidate.visible); const object = node ? this.objectByBlenderId.get(node.id) : undefined; if (!object) throw new Error("PAINT_DEPTH_UNAVAILABLE: Paint object is not available in the current viewport"); return samplePaintDepthVisibilityGPU({ renderer: this.renderer, scene: this.scene, camera: this.camera, object, request, backend: "MAIN_THREAD_WEBGL2", }); } selectGreasePencilMarquee( drawing: GreasePencilDrawingScopeIR, box: GreasePencilMarqueeBoxIR, baseRevision: number, baseSelectionRevision: number, additive: boolean, ): void { const result = selectGreasePencilMarquee({ schemaVersion: 1, baseRevision, baseSelectionRevision, drawing, box, candidates: greasePencilMarqueeCandidates(this.importedRoot, drawing, this.camera), }, this.currentSnapshot?.revision ?? -1); this.canvas.dataset.greasePencilMarqueeSelectionRevision = String(result.baseSelectionRevision); this.canvas.dataset.greasePencilMarqueeDrawingId = result.drawing.drawingId; this.canvas.dataset.greasePencilMarqueePointIds = result.selectedPoints.map((point) => point.pointId).join(","); this.canvas.dataset.greasePencilMarqueeStrokeIds = result.selectedStrokeIds.join(","); this.canvas.dataset.greasePencilMarqueeCount = String(result.selectedPoints.length); this.onGreasePencilMarqueeSelect?.(result, additive); } setCurveHandlePreview(dataId: string, handles: readonly CurveGizmoHandleIR[] | null): void { applyCurveHandlePreview(this.importedRoot, dataId, handles); this.canvas.dataset.curveGizmoPreview = handles ? String(handles.length) : "0"; } setGreasePencilPointPreview(dataId: string, layerId: string, frame: number, points: readonly GreasePencilPointPreview[] | null): void { applyGreasePencilPointPreview(this.importedRoot, dataId, layerId, frame, points); this.canvas.dataset.greasePencilPreview = points ? String(points.length) : "0"; } setCurveGizmoFrame(dataId: string | null, frame: CurveGizmoFrameIR | null): void { this.curveGizmoFrame = dataId && frame ? { dataId, frame } : null; this.publishCurveGizmoFrame(); } registerLOD(meshId: string, levels: readonly ThreeLODLevel[], radius: number): void { this.lodAdapter.register(meshId, levels, radius); } unregisterLOD(meshId: string): void { this.lodAdapter.unregister(meshId); } installLODLevels(meshId: string, levels: readonly WebEngineLODLevelResult[]): void { const source = [...this.objectByBlenderId.values()].find((object) => object.userData.meshId === meshId); if (!(source instanceof Mesh) || source instanceof InstancedMesh || levels.length === 0) return; const created: ThreeLODLevel[] = []; const sourceMaterials = Array.isArray(source.material) ? source.material : [source.material]; source.geometry.computeBoundingSphere(); const radius = source.geometry.boundingSphere?.radius ?? 1; for (const level of levels) { const payload = level.geometryBuffers.find((geometry) => geometry.meshId === level.meshId) ?? level.geometryBuffers[0]; if (!payload) continue; const mesh = new Mesh(this.createLODGeometry(payload, sourceMaterials.length), sourceMaterials); mesh.castShadow = true; mesh.receiveShadow = true; mesh.name = `${source.name} LOD ${level.level}`; mesh.position.copy(source.position); mesh.rotation.copy(source.rotation); mesh.scale.copy(source.scale); mesh.userData.sceneNodeId = source.userData.sceneNodeId; mesh.userData.blenderId = source.userData.blenderId; mesh.userData.meshId = meshId; this.importedRoot.add(mesh); created.push({ object: mesh, screenHeightThreshold: Math.max(24, 768 / 2 ** level.level) }); } if (created.length === 0) return; source.visible = false; this.lodAdapter.register(meshId, created, radius); } updateLODSelection(hysteresis = 0.08): Map { return this.lodAdapter.update(this.camera, Math.max(1, this.canvas.clientHeight), hysteresis); } private createLODGeometry(payload: MeshGeometryBuffer, materialCount: number): BufferGeometry { const positionsData = new Float32Array(payload.positions); const indicesData = new Uint32Array(payload.indices); const normalsData = payload.normals ? new Float32Array(payload.normals) : undefined; const cornerData = payload.triangleCornerIndices ? new Uint32Array(payload.triangleCornerIndices) : undefined; const uvData = payload.uvs ? new Float32Array(payload.uvs) : undefined; const colorData = payload.colors ? new Float32Array(payload.colors) : undefined; const materialData = payload.triangleMaterialIndices ? new Uint32Array(payload.triangleMaterialIndices) : undefined; const geometry = new BufferGeometry(); if (cornerData && (uvData || colorData)) { const positions: number[] = []; const normals: number[] = []; const uvs: number[] = []; const colors: number[] = []; for (let index = 0; index < indicesData.length; index++) { const vertex = indicesData[index] * 3; positions.push(positionsData[vertex], positionsData[vertex + 2], -positionsData[vertex + 1]); if (normalsData) normals.push(normalsData[vertex], normalsData[vertex + 2], -normalsData[vertex + 1]); const corner = cornerData[index]; if (uvData) uvs.push(uvData[corner * 2], uvData[corner * 2 + 1]); if (colorData) colors.push(colorData[corner * 4], colorData[corner * 4 + 1], colorData[corner * 4 + 2], colorData[corner * 4 + 3]); } geometry.setAttribute("position", new Float32BufferAttribute(positions, 3)); if (normals.length > 0) geometry.setAttribute("normal", new Float32BufferAttribute(normals, 3)); if (uvs.length > 0) geometry.setAttribute("uv", new Float32BufferAttribute(uvs, 2)); if (colors.length > 0) geometry.setAttribute("color", new Float32BufferAttribute(colors, 4)); } else { const positions = new Array(positionsData.length); for (let index = 0; index < positionsData.length; index += 3) { positions[index] = positionsData[index]; positions[index + 1] = positionsData[index + 2]; positions[index + 2] = -positionsData[index + 1]; } geometry.setAttribute("position", new Float32BufferAttribute(positions, 3)); if (normalsData) { const normals = new Array(normalsData.length); for (let index = 0; index < normalsData.length; index += 3) { normals[index] = normalsData[index]; normals[index + 1] = normalsData[index + 2]; normals[index + 2] = -normalsData[index + 1]; } geometry.setAttribute("normal", new Float32BufferAttribute(normals, 3)); } geometry.setIndex(new Uint32BufferAttribute(indicesData, 1)); if (!normalsData) geometry.computeVertexNormals(); } if (materialData && materialCount > 1) { for (let triangle = 0; triangle < materialData.length; triangle++) geometry.addGroup(triangle * 3, 3, Math.min(materialData[triangle], materialCount - 1)); } return geometry; } private instanceMatrix(transform: SceneSnapshotIR["nodes"][number]["transform"], visible = true): Matrix4 { const [x, y, z] = transform.translation; const [rx, ry, rz] = transform.rotationEuler; const scale = visible ? new Vector3(...transform.scale) : new Vector3(0, 0, 0); return new Matrix4().compose( new Vector3(x, z, -y), new Quaternion().setFromEuler(new Euler(rx, rz, -ry)), scale, ); } private coalesceMeshInstances(snapshot: SceneSnapshotIR): void { const instanceGroups = collectMeshInstanceGroups(snapshot); for (const [meshId, nodeIds] of instanceGroups) { const meshes = nodeIds.map((id) => this.objectByBlenderId.get(id)).filter((object): object is Mesh => object instanceof Mesh && !(object instanceof InstancedMesh)); if (meshes.length !== nodeIds.length || meshes.length < 2) continue; const first = meshes[0]; const instances = new InstancedMesh(first.geometry, first.material, meshes.length); instances.castShadow = true; instances.receiveShadow = true; instances.name = `${first.name} (${meshes.length} instances)`; instances.frustumCulled = true; instances.userData = { ...first.userData, blenderId: undefined, instanceNodeIds: [...nodeIds], meshId }; for (let index = 0; index < nodeIds.length; index++) { const node = snapshot.nodes.find((candidate) => candidate.id === nodeIds[index]); if (!node) continue; instances.setMatrixAt(index, this.instanceMatrix(node.transform, node.visible)); instances.setColorAt(index, new Color(0xffffff)); this.objectByBlenderId.set(node.id, instances); this.instanceIndexByBlenderId.set(node.id, index); } instances.instanceMatrix.needsUpdate = true; if (instances.instanceColor) instances.instanceColor.needsUpdate = true; for (let index = 0; index < meshes.length; index++) { const mesh = meshes[index]; this.importedRoot.remove(mesh); if (index === 0) continue; mesh.geometry.dispose(); const materials = Array.isArray(mesh.material) ? mesh.material : [mesh.material]; for (const material of materials) material.dispose(); } this.importedRoot.add(instances); } this.canvas.dataset.instanceGroups = String(instanceGroups.size); } private clearImportedScene(): void { while (this.importedRoot.children.length > 0) { const child = this.importedRoot.children.pop(); if (!child) continue; child.traverse((object) => { const mesh = object as Mesh; if (mesh.geometry && typeof mesh.geometry.dispose === "function") mesh.geometry.dispose(); const materials = Array.isArray(mesh.material) ? mesh.material : [mesh.material]; for (const material of materials) { if (mesh.userData.nanoVDBVolume && material instanceof MeshBasicMaterial) material.map?.dispose(); if (material && typeof material.dispose === "function") material.dispose(); } }); } while (this.importedLights.children.length > 0) { const child = this.importedLights.children.pop(); if (child && "dispose" in child && typeof child.dispose === "function") child.dispose(); } } private createMaterial(definition: MaterialIR | undefined, active: boolean): MeshPhysicalMaterial { return createPBRMaterial(definition, active); } private applyWorld(snapshot: SceneSnapshotIR): void { const sceneDefinition = snapshot.scenes.find((scene) => scene.id === snapshot.sceneId) ?? snapshot.scenes[0]; const worldId = snapshot.scenes[0]?.worldId; const world = snapshot.worlds.find((candidate) => candidate.id === worldId) ?? snapshot.worlds[0]; this.scene.background = world ? new Color().setRGB(...world.color) : new Color("#25272b"); configurePBRRenderer(this.renderer, sceneDefinition?.colorManagement?.exposure ?? world?.exposure ?? 0); this.canvas.dataset.viewTransform = sceneDefinition?.colorManagement?.viewTransform ?? ""; this.canvas.dataset.viewLook = sceneDefinition?.colorManagement?.look ?? ""; this.canvas.dataset.mist = world?.mist?.enabled ? "metadata-only" : "disabled"; } private applyCamera(snapshot: SceneSnapshotIR): void { const cameraObjectId = snapshot.scenes[0]?.cameraObjectId; const cameraNode = snapshot.nodes.find((node) => node.id === cameraObjectId && node.type === "CAMERA"); const definition = snapshot.cameras.find((camera) => camera.id === cameraNode?.dataId); if (!cameraNode || !definition) return; configurePBRCamera(this.camera, definition); } private publishLightingBudget(report: PBRLightingBudgetReport): void { this.canvas.dataset.renderBudgetBackend = report.backend; this.canvas.dataset.renderBudgetStatus = report.status.toLowerCase(); this.canvas.dataset.renderBudgetCode = report.issues[0]?.code ?? ""; this.canvas.dataset.renderBudgetLights = String(report.requestedLights); this.canvas.dataset.renderBudgetRenderedLights = String(report.renderedLightNodeIds.length); this.canvas.dataset.renderBudgetDroppedLights = String(report.droppedLightNodeIds.length); this.canvas.dataset.renderBudgetShadows = String(report.requestedShadowMaps); this.canvas.dataset.renderBudgetRenderedShadows = String(report.shadowLightNodeIds.length); this.canvas.dataset.renderBudgetBlockedShadows = String(report.shadowBlockedLightNodeIds.length); this.canvas.dataset.renderBudgetShadowMapDimension = String(report.budget.shadowMapDimension); } private populateLights(snapshot: SceneSnapshotIR, budget = planPBRLightingBudget(snapshot, "THREE_WEBGL2")): void { const lights = new Map(snapshot.lights.map((light) => [light.id, light])); const rendered = new Set(budget.renderedLightNodeIds); const shadowed = new Set(budget.shadowLightNodeIds); for (const node of snapshot.nodes) { if (node.type !== "LIGHT" || !node.visible || !node.dataId || !rendered.has(node.id)) continue; const definition = lights.get(node.dataId); if (!definition) continue; const light = createPBRLight(definition); configurePBRLight(light, node, this.importedLights, { shadowEnabled: shadowed.has(node.id), shadowMapDimension: budget.budget.shadowMapDimension, }); light.name = node.name; light.userData.sceneNodeId = node.id; light.userData.blenderId = node.id; light.userData.revision = snapshot.revision; this.importedLights.add(light); this.objectByBlenderId.set(node.id, light); } } private resize(): void { const width = Math.max(1, this.canvas.clientWidth); const height = Math.max(1, this.canvas.clientHeight); this.camera.aspect = width / height; this.camera.updateProjectionMatrix(); this.controls.rotateSpeed = VIEWPORT_ORBIT_ROTATE_SENSITIVITY * height / (2 * Math.PI); this.renderer.setSize(width, height, false); this.publishCameraState(); } private publishCameraState(): void { const orbit = orbitStateFromPosition(this.camera.position.toArray(), this.controls.target.toArray()); this.canvas.dataset.cameraPosition = this.camera.position.toArray().map((value) => Number(value.toFixed(6))).join(","); this.canvas.dataset.cameraTarget = orbit.target.map((value) => Number(value.toFixed(6))).join(","); this.canvas.dataset.cameraYaw = orbit.yaw.toFixed(6); this.canvas.dataset.cameraPitch = orbit.pitch.toFixed(6); this.canvas.dataset.cameraDistance = orbit.distance.toFixed(6); } private handleClick = (event: MouseEvent): void => { const bounds = this.canvas.getBoundingClientRect(); if (bounds.width <= 0 || bounds.height <= 0) return; this.pointer.set( ((event.clientX - bounds.left) / bounds.width) * 2 - 1, -((event.clientY - bounds.top) / bounds.height) * 2 + 1, ); this.raycaster.setFromCamera(this.pointer, this.camera); const hits = this.raycaster.intersectObjects(this.importedRoot.children, true); const greasePencilHit = this.editMode ? hits.find((intersection) => intersection.index !== undefined && greasePencilPointRef(intersection.object, intersection.index) !== null) : undefined; if (greasePencilHit?.index !== undefined) { const point = greasePencilPointRef(greasePencilHit.object, greasePencilHit.index); if (point) this.onGreasePencilPointSelect?.(point, event.shiftKey || event.ctrlKey || event.metaKey, this.greasePencilSelectionRevision); return; } const preferredNonMeshHit = this.editMode ? hits.find((intersection) => intersection.index !== undefined && Array.isArray(intersection.object.userData.nonMeshPointKindMap)) : undefined; const hit = preferredNonMeshHit ?? hits.find((intersection) => typeof intersection.object.userData.blenderId === "string" || Array.isArray(intersection.object.userData.instanceNodeIds)); if (!hit) return; const additive = event.shiftKey || event.ctrlKey || event.metaKey; const nonMeshDataId = hit.object.userData.nonMeshDataId; if (typeof nonMeshDataId === "string" && hit.index !== undefined) { const indexMap = hit.object.userData.nonMeshPointIndexMap as number[] | undefined; const pointIndex = indexMap?.[hit.index] ?? Math.max(0, Math.floor(hit.object.userData.nonMeshPointOffset ?? 0) + hit.index); const kindMap = hit.object.userData.nonMeshPointKindMap as NonMeshElementKind[] | undefined; const kind = kindMap?.[hit.index] ?? "CONTROL_POINT"; this.canvas.dataset.nonMeshLastPick = `${nonMeshDataId}:${kind}:${pointIndex}`; this.onElementSelect?.(nonMeshDataId, "VERT", pointIndex, additive, kind); return; } const meshId = hit.object.userData.meshId; const triangle = hit.faceIndex ?? -1; const sourceIndices = hit.object.userData.sourceIndices as number[] | undefined; if (this.editMode && typeof meshId === "string" && triangle >= 0 && sourceIndices) { const triangleVertices = sourceIndices.slice(triangle * 3, triangle * 3 + 3); let selectedIndex = -1; if (this.selectionMode === "FACE") { const triangleFaces = hit.object.userData.triangleFaceIndices as number[] | undefined; selectedIndex = triangleFaces?.[triangle] ?? triangle; } else if (this.selectionMode === "VERT") { const positions = hit.object.userData.sourcePositions as number[] | undefined; if (positions) { const localHit = hit.object.worldToLocal(hit.point.clone()); const local = [localHit.x, -localHit.z, localHit.y]; selectedIndex = triangleVertices.reduce((best, vertex) => { if (best < 0) return vertex; const distance = (positions[vertex * 3] - local[0]) ** 2 + (positions[vertex * 3 + 1] - local[1]) ** 2 + (positions[vertex * 3 + 2] - local[2]) ** 2; const bestDistance = (positions[best * 3] - local[0]) ** 2 + (positions[best * 3 + 1] - local[1]) ** 2 + (positions[best * 3 + 2] - local[2]) ** 2; return distance < bestDistance ? vertex : best; }, -1); } } else { const edges = hit.object.userData.edgeVertexIndices as number[] | undefined; if (edges) { const candidates = [[triangleVertices[0], triangleVertices[1]], [triangleVertices[1], triangleVertices[2]], [triangleVertices[2], triangleVertices[0]]]; selectedIndex = candidates.reduce((found, pair) => { if (found >= 0) return found; const low = Math.min(pair[0], pair[1]); const high = Math.max(pair[0], pair[1]); for (let edge = 0; edge < edges.length / 2; edge++) { if (Math.min(edges[edge * 2], edges[edge * 2 + 1]) === low && Math.max(edges[edge * 2], edges[edge * 2 + 1]) === high) return edge; } return -1; }, -1); } } if (selectedIndex >= 0) this.onElementSelect?.(meshId, this.selectionMode, selectedIndex, additive); return; } const instanceIds = hit.object.userData.instanceNodeIds as string[] | undefined; const objectId = instanceIds && hit.instanceId !== undefined ? instanceIds[hit.instanceId] : hit.object.userData.blenderId; if (typeof objectId === "string") this.onSelect?.(objectId, additive); }; private renderLoop = (): void => { if (this.disposed) return; if (!this.contextLost) { this.controls.update(); this.publishCameraState(); this.lodAdapter.update(this.camera, Math.max(1, this.canvas.clientHeight)); this.renderer.render(this.scene, this.camera); this.publishCurveGizmoFrame(); if (this.canvas.dataset.deviceStatus === "restoring") { this.canvas.dataset.deviceStatus = "ready"; this.canvas.dispatchEvent(new CustomEvent("viewport-device-restored")); } } this.animationFrame = window.requestAnimationFrame(this.renderLoop); }; private handleContextLost = (event: Event): void => { event.preventDefault(); this.contextLost = true; this.canvas.dataset.deviceStatus = "lost"; }; private handleContextRestored = (): void => { this.contextLost = false; this.canvas.dataset.deviceStatus = "restoring"; configurePBRRenderer(this.renderer); this.renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); this.renderer.setClearColor(new Color("#25272b")); this.resize(); this.volumeRenderCache.clear(); void this.refreshVolumes(); }; private publishCurveGizmoFrame(): void { const active = this.curveGizmoFrame; let frame: CurveGizmoScreenFrameIR | null = null; const node = active ? this.currentSnapshot?.nodes.find((candidate) => candidate.dataId === active.dataId && candidate.id === this.currentSnapshot?.activeObjectId) : undefined; const object = node ? this.objectByBlenderId.get(node.id) : undefined; if (active && object) { object.updateWorldMatrix(true, false); this.camera.updateMatrixWorld(true); const project = (value: readonly number[]): Vector3 => new Vector3(value[0], value[2], -value[1]).applyMatrix4(object.matrixWorld).project(this.camera); const origin = project(active.frame.origin); const axes = active.frame.axes.map((axis) => { const endpoint = project([active.frame.origin[0] + axis[0], active.frame.origin[1] + axis[1], active.frame.origin[2] + axis[2]]); const x = endpoint.x - origin.x; const y = origin.y - endpoint.y; const magnitude = Math.hypot(x, y); return magnitude > 1e-8 ? [x / magnitude, y / magnitude] as [number, number] : [0, 0] as [number, number]; }) as CurveGizmoScreenFrameIR["axes"]; frame = { origin: [(origin.x + 1) / 2, (1 - origin.y) / 2], axes }; } const serialized = JSON.stringify(frame); if (serialized === this.curveGizmoScreenFrame) return; this.curveGizmoScreenFrame = serialized; this.canvas.dispatchEvent(new CustomEvent("curve-gizmo-frame", { detail: frame })); } dispose(): void { this.disposed = true; window.cancelAnimationFrame(this.animationFrame); this.resizeObserver.disconnect(); this.canvas.removeEventListener("click", this.handleClick); this.canvas.removeEventListener("webglcontextlost", this.handleContextLost); this.canvas.removeEventListener("webglcontextrestored", this.handleContextRestored); this.controls.dispose(); this.lodAdapter.clear(); this.clearImportedScene(); this.volumeRenderGeneration++; this.volumeRenderCache.clear(); this.volumeRenderSession.dispose(); this.textureStore.dispose(); this.renderer.dispose(); } }