269 lines
14 KiB
TypeScript
269 lines
14 KiB
TypeScript
import {
|
|
evaluateVDBProjectBinding,
|
|
validateNanoVDBBundleManifest,
|
|
validateVDBProjectBinding,
|
|
verifyNanoVDBChunk,
|
|
type NanoVDBBundleManifestIR,
|
|
type VDBProjectBindingIR,
|
|
type VDBProjectBindingStatusIR,
|
|
type VDBProjectReopenContextIR,
|
|
} from "../../../protocol/volume-vdb";
|
|
import { validateProjectId, validateSha256 } from "../storage/opfs-files";
|
|
import { IncrementalSha256 } from "./incremental-sha256";
|
|
import { streamNanoVDBChunks, type NanoVDBRangeSource } from "./nanovdb-stream";
|
|
|
|
type OpfsStorage = StorageManager & { getDirectory?: () => Promise<FileSystemDirectoryHandle> };
|
|
type MovableFile = FileSystemFileHandle & { move?: (name: string) => Promise<void> };
|
|
type DirectoryEntries = AsyncIterableIterator<[string, FileSystemHandle]>;
|
|
|
|
export interface NanoVDBOPFSCommitResult {
|
|
projectId: string;
|
|
bundleSha256: string;
|
|
bundleByteLength: number;
|
|
chunks: number;
|
|
deduplicated: boolean;
|
|
}
|
|
|
|
export interface NanoVDBOPFSOpenResult {
|
|
manifest: NanoVDBBundleManifestIR;
|
|
binding?: VDBProjectBindingIR;
|
|
bindingStatus?: VDBProjectBindingStatusIR;
|
|
source: NanoVDBRangeSource;
|
|
}
|
|
|
|
export async function listVDBProjectBindings(projectId: string, storage?: StorageManager): Promise<VDBProjectBindingIR[]> {
|
|
const cache = await rootFor(projectId, storage);
|
|
const bindings = await directory(cache, "bindings");
|
|
const result: VDBProjectBindingIR[] = [];
|
|
const entries = (bindings as unknown as { entries: () => DirectoryEntries }).entries();
|
|
for await (const [name, handle] of entries) {
|
|
if (handle.kind !== "file" || !/^[a-f0-9]{64}\.json$/.test(name)) continue;
|
|
try { result.push(validateVDBProjectBinding(await readJson<VDBProjectBindingIR>(bindings, name))); }
|
|
catch { /* Invalid binding records are ignored and cannot make a bundle discoverable. */ }
|
|
}
|
|
return result.sort((left, right) => right.committedAt.localeCompare(left.committedAt));
|
|
}
|
|
|
|
async function directory(parent: FileSystemDirectoryHandle, name: string, create = true): Promise<FileSystemDirectoryHandle> {
|
|
if (!/^[A-Za-z0-9._-]{1,128}$/.test(name) || name === "." || name === "..") throw new Error("NANOVDB_MANIFEST_INVALID: OPFS directory name");
|
|
return parent.getDirectoryHandle(name, { create });
|
|
}
|
|
|
|
async function rootFor(projectId: string, storage?: StorageManager): Promise<FileSystemDirectoryHandle> {
|
|
validateProjectId(projectId);
|
|
const manager = (storage ?? navigator.storage) as OpfsStorage;
|
|
if (!manager.getDirectory) throw new Error("NANOVDB_STREAM_INCOMPLETE: OPFS is unavailable");
|
|
let current = await manager.getDirectory();
|
|
for (const name of ["projects", projectId, "cache", "vdb"]) current = await directory(current, name);
|
|
await directory(current, "bindings");
|
|
return current;
|
|
}
|
|
|
|
async function writeFile(parent: FileSystemDirectoryHandle, name: string, value: ArrayBuffer | string): Promise<void> {
|
|
const handle = await parent.getFileHandle(name, { create: true });
|
|
const writer = await handle.createWritable();
|
|
await writer.write(value);
|
|
await writer.close();
|
|
}
|
|
|
|
async function atomicWrite(parent: FileSystemDirectoryHandle, name: string, value: ArrayBuffer | string): Promise<void> {
|
|
const stageName = `${name}.${crypto.randomUUID()}.stage`;
|
|
await writeFile(parent, stageName, value);
|
|
const stage = await parent.getFileHandle(stageName) as MovableFile;
|
|
if (stage.move) await stage.move(name);
|
|
else {
|
|
const bytes = await (await stage.getFile()).arrayBuffer();
|
|
await writeFile(parent, name, bytes);
|
|
await parent.removeEntry(stageName);
|
|
}
|
|
}
|
|
|
|
async function readJson<T>(parent: FileSystemDirectoryHandle, name: string): Promise<T> {
|
|
const bytes = await (await (await parent.getFileHandle(name)).getFile()).arrayBuffer();
|
|
return JSON.parse(new TextDecoder().decode(bytes)) as T;
|
|
}
|
|
|
|
async function remove(parent: FileSystemDirectoryHandle, name: string, recursive = false): Promise<void> {
|
|
try { await parent.removeEntry(name, { recursive }); }
|
|
catch (error) { if (!(error instanceof DOMException) || error.name !== "NotFoundError") throw error; }
|
|
}
|
|
|
|
function chunkName(index: number): string {
|
|
return `${String(index).padStart(5, "0")}.chunk`;
|
|
}
|
|
|
|
async function digestJson(value: unknown): Promise<string> {
|
|
const bytes = new TextEncoder().encode(JSON.stringify(value));
|
|
const hash = await crypto.subtle.digest("SHA-256", bytes);
|
|
return Array.from(new Uint8Array(hash), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
}
|
|
|
|
export async function createVDBProjectBinding(
|
|
manifestValue: NanoVDBBundleManifestIR,
|
|
sourceBlendSha256: string,
|
|
): Promise<VDBProjectBindingIR> {
|
|
const manifest = validateNanoVDBBundleManifest(manifestValue);
|
|
validateSha256(sourceBlendSha256);
|
|
const manifestSha256 = await digestJson(manifest);
|
|
return validateVDBProjectBinding({
|
|
schemaVersion: 1,
|
|
projectId: manifest.projectId,
|
|
sourceBlendSha256,
|
|
sourcePath: manifest.sourcePath,
|
|
sourceSha256: manifest.sourceSha256,
|
|
conversionRequestSha256: manifest.conversionRequestSha256,
|
|
bundleSha256: manifest.bundleSha256,
|
|
bundleByteLength: manifest.bundleByteLength,
|
|
manifestSha256,
|
|
converter: manifest.converter,
|
|
shaderSemanticVersion: manifest.gpu.shaderSemanticVersion,
|
|
material: manifest.material,
|
|
committedAt: new Date().toISOString(),
|
|
});
|
|
}
|
|
|
|
export async function commitNanoVDBToOPFS(
|
|
manifestValue: NanoVDBBundleManifestIR,
|
|
source: NanoVDBRangeSource,
|
|
signal: AbortSignal,
|
|
bindingValue?: VDBProjectBindingIR,
|
|
storage?: StorageManager,
|
|
): Promise<NanoVDBOPFSCommitResult> {
|
|
const manifest = validateNanoVDBBundleManifest(manifestValue);
|
|
const binding = bindingValue ? validateVDBProjectBinding(bindingValue) : undefined;
|
|
const manifestSha256 = await digestJson(manifest);
|
|
if (binding && (binding.projectId !== manifest.projectId || binding.bundleSha256 !== manifest.bundleSha256 || binding.conversionRequestSha256 !== manifest.conversionRequestSha256 || binding.manifestSha256 !== manifestSha256)) throw new Error("NANOVDB_HASH_MISMATCH: project binding does not match manifest");
|
|
const cache = await rootFor(manifest.projectId, storage);
|
|
const bundle = await directory(cache, manifest.bundleSha256);
|
|
try {
|
|
const existing = validateNanoVDBBundleManifest(await readJson<NanoVDBBundleManifestIR>(bundle, "manifest.json"));
|
|
if (existing.bundleSha256 === manifest.bundleSha256 && existing.conversionRequestSha256 === manifest.conversionRequestSha256 && await digestJson(existing) === manifestSha256) {
|
|
for (const chunk of existing.chunks) {
|
|
const data = await (await (await bundle.getFileHandle(chunkName(chunk.index))).getFile()).arrayBuffer();
|
|
await verifyNanoVDBChunk(chunk, data);
|
|
}
|
|
await atomicWrite(bundle, "access.json", JSON.stringify({ lastAccessAt: new Date().toISOString(), bytes: manifest.bundleByteLength }));
|
|
if (binding) {
|
|
const bindings = await directory(cache, "bindings");
|
|
await atomicWrite(bindings, `${binding.conversionRequestSha256}.json`, JSON.stringify(binding));
|
|
}
|
|
return { projectId: manifest.projectId, bundleSha256: manifest.bundleSha256, bundleByteLength: manifest.bundleByteLength, chunks: manifest.chunks.length, deduplicated: true };
|
|
}
|
|
}
|
|
catch { /* An incomplete directory is staging and remains undiscoverable until manifest commit. */ }
|
|
|
|
const hasher = new IncrementalSha256();
|
|
const staged: string[] = [];
|
|
try {
|
|
await streamNanoVDBChunks(manifest, source, async (range, data) => {
|
|
if (signal.aborted) throw new DOMException("NanoVDB OPFS commit cancelled", "AbortError");
|
|
hasher.update(data);
|
|
const name = `${chunkName(range.chunkIndex)}.${crypto.randomUUID()}.stage`;
|
|
staged.push(name);
|
|
await writeFile(bundle, name, data);
|
|
const written = await (await bundle.getFileHandle(name)).getFile();
|
|
await verifyNanoVDBChunk(manifest.chunks[range.chunkIndex], await written.arrayBuffer());
|
|
}, signal);
|
|
if (hasher.hex() !== manifest.bundleSha256) throw new Error("NANOVDB_HASH_MISMATCH: streamed bundle SHA-256 mismatch");
|
|
for (let index = 0; index < staged.length; index++) {
|
|
const handle = await bundle.getFileHandle(staged[index]) as MovableFile;
|
|
if (handle.move) await handle.move(chunkName(index));
|
|
else {
|
|
await writeFile(bundle, chunkName(index), await (await handle.getFile()).arrayBuffer());
|
|
await bundle.removeEntry(staged[index]);
|
|
}
|
|
}
|
|
await atomicWrite(bundle, "access.json", JSON.stringify({ lastAccessAt: new Date().toISOString(), bytes: manifest.bundleByteLength }));
|
|
await atomicWrite(bundle, "manifest.json", JSON.stringify(manifest));
|
|
if (binding) {
|
|
const bindings = await directory(cache, "bindings");
|
|
await atomicWrite(bindings, `${binding.conversionRequestSha256}.json`, JSON.stringify(binding));
|
|
}
|
|
return { projectId: manifest.projectId, bundleSha256: manifest.bundleSha256, bundleByteLength: manifest.bundleByteLength, chunks: manifest.chunks.length, deduplicated: false };
|
|
}
|
|
catch (error) {
|
|
await Promise.all(staged.map((name) => remove(bundle, name)));
|
|
await remove(bundle, "manifest.json");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export async function openNanoVDBFromOPFS(
|
|
projectId: string,
|
|
bundleSha256: string,
|
|
conversionRequestSha256?: string,
|
|
reopenContext?: VDBProjectReopenContextIR,
|
|
storage?: StorageManager,
|
|
): Promise<NanoVDBOPFSOpenResult> {
|
|
validateSha256(bundleSha256);
|
|
if (conversionRequestSha256) validateSha256(conversionRequestSha256);
|
|
const cache = await rootFor(projectId, storage);
|
|
const bundle = await directory(cache, bundleSha256, false);
|
|
const manifest = validateNanoVDBBundleManifest(await readJson<NanoVDBBundleManifestIR>(bundle, "manifest.json"));
|
|
if (manifest.projectId !== projectId || manifest.bundleSha256 !== bundleSha256) throw new Error("NANOVDB_HASH_MISMATCH: OPFS bundle identity mismatch");
|
|
let binding: VDBProjectBindingIR | undefined;
|
|
let bindingStatus: VDBProjectBindingStatusIR | undefined;
|
|
if (conversionRequestSha256) {
|
|
const bindings = await directory(cache, "bindings");
|
|
try { binding = validateVDBProjectBinding(await readJson<VDBProjectBindingIR>(bindings, `${conversionRequestSha256}.json`)); }
|
|
catch { binding = undefined; }
|
|
if (binding && binding.manifestSha256 !== await digestJson(manifest)) throw new Error("NANOVDB_HASH_MISMATCH: OPFS manifest changed after project commit");
|
|
if (reopenContext) bindingStatus = evaluateVDBProjectBinding(binding, reopenContext);
|
|
}
|
|
await atomicWrite(bundle, "access.json", JSON.stringify({ lastAccessAt: new Date().toISOString(), bytes: manifest.bundleByteLength }));
|
|
const source: NanoVDBRangeSource = async (range, signal) => {
|
|
if (signal.aborted) throw new DOMException("NanoVDB OPFS read cancelled", "AbortError");
|
|
const declared = manifest.chunks[range.chunkIndex];
|
|
if (!declared || range.start !== declared.byteOffset || range.endExclusive !== declared.byteOffset + declared.byteLength) throw new Error("NANOVDB_STREAM_INCOMPLETE: OPFS range is not a declared chunk");
|
|
const file = await (await bundle.getFileHandle(chunkName(range.chunkIndex))).getFile();
|
|
if (file.size !== declared.byteLength) throw new Error("NANOVDB_STREAM_INCOMPLETE: OPFS chunk length mismatch");
|
|
const data = await file.arrayBuffer();
|
|
await verifyNanoVDBChunk(declared, data);
|
|
return data;
|
|
};
|
|
return { manifest, binding, bindingStatus, source };
|
|
}
|
|
|
|
export async function recoverNanoVDBOPFS(projectId: string, storage?: StorageManager): Promise<{ removedStaging: number; removedIncompleteBundles: number }> {
|
|
const cache = await rootFor(projectId, storage);
|
|
let removedStaging = 0;
|
|
let removedIncompleteBundles = 0;
|
|
const entries = (cache as unknown as { entries: () => DirectoryEntries }).entries();
|
|
for await (const [name, handle] of entries) {
|
|
if (handle.kind !== "directory" || name === "bindings") continue;
|
|
if (!/^[a-f0-9]{64}$/.test(name)) { await remove(cache, name, true); removedIncompleteBundles++; continue; }
|
|
const bundle = handle as FileSystemDirectoryHandle;
|
|
let validManifest: boolean;
|
|
try { validManifest = validateNanoVDBBundleManifest(await readJson<NanoVDBBundleManifestIR>(bundle, "manifest.json")).bundleSha256 === name; }
|
|
catch { validManifest = false; }
|
|
if (!validManifest) { await remove(cache, name, true); removedIncompleteBundles++; continue; }
|
|
const files = (bundle as unknown as { entries: () => DirectoryEntries }).entries();
|
|
for await (const [fileName] of files) if (fileName.endsWith(".stage")) { await remove(bundle, fileName); removedStaging++; }
|
|
}
|
|
return { removedStaging, removedIncompleteBundles };
|
|
}
|
|
|
|
export async function pruneNanoVDBOPFS(projectId: string, maxBytes: number, storage?: StorageManager): Promise<{ removed: string[]; retainedBytes: number }> {
|
|
if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) throw new Error("NANOVDB_GPU_BUDGET_EXCEEDED: invalid OPFS cache budget");
|
|
const cache = await rootFor(projectId, storage);
|
|
const bundles: Array<{ name: string; bytes: number; lastAccessAt: string }> = [];
|
|
const entries = (cache as unknown as { entries: () => DirectoryEntries }).entries();
|
|
for await (const [name, handle] of entries) {
|
|
if (handle.kind !== "directory" || !/^[a-f0-9]{64}$/.test(name)) continue;
|
|
try {
|
|
const access = await readJson<{ bytes: number; lastAccessAt: string }>(handle as FileSystemDirectoryHandle, "access.json");
|
|
if (Number.isSafeInteger(access.bytes) && access.bytes > 0 && Number.isFinite(Date.parse(access.lastAccessAt))) bundles.push({ name, ...access });
|
|
}
|
|
catch { /* Recovery owns incomplete entries. */ }
|
|
}
|
|
let total = bundles.reduce((sum, item) => sum + item.bytes, 0);
|
|
const removed: string[] = [];
|
|
for (const item of bundles.sort((left, right) => left.lastAccessAt.localeCompare(right.lastAccessAt))) {
|
|
if (total <= maxBytes) break;
|
|
await remove(cache, item.name, true);
|
|
total -= item.bytes;
|
|
removed.push(item.name);
|
|
}
|
|
return { removed, retainedBytes: total };
|
|
}
|