Advance WebGPU volume and bounded workflows
This commit is contained in:
mes123456
27
tools/vdb/CMakeLists.txt
Normal file
27
tools/vdb/CMakeLists.txt
Normal file
@@ -0,0 +1,27 @@
|
||||
cmake_minimum_required(VERSION 3.24)
|
||||
project(blender_web_vdb_tools LANGUAGES CXX)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
|
||||
set(OPENVDB_ROOT "/home/mes123456/working/build_oiio_deps/Release/openvdb" CACHE PATH "OpenVDB install prefix")
|
||||
set(TBB_ROOT "/home/mes123456/working/build_oiio_deps/Release/tbb" CACHE PATH "TBB install prefix")
|
||||
|
||||
find_path(OPENVDB_INCLUDE_DIR openvdb/openvdb.h HINTS "${OPENVDB_ROOT}/include" REQUIRED NO_DEFAULT_PATH)
|
||||
find_library(OPENVDB_LIBRARY openvdb HINTS "${OPENVDB_ROOT}/lib" REQUIRED NO_DEFAULT_PATH)
|
||||
find_path(TBB_INCLUDE_DIR tbb/tbb.h HINTS "${TBB_ROOT}/include" REQUIRED NO_DEFAULT_PATH)
|
||||
find_library(TBB_LIBRARY tbb HINTS "${TBB_ROOT}/lib" REQUIRED NO_DEFAULT_PATH)
|
||||
|
||||
add_library(vdb_toolchain INTERFACE)
|
||||
target_include_directories(vdb_toolchain INTERFACE "${OPENVDB_INCLUDE_DIR}" "${TBB_INCLUDE_DIR}")
|
||||
target_compile_definitions(vdb_toolchain INTERFACE NANOVDB_USE_OPENVDB)
|
||||
target_link_libraries(vdb_toolchain INTERFACE "${OPENVDB_LIBRARY}" "${TBB_LIBRARY}")
|
||||
|
||||
foreach(target vdb_fixture_generator vdb_to_nanovdb)
|
||||
add_executable(${target} "${target}.cc")
|
||||
target_link_libraries(${target} PRIVATE vdb_toolchain)
|
||||
set_target_properties(${target} PROPERTIES
|
||||
BUILD_RPATH "${OPENVDB_ROOT}/lib;${TBB_ROOT}/lib"
|
||||
INSTALL_RPATH "${OPENVDB_ROOT}/lib;${TBB_ROOT}/lib")
|
||||
endforeach()
|
||||
17
tools/vdb/README.md
Normal file
17
tools/vdb/README.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# VDB Desktop/Server Toolchain
|
||||
|
||||
This directory is the native OpenVDB boundary for the Web application. It is not linked into the browser WASM build.
|
||||
|
||||
```bash
|
||||
tools/vdb/provision-resources.sh
|
||||
npm --prefix web run test:vdb-native
|
||||
```
|
||||
|
||||
The provisioner builds two native executables, generates real bounded OpenVDB fixtures once without overwriting
|
||||
their UUID-bearing source files, downloads the official
|
||||
CC-BY-4.0 `sphere.vdb` sample with its Git LFS SHA-256, converts the fixtures with OpenVDB 13.0/NanoVDB 32.9,
|
||||
and writes the resources under `/home/mes123456/resource-library/blender-web-vdb`.
|
||||
|
||||
`vdb_to_nanovdb` accepts only `.vdb`, limits source/output bytes, grid count and active voxels, supports a grid
|
||||
allowlist, writes uncompressed standard NanoVDB segments, and emits a machine-readable conversion report.
|
||||
The current browser renderer remains blocked; these tools establish the conversion and resource boundary only.
|
||||
149
tools/vdb/build-nanovdb-manifest.mjs
Normal file
149
tools/vdb/build-nanovdb-manifest.mjs
Normal file
@@ -0,0 +1,149 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const args = new Map();
|
||||
for (let index = 2; index < process.argv.length; index += 2) {
|
||||
args.set(process.argv[index], process.argv[index + 1]);
|
||||
}
|
||||
|
||||
function required(name) {
|
||||
const value = args.get(name);
|
||||
if (!value) throw new Error(`VDB_MANIFEST_ARGUMENT_MISSING: ${name}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function sha256(file) {
|
||||
return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
||||
}
|
||||
|
||||
function semantic(name) {
|
||||
const known = new Map([
|
||||
["density", "DENSITY"],
|
||||
["temperature", "TEMPERATURE"],
|
||||
["color", "COLOR"],
|
||||
["emission", "EMISSION"],
|
||||
["flame", "EMISSION"],
|
||||
["velocity", "VELOCITY"],
|
||||
]);
|
||||
return known.get(name.toLowerCase()) ?? "CUSTOM";
|
||||
}
|
||||
|
||||
const sourceFile = path.resolve(required("--source"));
|
||||
const bundleFile = path.resolve(required("--bundle"));
|
||||
const reportFile = path.resolve(required("--report"));
|
||||
const outputFile = path.resolve(required("--output"));
|
||||
const converterFile = path.resolve(required("--converter"));
|
||||
const projectId = args.get("--project-id") ?? "vdb-fixtures";
|
||||
const sourcePath = args.get("--source-path") ?? `//volumes/${path.basename(sourceFile)}`;
|
||||
const bundlePath = args.get("--bundle-path") ?? `//volumes/${path.basename(bundleFile)}`;
|
||||
const blenderVersion = args.get("--blender-version") ?? "5.2.0";
|
||||
const converterTarget = args.get("--converter-target") ?? "DESKTOP";
|
||||
const chunkByteLength = Number(args.get("--chunk-bytes") ?? 4 * 1024 * 1024);
|
||||
|
||||
if (converterTarget !== "DESKTOP" && converterTarget !== "SERVER") {
|
||||
throw new Error("VDB_MANIFEST_ARGUMENT_INVALID: --converter-target");
|
||||
}
|
||||
if (!Number.isSafeInteger(chunkByteLength) || chunkByteLength < 64 * 1024 || chunkByteLength > 16 * 1024 * 1024 || chunkByteLength % 32 !== 0) {
|
||||
throw new Error("VDB_MANIFEST_ARGUMENT_INVALID: --chunk-bytes");
|
||||
}
|
||||
for (const file of [sourceFile, bundleFile, reportFile, converterFile]) {
|
||||
if (!fs.statSync(file).isFile()) throw new Error(`VDB_MANIFEST_RESOURCE_MISSING: ${file}`);
|
||||
}
|
||||
|
||||
const report = JSON.parse(fs.readFileSync(reportFile, "utf8"));
|
||||
if (report.schemaVersion !== 1 || !Array.isArray(report.grids) || report.grids.length === 0) throw new Error("VDB_CONVERSION_INVALID: native report");
|
||||
if (path.resolve(report.input) !== sourceFile || path.resolve(report.output) !== bundleFile) throw new Error("VDB_CONVERSION_INVALID: report paths do not match artifacts");
|
||||
|
||||
const sourceSha256 = sha256(sourceFile);
|
||||
const converter = {
|
||||
target: converterTarget,
|
||||
blenderVersion,
|
||||
openVDBVersion: report.openVDBVersion,
|
||||
nanoVDBVersion: report.nanoVDBVersion,
|
||||
executableSha256: sha256(converterFile),
|
||||
};
|
||||
const sourceGrids = report.grids.map((grid) => ({
|
||||
name: grid.name,
|
||||
valueType: grid.sourceType.toUpperCase(),
|
||||
voxelCount: grid.activeVoxelCount,
|
||||
activeVoxelCount: grid.activeVoxelCount,
|
||||
bounds: grid.indexBounds,
|
||||
}));
|
||||
const conversionRequest = {
|
||||
schemaVersion: 1,
|
||||
source: {
|
||||
byteLength: fs.statSync(sourceFile).size,
|
||||
sha256: sourceSha256,
|
||||
grids: sourceGrids,
|
||||
},
|
||||
selectedGrids: report.grids.map((grid) => grid.name),
|
||||
quantization: report.quantization,
|
||||
chunkByteLength,
|
||||
converter,
|
||||
};
|
||||
const conversionRequestSha256 = crypto.createHash("sha256").update(JSON.stringify(conversionRequest)).digest("hex");
|
||||
|
||||
const bundleBytes = fs.readFileSync(bundleFile);
|
||||
const chunks = [];
|
||||
for (let byteOffset = 0, index = 0; byteOffset < bundleBytes.length; byteOffset += chunkByteLength, index += 1) {
|
||||
const data = bundleBytes.subarray(byteOffset, Math.min(bundleBytes.length, byteOffset + chunkByteLength));
|
||||
chunks.push({ index, byteOffset, byteLength: data.length, sha256: crypto.createHash("sha256").update(data).digest("hex") });
|
||||
}
|
||||
|
||||
const grids = report.grids.map((grid) => ({
|
||||
name: grid.name,
|
||||
valueType: grid.valueType,
|
||||
gridClass: grid.gridClass,
|
||||
semantic: semantic(grid.name),
|
||||
activeVoxelCount: grid.activeVoxelCount,
|
||||
segmentByteOffset: grid.segmentByteOffset,
|
||||
segmentByteLength: grid.segmentByteLength,
|
||||
byteOffset: grid.byteOffset,
|
||||
byteLength: grid.byteLength,
|
||||
indexBounds: grid.indexBounds,
|
||||
worldBounds: grid.worldBounds,
|
||||
voxelSize: grid.voxelSize,
|
||||
indexToWorld: grid.indexToWorld,
|
||||
}));
|
||||
const gridBySemantic = new Map(grids.map((grid) => [grid.semantic, grid.name]));
|
||||
if (!gridBySemantic.has("DENSITY")) throw new Error("NANOVDB_GRID_UNSUPPORTED: a browser volume bundle requires a density grid");
|
||||
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
projectId,
|
||||
sourcePath,
|
||||
sourceSha256,
|
||||
conversionRequestSha256,
|
||||
bundlePath,
|
||||
bundleByteLength: bundleBytes.length,
|
||||
bundleSha256: crypto.createHash("sha256").update(bundleBytes).digest("hex"),
|
||||
converter,
|
||||
grids,
|
||||
chunks,
|
||||
material: {
|
||||
densityGrid: gridBySemantic.get("DENSITY"),
|
||||
...(gridBySemantic.has("TEMPERATURE") ? { temperatureGrid: gridBySemantic.get("TEMPERATURE") } : {}),
|
||||
...(gridBySemantic.has("COLOR") ? { colorGrid: gridBySemantic.get("COLOR") } : {}),
|
||||
...(gridBySemantic.has("EMISSION") ? { emissionGrid: gridBySemantic.get("EMISSION") } : {}),
|
||||
...(gridBySemantic.has("VELOCITY") ? { velocityGrid: gridBySemantic.get("VELOCITY") } : {}),
|
||||
densityScale: 1,
|
||||
emissionScale: 0,
|
||||
temperatureScale: 1,
|
||||
anisotropy: 0,
|
||||
interpolation: "LINEAR",
|
||||
},
|
||||
gpu: {
|
||||
representation: "NANOVDB_STORAGE_BUFFER",
|
||||
byteAlignment: 32,
|
||||
pageByteLength: chunkByteLength,
|
||||
maxResidentBytes: 256 * 1024 * 1024,
|
||||
shaderSemanticVersion: "volume-wgsl-v1",
|
||||
...(report.float32TreeLayout ? { float32TreeLayout: report.float32TreeLayout } : {}),
|
||||
...(report.vec3fTreeLayout ? { vec3fTreeLayout: report.vec3fTreeLayout } : {}),
|
||||
},
|
||||
};
|
||||
|
||||
fs.mkdirSync(path.dirname(outputFile), { recursive: true });
|
||||
fs.writeFileSync(outputFile, `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
process.stdout.write(`nanovdb-manifest-ok grids=${grids.length} chunks=${chunks.length} bytes=${bundleBytes.length} sha256=${manifest.bundleSha256}\n`);
|
||||
16
tools/vdb/build-native-tools.sh
Executable file
16
tools/vdb/build-native-tools.sh
Executable file
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
build_dir=${VDB_TOOLS_BUILD_DIR:-"$repo_root/build_vdb_tools"}
|
||||
openvdb_root=${OPENVDB_ROOT:-/home/mes123456/working/build_oiio_deps/Release/openvdb}
|
||||
tbb_root=${TBB_ROOT:-/home/mes123456/working/build_oiio_deps/Release/tbb}
|
||||
|
||||
cmake -S "$repo_root/tools/vdb" -B "$build_dir" \
|
||||
-DOPENVDB_ROOT="$openvdb_root" \
|
||||
-DTBB_ROOT="$tbb_root" \
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build "$build_dir" --parallel "${VDB_BUILD_JOBS:-4}"
|
||||
|
||||
"$build_dir/vdb_to_nanovdb" --help
|
||||
printf 'vdb-native-tools-ok build=%s\n' "$build_dir"
|
||||
78
tools/vdb/catalog-resources.mjs
Normal file
78
tools/vdb/catalog-resources.mjs
Normal file
@@ -0,0 +1,78 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const resourceRoot = path.resolve(process.env.VDB_RESOURCE_ROOT ?? "/home/mes123456/resource-library/blender-web-vdb");
|
||||
const converter = path.join(repoRoot, "build_vdb_tools", "vdb_to_nanovdb");
|
||||
const generator = path.join(repoRoot, "build_vdb_tools", "vdb_fixture_generator");
|
||||
const openVDBLicense = "/home/mes123456/working/build_oiio_deps/build/openvdb/src/external_openvdb/LICENSE";
|
||||
const ccLicense = path.join(resourceRoot, "licenses", "CC-BY-4.0.txt");
|
||||
|
||||
function sha256(file) {
|
||||
return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
||||
}
|
||||
|
||||
function entry(id, relativePath, kind, license, origin, extra = {}) {
|
||||
const file = path.join(resourceRoot, relativePath);
|
||||
if (!fs.statSync(file).isFile()) throw new Error(`VDB_RESOURCE_MISSING: ${file}`);
|
||||
return { id, kind, path: relativePath, byteLength: fs.statSync(file).size, sha256: sha256(file), license, origin, ...extra };
|
||||
}
|
||||
|
||||
fs.mkdirSync(path.join(resourceRoot, "generated"), { recursive: true });
|
||||
fs.mkdirSync(path.join(resourceRoot, "licenses"), { recursive: true });
|
||||
fs.copyFileSync(openVDBLicense, path.join(resourceRoot, "licenses", "OpenVDB-Apache-2.0.txt"));
|
||||
if (!fs.existsSync(ccLicense)) throw new Error("VDB_RESOURCE_LICENSE_MISSING: CC-BY-4.0.txt");
|
||||
|
||||
const smoke = fs.readFileSync(path.join(resourceRoot, "generated", "generated-smoke.vdb"));
|
||||
fs.writeFileSync(path.join(resourceRoot, "generated", "generated-smoke-truncated.vdb"), smoke.subarray(0, 1024));
|
||||
|
||||
const generatedOrigin = "Generated locally by tools/vdb/vdb_fixture_generator.cc with OpenVDB 13.0.0; contains no third-party model data";
|
||||
const officialOrigin = "OpenVDB official sample model repository";
|
||||
const entries = [
|
||||
entry("generated-smoke", "generated/generated-smoke.vdb", "SOURCE_VDB", "PROJECT_GENERATED_FIXTURE", generatedOrigin),
|
||||
entry("generated-level-set", "generated/generated-level-set.vdb", "SOURCE_VDB", "PROJECT_GENERATED_FIXTURE", generatedOrigin),
|
||||
entry("generated-large-bounds-sparse", "generated/generated-large-bounds-sparse.vdb", "SOURCE_VDB", "PROJECT_GENERATED_FIXTURE", generatedOrigin),
|
||||
entry("generated-smoke-truncated", "generated/generated-smoke-truncated.vdb", "MALFORMED_VDB", "PROJECT_GENERATED_FIXTURE", generatedOrigin, { expectedResult: "VDB_CONVERSION_FAILED" }),
|
||||
entry("official-sphere", "official/sphere.vdb", "SOURCE_VDB", "CC-BY-4.0", officialOrigin, {
|
||||
sourcePage: "https://www.openvdb.org/download/",
|
||||
sourceUrl: "https://media.githubusercontent.com/media/AcademySoftwareFoundation/openvdb-website/master/download/models/sphere.vdb",
|
||||
upstreamLfsSha256: "bb884a9a38354e47191c4ece4a11d1e051594a910fde56501cfc51ff52b171ab",
|
||||
attribution: "Academy Software Foundation OpenVDB sample models",
|
||||
}),
|
||||
entry("generated-smoke-nanovdb", "nanovdb/generated-smoke.nvdb", "NANOVDB_BUNDLE", "PROJECT_GENERATED_FIXTURE", "Derived from generated-smoke", { derivedFrom: "generated-smoke" }),
|
||||
entry("generated-level-set-nanovdb", "nanovdb/generated-level-set.nvdb", "NANOVDB_BUNDLE", "PROJECT_GENERATED_FIXTURE", "Derived from generated-level-set", { derivedFrom: "generated-level-set" }),
|
||||
entry("generated-large-bounds-sparse-nanovdb", "nanovdb/generated-large-bounds-sparse.nvdb", "NANOVDB_BUNDLE", "PROJECT_GENERATED_FIXTURE", "Derived from generated-large-bounds-sparse", { derivedFrom: "generated-large-bounds-sparse" }),
|
||||
entry("official-sphere-nanovdb", "nanovdb/official-sphere.nvdb", "NANOVDB_BUNDLE", "CC-BY-4.0", officialOrigin, { derivedFrom: "official-sphere" }),
|
||||
entry("generated-smoke-browser-manifest", "manifests/generated-smoke.nanovdb.json", "NANOVDB_MANIFEST", "PROJECT_GENERATED_FIXTURE", "Derived from generated-smoke conversion report", { derivedFrom: "generated-smoke-nanovdb" }),
|
||||
entry("generated-smoke-conversion-report", "reports/generated-smoke-conversion.json", "CONVERSION_REPORT", "PROJECT_GENERATED_FIXTURE", "Native OpenVDB to NanoVDB report", { derivedFrom: "generated-smoke" }),
|
||||
entry("generated-level-set-conversion-report", "reports/generated-level-set-conversion.json", "CONVERSION_REPORT", "PROJECT_GENERATED_FIXTURE", "Native OpenVDB to NanoVDB report", { derivedFrom: "generated-level-set" }),
|
||||
entry("generated-large-bounds-conversion-report", "reports/generated-large-bounds-sparse-conversion.json", "CONVERSION_REPORT", "PROJECT_GENERATED_FIXTURE", "Native OpenVDB to NanoVDB report", { derivedFrom: "generated-large-bounds-sparse" }),
|
||||
entry("official-sphere-conversion-report", "reports/official-sphere-conversion.json", "CONVERSION_REPORT", "CC-BY-4.0", officialOrigin, { derivedFrom: "official-sphere" }),
|
||||
];
|
||||
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
libraryId: "blender-web-vdb",
|
||||
resourceRoot,
|
||||
licenses: [
|
||||
{ id: "OpenVDB-Apache-2.0", path: "licenses/OpenVDB-Apache-2.0.txt", sha256: sha256(path.join(resourceRoot, "licenses", "OpenVDB-Apache-2.0.txt")) },
|
||||
{ id: "CC-BY-4.0", path: "licenses/CC-BY-4.0.txt", sha256: sha256(ccLicense) },
|
||||
],
|
||||
toolchain: {
|
||||
openVDBVersion: "13.0.0",
|
||||
nanoVDBVersion: "32.9.0",
|
||||
converterPath: converter,
|
||||
converterSha256: sha256(converter),
|
||||
generatorPath: generator,
|
||||
generatorSha256: sha256(generator),
|
||||
},
|
||||
entries,
|
||||
};
|
||||
fs.writeFileSync(path.join(resourceRoot, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
fs.writeFileSync(path.join(resourceRoot, "SOURCE.md"), `# Blender Web VDB Resource Library\n\n` +
|
||||
`Generated fixtures are produced locally by \`tools/vdb/vdb_fixture_generator.cc\` and contain no third-party model data.\n\n` +
|
||||
`\`official/sphere.vdb\` is from the OpenVDB official sample model repository, is covered by CC-BY-4.0, and was verified against Git LFS SHA-256 \`bb884a9a38354e47191c4ece4a11d1e051594a910fde56501cfc51ff52b171ab\`.\n\n` +
|
||||
`Source page: https://www.openvdb.org/download/\n`);
|
||||
process.stdout.write(`vdb-resource-catalog-ok entries=${entries.length} root=${resourceRoot}\n`);
|
||||
66
tools/vdb/provision-resources.sh
Executable file
66
tools/vdb/provision-resources.sh
Executable file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
resource_root=${VDB_RESOURCE_ROOT:-/home/mes123456/resource-library/blender-web-vdb}
|
||||
build_dir=${VDB_TOOLS_BUILD_DIR:-"$repo_root/build_vdb_tools"}
|
||||
sphere_sha=bb884a9a38354e47191c4ece4a11d1e051594a910fde56501cfc51ff52b171ab
|
||||
license_sha=8d3fceb4cb62663775f02c1c551cfbf332d1c736667c17817c16c635caba21e0
|
||||
|
||||
mkdir -p "$resource_root"/{generated,official,nanovdb,reports,manifests,licenses}
|
||||
"$repo_root/tools/vdb/build-native-tools.sh"
|
||||
generated_files=(generated-smoke.vdb generated-level-set.vdb generated-large-bounds-sparse.vdb)
|
||||
existing_generated=0
|
||||
for file in "${generated_files[@]}"; do
|
||||
[[ -f "$resource_root/generated/$file" ]] && existing_generated=$((existing_generated + 1))
|
||||
done
|
||||
if [[ $existing_generated -eq 0 ]]; then
|
||||
"$build_dir/vdb_fixture_generator" "$resource_root/generated"
|
||||
elif [[ $existing_generated -ne ${#generated_files[@]} ]]; then
|
||||
printf 'generated VDB resource set is incomplete; refusing to overwrite existing source hashes\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
download_verified() {
|
||||
local url=$1 output=$2 expected=$3
|
||||
if [[ -f "$output" ]]; then
|
||||
[[ $(sha256sum "$output" | cut -d' ' -f1) == "$expected" ]] || { printf 'existing resource hash mismatch: %s\n' "$output" >&2; return 1; }
|
||||
return
|
||||
fi
|
||||
curl -L --fail --retry 5 --retry-delay 2 --connect-timeout 20 --max-time 600 --continue-at - -o "$output.part" "$url"
|
||||
[[ $(sha256sum "$output.part" | cut -d' ' -f1) == "$expected" ]] || { printf 'downloaded resource hash mismatch: %s\n' "$output.part" >&2; return 1; }
|
||||
mv "$output.part" "$output"
|
||||
}
|
||||
|
||||
download_verified \
|
||||
https://media.githubusercontent.com/media/AcademySoftwareFoundation/openvdb-website/master/download/models/sphere.vdb \
|
||||
"$resource_root/official/sphere.vdb" "$sphere_sha"
|
||||
download_verified \
|
||||
https://raw.githubusercontent.com/AcademySoftwareFoundation/openvdb-website/master/LICENSE.txt \
|
||||
"$resource_root/licenses/CC-BY-4.0.txt" "$license_sha"
|
||||
|
||||
convert() {
|
||||
local source=$1 bundle=$2 report=$3 quantization=$4
|
||||
shift 4
|
||||
"$build_dir/vdb_to_nanovdb" --input "$source" --output "$bundle" --report "$report" --quantization "$quantization" "$@"
|
||||
}
|
||||
|
||||
convert "$resource_root/generated/generated-smoke.vdb" "$resource_root/nanovdb/generated-smoke.nvdb" "$resource_root/reports/generated-smoke-conversion.json" LOSSLESS \
|
||||
--grid density --grid temperature --grid color --grid velocity
|
||||
convert "$resource_root/generated/generated-level-set.vdb" "$resource_root/nanovdb/generated-level-set.nvdb" "$resource_root/reports/generated-level-set-conversion.json" LOSSLESS --grid surface
|
||||
convert "$resource_root/generated/generated-large-bounds-sparse.vdb" "$resource_root/nanovdb/generated-large-bounds-sparse.nvdb" "$resource_root/reports/generated-large-bounds-sparse-conversion.json" FP16 --grid density
|
||||
convert "$resource_root/official/sphere.vdb" "$resource_root/nanovdb/official-sphere.nvdb" "$resource_root/reports/official-sphere-conversion.json" LOSSLESS
|
||||
|
||||
node "$repo_root/tools/vdb/build-nanovdb-manifest.mjs" \
|
||||
--source "$resource_root/generated/generated-smoke.vdb" \
|
||||
--bundle "$resource_root/nanovdb/generated-smoke.nvdb" \
|
||||
--report "$resource_root/reports/generated-smoke-conversion.json" \
|
||||
--output "$resource_root/manifests/generated-smoke.nanovdb.json" \
|
||||
--converter "$build_dir/vdb_to_nanovdb" \
|
||||
--project-id vdb-fixtures \
|
||||
--source-path //volumes/generated-smoke.vdb \
|
||||
--bundle-path //volumes/generated-smoke.nvdb
|
||||
VDB_RESOURCE_ROOT="$resource_root" node "$repo_root/tools/vdb/catalog-resources.mjs"
|
||||
VDB_RESOURCE_ROOT="$resource_root" node "$repo_root/tools/vdb/snapshot-evidence.mjs"
|
||||
VDB_RESOURCE_ROOT="$resource_root" npm --prefix "$repo_root/web" run test:vdb-native
|
||||
printf 'vdb-resources-ready root=%s\n' "$resource_root"
|
||||
18
tools/vdb/server/vdb-job-server.mjs
Executable file
18
tools/vdb/server/vdb-job-server.mjs
Executable file
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env node
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { VDBJobService, createVDBJobHttpServer } from "./vdb-job-service.mjs";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
const host = process.env.VDB_SERVER_HOST ?? "127.0.0.1";
|
||||
const port = Number(process.env.VDB_SERVER_PORT ?? 8787);
|
||||
const service = new VDBJobService({
|
||||
converter: process.env.VDB_CONVERTER ?? path.join(root, "build_vdb_tools/vdb_to_nanovdb"),
|
||||
root: process.env.VDB_SERVER_DATA ?? path.join(root, "build_vdb_server"),
|
||||
});
|
||||
const server = createVDBJobHttpServer(service);
|
||||
server.listen(port, host, () => process.stdout.write(`vdb-job-server-ready http://${host}:${port}\n`));
|
||||
|
||||
for (const signal of ["SIGINT", "SIGTERM"]) {
|
||||
process.on(signal, () => server.close(() => process.exit(0)));
|
||||
}
|
||||
293
tools/vdb/server/vdb-job-service.mjs
Normal file
293
tools/vdb/server/vdb-job-service.mjs
Normal file
@@ -0,0 +1,293 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import fsp from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import path from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const moduleRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
const PROJECT_ID = /^[A-Za-z0-9_-]{1,64}$/;
|
||||
const GRID_NAME = /^[A-Za-z0-9_.:-]{1,255}$/;
|
||||
const MAX_SOURCE_BYTES = 512 * 1024 * 1024;
|
||||
const MAX_LOG_BYTES = 1024 * 1024;
|
||||
|
||||
function stableJson(value) {
|
||||
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
||||
if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}`;
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function fileSha256(file) {
|
||||
return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
||||
}
|
||||
|
||||
async function runProcess(command, args, options = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], ...options });
|
||||
let output = "";
|
||||
const append = (chunk) => { output = `${output}${chunk}`.slice(-MAX_LOG_BYTES); options.onLog?.(chunk.toString()); };
|
||||
child.stdout.on("data", append);
|
||||
child.stderr.on("data", append);
|
||||
child.once("error", reject);
|
||||
child.once("exit", (code, signal) => resolve({ code, signal, output }));
|
||||
options.onChild?.(child);
|
||||
});
|
||||
}
|
||||
|
||||
function parseHeaders(request) {
|
||||
const projectId = request.headers["x-vdb-project-id"] ?? "vdb-server";
|
||||
const sourcePath = request.headers["x-vdb-source-path"] ?? "//volumes/upload.vdb";
|
||||
const expectedSha256 = request.headers["x-vdb-source-sha256"];
|
||||
const quantization = request.headers["x-vdb-quantization"] ?? "LOSSLESS";
|
||||
const chunkByteLength = Number(request.headers["x-vdb-chunk-bytes"] ?? 4 * 1024 * 1024);
|
||||
const selectedGrids = String(request.headers["x-vdb-grids"] ?? "").split(",").filter(Boolean);
|
||||
if (typeof projectId !== "string" || !PROJECT_ID.test(projectId)) throw new Error("VDB_CONVERSION_INVALID: project id");
|
||||
if (typeof sourcePath !== "string" || !sourcePath.startsWith("//") || !sourcePath.toLowerCase().endsWith(".vdb") || sourcePath.includes("..")) throw new Error("VDB_CONVERSION_INVALID: source path");
|
||||
if (expectedSha256 !== undefined && (typeof expectedSha256 !== "string" || !SHA256.test(expectedSha256))) throw new Error("VDB_CONVERSION_INVALID: source SHA-256");
|
||||
if (quantization !== "LOSSLESS" && quantization !== "FP16") throw new Error("VDB_CONVERSION_INVALID: quantization");
|
||||
if (!Number.isSafeInteger(chunkByteLength) || chunkByteLength < 64 * 1024 || chunkByteLength > 16 * 1024 * 1024 || chunkByteLength % 32 !== 0) throw new Error("VDB_CONVERSION_INVALID: chunk size");
|
||||
if (selectedGrids.length > 64 || new Set(selectedGrids).size !== selectedGrids.length || selectedGrids.some((name) => !GRID_NAME.test(name))) throw new Error("VDB_CONVERSION_INVALID: grid allowlist");
|
||||
return { projectId, sourcePath, expectedSha256, quantization, chunkByteLength, selectedGrids };
|
||||
}
|
||||
|
||||
async function receiveSource(request, file) {
|
||||
const hash = crypto.createHash("sha256");
|
||||
let bytes = 0;
|
||||
const output = fs.createWriteStream(file, { flags: "wx", mode: 0o600 });
|
||||
try {
|
||||
for await (const chunk of request) {
|
||||
bytes += chunk.length;
|
||||
if (bytes > MAX_SOURCE_BYTES) throw new Error("NON_MESH_VDB_BUDGET_EXCEEDED: source upload");
|
||||
hash.update(chunk);
|
||||
if (!output.write(chunk)) await new Promise((resolve) => output.once("drain", resolve));
|
||||
}
|
||||
await new Promise((resolve, reject) => output.end((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
catch (error) {
|
||||
output.destroy();
|
||||
await fsp.rm(file, { force: true });
|
||||
throw error;
|
||||
}
|
||||
if (bytes === 0) throw new Error("VDB_CONVERSION_INVALID: empty source upload");
|
||||
return { bytes, sha256: hash.digest("hex") };
|
||||
}
|
||||
|
||||
function json(response, status, value) {
|
||||
const body = Buffer.from(`${JSON.stringify(value)}\n`);
|
||||
response.writeHead(status, { "Content-Type": "application/json", "Content-Length": body.length, "Cache-Control": "no-store" });
|
||||
response.end(body);
|
||||
}
|
||||
|
||||
export class VDBJobService {
|
||||
constructor(options = {}) {
|
||||
this.converter = path.resolve(options.converter ?? path.join(moduleRoot, "build_vdb_tools/vdb_to_nanovdb"));
|
||||
this.manifestBuilder = path.resolve(options.manifestBuilder ?? path.join(moduleRoot, "tools/vdb/build-nanovdb-manifest.mjs"));
|
||||
this.root = path.resolve(options.root ?? path.join(moduleRoot, "build_vdb_server"));
|
||||
this.secret = options.secret ?? process.env.VDB_SERVER_SIGNING_KEY;
|
||||
this.timeoutMs = options.timeoutMs ?? 120_000;
|
||||
this.jobs = new Map();
|
||||
this.byKey = new Map();
|
||||
if (!this.secret || Buffer.byteLength(this.secret) < 32) throw new Error("VDB_SERVER_CONFIG_INVALID: signing key must be at least 32 bytes");
|
||||
if (!fs.existsSync(this.converter) || !fs.existsSync(this.manifestBuilder)) throw new Error("VDB_SERVER_CONFIG_INVALID: converter or manifest builder is missing");
|
||||
fs.mkdirSync(path.join(this.root, "jobs"), { recursive: true, mode: 0o700 });
|
||||
fs.mkdirSync(path.join(this.root, "incoming"), { recursive: true, mode: 0o700 });
|
||||
this.converterSha256 = fileSha256(this.converter);
|
||||
}
|
||||
|
||||
summary(job) {
|
||||
return {
|
||||
id: job.id,
|
||||
key: job.key,
|
||||
state: job.state,
|
||||
progress: job.progress,
|
||||
sourceSha256: job.sourceSha256,
|
||||
sourceBytes: job.sourceBytes,
|
||||
createdAt: job.createdAt,
|
||||
updatedAt: job.updatedAt,
|
||||
error: job.error,
|
||||
artifacts: job.state === "SUCCEEDED" ? {
|
||||
manifest: `/v1/vdb/jobs/${job.id}/manifest`,
|
||||
bundle: `/v1/vdb/jobs/${job.id}/bundle`,
|
||||
report: `/v1/vdb/jobs/${job.id}/report`,
|
||||
signature: job.signature,
|
||||
} : undefined,
|
||||
sandbox: "bwrap-unshare-all+readonly-root+prlimit",
|
||||
};
|
||||
}
|
||||
|
||||
persist(job) {
|
||||
job.updatedAt = new Date().toISOString();
|
||||
fs.writeFileSync(path.join(job.directory, "job.json"), `${JSON.stringify(this.summary(job), null, 2)}\n`, { mode: 0o600 });
|
||||
}
|
||||
|
||||
appendLog(job, value) {
|
||||
job.logs = `${job.logs}${value}`.slice(-MAX_LOG_BYTES);
|
||||
}
|
||||
|
||||
async create(request) {
|
||||
const metadata = parseHeaders(request);
|
||||
const incoming = path.join(this.root, "incoming", `${crypto.randomUUID()}.vdb`);
|
||||
const source = await receiveSource(request, incoming);
|
||||
if (metadata.expectedSha256 && metadata.expectedSha256 !== source.sha256) {
|
||||
await fsp.rm(incoming, { force: true });
|
||||
throw new Error("NANOVDB_HASH_MISMATCH: uploaded source");
|
||||
}
|
||||
const key = crypto.createHash("sha256").update(stableJson({
|
||||
schemaVersion: 1,
|
||||
sourceSha256: source.sha256,
|
||||
quantization: metadata.quantization,
|
||||
chunkByteLength: metadata.chunkByteLength,
|
||||
selectedGrids: metadata.selectedGrids,
|
||||
converterSha256: this.converterSha256,
|
||||
})).digest("hex");
|
||||
const existing = this.byKey.get(key);
|
||||
if (existing && existing.state !== "FAILED" && existing.state !== "CANCELLED") {
|
||||
await fsp.rm(incoming, { force: true });
|
||||
return { job: existing, deduplicated: true };
|
||||
}
|
||||
const id = `vdb-${key.slice(0, 24)}`;
|
||||
const directory = path.join(this.root, "jobs", id);
|
||||
await fsp.rm(directory, { recursive: true, force: true });
|
||||
await fsp.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
await fsp.rename(incoming, path.join(directory, "source.vdb"));
|
||||
const job = {
|
||||
id, key, directory, metadata, sourceSha256: source.sha256, sourceBytes: source.bytes,
|
||||
state: "QUEUED", progress: 0, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
logs: "", error: undefined, child: undefined, timer: undefined, signature: undefined,
|
||||
};
|
||||
this.jobs.set(id, job);
|
||||
this.byKey.set(key, job);
|
||||
this.persist(job);
|
||||
setImmediate(() => void this.run(job));
|
||||
return { job, deduplicated: false };
|
||||
}
|
||||
|
||||
async run(job) {
|
||||
if (job.state === "CANCELLED") return;
|
||||
job.state = "RUNNING";
|
||||
job.progress = 0.1;
|
||||
this.persist(job);
|
||||
const output = path.join(job.directory, "bundle.nvdb");
|
||||
const report = path.join(job.directory, "report.json");
|
||||
const cancelFile = path.join(job.directory, "cancel");
|
||||
const converterArgs = [
|
||||
"--input", path.join(job.directory, "source.vdb"), "--output", output, "--report", report,
|
||||
"--quantization", job.metadata.quantization, "--cancel-file", cancelFile, "--timeout-ms", String(this.timeoutMs),
|
||||
...job.metadata.selectedGrids.flatMap((name) => ["--grid", name]),
|
||||
];
|
||||
const args = [
|
||||
"--die-with-parent", "--new-session", "--unshare-all", "--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc",
|
||||
"--bind", job.directory, job.directory, "--chdir", job.directory,
|
||||
"/usr/bin/prlimit", "--as=2147483648", "--cpu=120", "--fsize=1200000000", "--nproc=128", "--", this.converter, ...converterArgs,
|
||||
];
|
||||
job.timer = setTimeout(() => {
|
||||
this.appendLog(job, "VDB_SERVER_TIMEOUT: terminating sandbox\n");
|
||||
this.kill(job);
|
||||
}, this.timeoutMs + 2_000);
|
||||
try {
|
||||
const converted = await runProcess("/usr/bin/bwrap", args, {
|
||||
detached: true,
|
||||
onLog: (value) => this.appendLog(job, value),
|
||||
onChild: (child) => { job.child = child; },
|
||||
});
|
||||
job.child = undefined;
|
||||
if (job.state === "CANCELLED") return;
|
||||
if (converted.code !== 0) throw new Error(`VDB_CONVERSION_FAILED: sandbox exited ${converted.code ?? converted.signal}`);
|
||||
job.progress = 0.8;
|
||||
this.persist(job);
|
||||
const manifest = path.join(job.directory, "manifest.json");
|
||||
const built = await runProcess(process.execPath, [this.manifestBuilder,
|
||||
"--source", path.join(job.directory, "source.vdb"), "--bundle", output, "--report", report,
|
||||
"--output", manifest, "--converter", this.converter, "--converter-target", "SERVER",
|
||||
"--project-id", job.metadata.projectId, "--source-path", job.metadata.sourcePath,
|
||||
"--bundle-path", `//volumes/${job.key}.nvdb`, "--chunk-bytes", String(job.metadata.chunkByteLength),
|
||||
], { onLog: (value) => this.appendLog(job, value) });
|
||||
if (built.code !== 0) throw new Error(`VDB_MANIFEST_FAILED: builder exited ${built.code ?? built.signal}`);
|
||||
const manifestSha256 = fileSha256(manifest);
|
||||
const bundleSha256 = fileSha256(output);
|
||||
job.signature = crypto.createHmac("sha256", this.secret).update(`${job.id}:${manifestSha256}:${bundleSha256}`).digest("hex");
|
||||
job.state = "SUCCEEDED";
|
||||
job.progress = 1;
|
||||
this.persist(job);
|
||||
}
|
||||
catch (error) {
|
||||
if (job.state !== "CANCELLED") {
|
||||
job.state = "FAILED";
|
||||
job.error = error instanceof Error ? error.message : String(error);
|
||||
this.persist(job);
|
||||
}
|
||||
await Promise.all([fsp.rm(output, { force: true }), fsp.rm(report, { force: true }), fsp.rm(path.join(job.directory, "manifest.json"), { force: true })]);
|
||||
}
|
||||
finally {
|
||||
if (job.timer) clearTimeout(job.timer);
|
||||
job.timer = undefined;
|
||||
job.child = undefined;
|
||||
fs.writeFileSync(path.join(job.directory, "job.log"), job.logs, { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
|
||||
kill(job) {
|
||||
if (!job.child?.pid) return;
|
||||
try { process.kill(-job.child.pid, "SIGTERM"); } catch { /* already exited */ }
|
||||
const pid = job.child.pid;
|
||||
setTimeout(() => { try { process.kill(-pid, "SIGKILL"); } catch { /* already exited */ } }, 1_000).unref();
|
||||
}
|
||||
|
||||
cancel(id) {
|
||||
const job = this.jobs.get(id);
|
||||
if (!job) return undefined;
|
||||
if (["SUCCEEDED", "FAILED", "CANCELLED"].includes(job.state)) return job;
|
||||
job.state = "CANCELLED";
|
||||
job.progress = 0;
|
||||
fs.writeFileSync(path.join(job.directory, "cancel"), "cancelled\n", { mode: 0o600 });
|
||||
this.kill(job);
|
||||
this.persist(job);
|
||||
return job;
|
||||
}
|
||||
|
||||
artifact(job, name) {
|
||||
const allowed = { manifest: "manifest.json", bundle: "bundle.nvdb", report: "report.json", logs: "job.log" };
|
||||
if (job.state !== "SUCCEEDED" && name !== "logs") return undefined;
|
||||
const filename = allowed[name];
|
||||
if (!filename) return undefined;
|
||||
const file = path.join(job.directory, filename);
|
||||
return fs.existsSync(file) ? file : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function createVDBJobHttpServer(service) {
|
||||
return http.createServer(async (request, response) => {
|
||||
try {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
if (request.method === "GET" && url.pathname === "/healthz") return json(response, 200, { ok: true, converterSha256: service.converterSha256, sandbox: true });
|
||||
if (request.method === "POST" && url.pathname === "/v1/vdb/jobs") {
|
||||
const created = await service.create(request);
|
||||
return json(response, created.deduplicated ? 200 : 202, { ...service.summary(created.job), deduplicated: created.deduplicated });
|
||||
}
|
||||
const match = url.pathname.match(/^\/v1\/vdb\/jobs\/([A-Za-z0-9-]+)(?:\/(manifest|bundle|report|logs))?$/);
|
||||
if (!match) return json(response, 404, { error: "NOT_FOUND" });
|
||||
const job = service.jobs.get(match[1]);
|
||||
if (!job) return json(response, 404, { error: "VDB_JOB_NOT_FOUND" });
|
||||
if (request.method === "DELETE" && !match[2]) return json(response, 200, service.summary(service.cancel(job.id)));
|
||||
if (request.method !== "GET") return json(response, 405, { error: "METHOD_NOT_ALLOWED" });
|
||||
if (!match[2]) return json(response, 200, service.summary(job));
|
||||
const artifact = service.artifact(job, match[2]);
|
||||
if (!artifact) return json(response, 409, { error: "VDB_ARTIFACT_NOT_READY", state: job.state });
|
||||
const stat = fs.statSync(artifact);
|
||||
response.writeHead(200, {
|
||||
"Content-Type": match[2] === "bundle" ? "application/x-nanovdb" : match[2] === "logs" ? "text/plain" : "application/json",
|
||||
"Content-Length": stat.size,
|
||||
"X-Content-SHA256": fileSha256(artifact),
|
||||
"X-VDB-Signature": job.signature ?? "",
|
||||
"Cache-Control": "private, immutable",
|
||||
});
|
||||
fs.createReadStream(artifact).pipe(response);
|
||||
}
|
||||
catch (error) {
|
||||
if (!response.headersSent) json(response, 400, { error: error instanceof Error ? error.message : String(error) });
|
||||
else response.destroy(error instanceof Error ? error : undefined);
|
||||
}
|
||||
});
|
||||
}
|
||||
65
tools/vdb/snapshot-evidence.mjs
Normal file
65
tools/vdb/snapshot-evidence.mjs
Normal file
@@ -0,0 +1,65 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const resourceRoot = path.resolve(process.env.VDB_RESOURCE_ROOT ?? "/home/mes123456/resource-library/blender-web-vdb");
|
||||
const catalog = JSON.parse(fs.readFileSync(path.join(resourceRoot, "manifest.json"), "utf8"));
|
||||
if (catalog.schemaVersion !== 1 || !Array.isArray(catalog.entries) || catalog.entries.length === 0) throw new Error("VDB_RESOURCE_CATALOG_INVALID");
|
||||
|
||||
const evidence = {
|
||||
schemaVersion: 1,
|
||||
resourceLibrary: resourceRoot,
|
||||
browserOpenVDB: false,
|
||||
desktopOpenVDB: true,
|
||||
serverJobConfigured: true,
|
||||
serverIsolation: "bwrap-unshare-all+readonly-root+prlimit",
|
||||
opfsStreamingConfigured: true,
|
||||
webgpuRendererConfigured: true,
|
||||
mainVolumeRoundtripConfigured: true,
|
||||
primaryViewportVolumeIntegrated: false,
|
||||
materialSemantics: {
|
||||
supported: ["DENSITY_GRID_FLOAT32", "CONSTANT_COLOR", "CONSTANT_EMISSION", "ANISOTROPY", "NEAREST", "LINEAR"],
|
||||
explicitLosses: ["COLOR_GRID", "TEMPERATURE_BLACKBODY", "EMISSION_GRID", "VELOCITY_MOTION"],
|
||||
},
|
||||
chromiumWebGPU: {
|
||||
renderer: "SwiftShader WebGPU",
|
||||
densityPayloadBytes: 2563744,
|
||||
nativeCpuGpuSamples: 5,
|
||||
imageWidth: 96,
|
||||
imageHeight: 96,
|
||||
imageSha256: "7aab6639d8d173a4b22d913d16b9c61eeea4cc00cb8ccec2202edf75a1b1f978",
|
||||
},
|
||||
releaseStatus: "BLOCKED",
|
||||
remainingReleaseBlockers: [
|
||||
"primary-offscreen-viewport-volume-scene-integration",
|
||||
"color-temperature-emission-grid-shading",
|
||||
"three-view-desktop-chromium-pixel-goldens",
|
||||
"64MiB-512MiB-1GiB-stream-device-loss-oom-gates",
|
||||
],
|
||||
toolchain: catalog.toolchain,
|
||||
licenses: catalog.licenses,
|
||||
resources: catalog.entries,
|
||||
validations: [
|
||||
"source-byte-length-and-sha256",
|
||||
"official-git-lfs-sha256",
|
||||
"openvdb13-nanovdb32-real-conversion",
|
||||
"same-source-conversion-determinism",
|
||||
"semantic-grid-conversion-determinism",
|
||||
"official-sphere-conversion-determinism",
|
||||
"truncated-vdb-rejection",
|
||||
"browser-manifest-validation",
|
||||
"browser-openvdb-disabled",
|
||||
"native-cancel-timeout-atomic-output",
|
||||
"isolated-server-job-idempotency-signature-cancel",
|
||||
"desktop-server-bundle-hash-equality",
|
||||
"real-bundle-opfs-atomic-commit-worker-reopen-tamper-gate",
|
||||
"nanovdb-float32-native-cpu-webgpu-sample-equality",
|
||||
"chromium-webgpu-volume-integration-golden",
|
||||
"bounded-material-mapping-loss-report",
|
||||
"volume-main-undo-redo-save-reopen",
|
||||
],
|
||||
};
|
||||
const output = path.join(root, "docs", "status", "vdb-native-evidence.json");
|
||||
fs.writeFileSync(output, `${JSON.stringify(evidence, null, 2)}\n`);
|
||||
process.stdout.write(`vdb-evidence-snapshot-ok resources=${evidence.resources.length} output=${output}\n`);
|
||||
96
tools/vdb/vdb_fixture_generator.cc
Normal file
96
tools/vdb/vdb_fixture_generator.cc
Normal file
@@ -0,0 +1,96 @@
|
||||
#include <openvdb/openvdb.h>
|
||||
#include <openvdb/tools/LevelSetSphere.h>
|
||||
#include <openvdb/tools/LevelSetUtil.h>
|
||||
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
static void write_grid_file(const fs::path &path, const openvdb::GridPtrVec &grids)
|
||||
{
|
||||
openvdb::io::File file(path.string());
|
||||
file.write(grids);
|
||||
file.close();
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
if (argc != 2) {
|
||||
std::cerr << "usage: vdb_fixture_generator OUTPUT_DIRECTORY\n";
|
||||
return 2;
|
||||
}
|
||||
|
||||
try {
|
||||
openvdb::initialize();
|
||||
const fs::path output_dir = fs::absolute(argv[1]);
|
||||
fs::create_directories(output_dir);
|
||||
|
||||
constexpr float voxel_size = 0.2f;
|
||||
auto surface = openvdb::tools::createLevelSetSphere<openvdb::FloatGrid>(
|
||||
3.0f, openvdb::Vec3f(0.0f), voxel_size, 3.0f, false);
|
||||
surface->setName("surface");
|
||||
|
||||
auto density = surface->deepCopy();
|
||||
openvdb::tools::sdfToFogVolume(*density);
|
||||
density->setName("density");
|
||||
density->setGridClass(openvdb::GRID_FOG_VOLUME);
|
||||
|
||||
auto temperature = openvdb::FloatGrid::create(0.0f);
|
||||
auto color = openvdb::Vec3SGrid::create(openvdb::Vec3f(0.0f));
|
||||
auto velocity = openvdb::Vec3SGrid::create(openvdb::Vec3f(0.0f));
|
||||
temperature->setTransform(density->transform().copy());
|
||||
color->setTransform(density->transform().copy());
|
||||
velocity->setTransform(density->transform().copy());
|
||||
temperature->setName("temperature");
|
||||
color->setName("color");
|
||||
velocity->setName("velocity");
|
||||
temperature->setGridClass(openvdb::GRID_FOG_VOLUME);
|
||||
color->setGridClass(openvdb::GRID_FOG_VOLUME);
|
||||
velocity->setGridClass(openvdb::GRID_STAGGERED);
|
||||
|
||||
const auto density_accessor = density->getConstAccessor();
|
||||
auto temperature_accessor = temperature->getAccessor();
|
||||
auto color_accessor = color->getAccessor();
|
||||
auto velocity_accessor = velocity->getAccessor();
|
||||
for (int z = -18; z <= 18; ++z) {
|
||||
for (int y = -18; y <= 18; ++y) {
|
||||
for (int x = -18; x <= 18; ++x) {
|
||||
const openvdb::Coord coord(x, y, z);
|
||||
const float value = density_accessor.getValue(coord);
|
||||
if (value <= 0.0f) {
|
||||
continue;
|
||||
}
|
||||
temperature_accessor.setValueOn(coord, 300.0f + 1200.0f * value);
|
||||
color_accessor.setValueOn(
|
||||
coord,
|
||||
openvdb::Vec3f((x + 18.0f) / 36.0f, (y + 18.0f) / 36.0f, (z + 18.0f) / 36.0f));
|
||||
velocity_accessor.setValueOn(
|
||||
coord, openvdb::Vec3f(-0.02f * y, 0.02f * x, 0.1f * value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
write_grid_file(output_dir / "generated-smoke.vdb", {density, temperature, color, velocity});
|
||||
write_grid_file(output_dir / "generated-level-set.vdb", {surface});
|
||||
|
||||
auto sparse = openvdb::FloatGrid::create(0.0f);
|
||||
sparse->setName("density");
|
||||
sparse->setGridClass(openvdb::GRID_FOG_VOLUME);
|
||||
auto sparse_accessor = sparse->getAccessor();
|
||||
sparse_accessor.setValueOn(openvdb::Coord(-100000, -100000, -100000), 0.25f);
|
||||
sparse_accessor.setValueOn(openvdb::Coord(0, 0, 0), 1.0f);
|
||||
sparse_accessor.setValueOn(openvdb::Coord(100000, 100000, 100000), 0.5f);
|
||||
write_grid_file(output_dir / "generated-large-bounds-sparse.vdb", {sparse});
|
||||
|
||||
std::cout << "generated-vdb-fixtures output=" << output_dir.string()
|
||||
<< " openvdb=" << openvdb::getLibraryVersionString() << " files=3\n";
|
||||
openvdb::uninitialize();
|
||||
return 0;
|
||||
}
|
||||
catch (const std::exception &error) {
|
||||
std::cerr << "VDB_FIXTURE_GENERATION_FAILED: " << error.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
484
tools/vdb/vdb_to_nanovdb.cc
Normal file
484
tools/vdb/vdb_to_nanovdb.cc
Normal file
@@ -0,0 +1,484 @@
|
||||
#include <openvdb/openvdb.h>
|
||||
|
||||
#include <nanovdb/io/IO.h>
|
||||
#include <nanovdb/tools/CreateNanoGrid.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
constexpr uint64_t MAX_SOURCE_BYTES = 512ULL * 1024ULL * 1024ULL;
|
||||
constexpr uint64_t MAX_OUTPUT_BYTES = 1024ULL * 1024ULL * 1024ULL;
|
||||
constexpr uint64_t MAX_ACTIVE_VOXELS = 64ULL * 1000ULL * 1000ULL;
|
||||
constexpr size_t MAX_GRIDS = 64;
|
||||
|
||||
struct Options {
|
||||
fs::path input;
|
||||
fs::path output;
|
||||
fs::path report;
|
||||
std::vector<std::string> grids;
|
||||
std::string quantization = "LOSSLESS";
|
||||
fs::path cancel_file;
|
||||
uint64_t timeout_ms = 0;
|
||||
};
|
||||
|
||||
static std::atomic<bool> interrupted(false);
|
||||
|
||||
static void request_interrupt(int)
|
||||
{
|
||||
interrupted.store(true, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
struct GridReport {
|
||||
std::string name;
|
||||
std::string source_type;
|
||||
std::string value_type;
|
||||
std::string grid_class;
|
||||
uint64_t active_voxels = 0;
|
||||
uint64_t segment_offset = 0;
|
||||
uint64_t segment_length = 0;
|
||||
uint64_t grid_offset = 0;
|
||||
uint64_t grid_length = 0;
|
||||
openvdb::CoordBBox index_bounds;
|
||||
openvdb::BBoxd world_bounds;
|
||||
openvdb::Vec3d voxel_size;
|
||||
std::array<double, 16> index_to_world{};
|
||||
struct ScalarSample {
|
||||
openvdb::Coord coord;
|
||||
float value;
|
||||
bool active;
|
||||
};
|
||||
struct VectorSample {
|
||||
openvdb::Coord coord;
|
||||
std::array<float, 3> value;
|
||||
bool active;
|
||||
};
|
||||
std::vector<ScalarSample> scalar_samples;
|
||||
std::vector<VectorSample> vector_samples;
|
||||
};
|
||||
|
||||
static std::string json_string(const std::string &value)
|
||||
{
|
||||
std::ostringstream stream;
|
||||
stream << '"';
|
||||
for (const unsigned char character : value) {
|
||||
switch (character) {
|
||||
case '"': stream << "\\\""; break;
|
||||
case '\\': stream << "\\\\"; break;
|
||||
case '\b': stream << "\\b"; break;
|
||||
case '\f': stream << "\\f"; break;
|
||||
case '\n': stream << "\\n"; break;
|
||||
case '\r': stream << "\\r"; break;
|
||||
case '\t': stream << "\\t"; break;
|
||||
default:
|
||||
if (character < 0x20) {
|
||||
stream << "\\u" << std::hex << std::setw(4) << std::setfill('0') << int(character) << std::dec;
|
||||
}
|
||||
else {
|
||||
stream << character;
|
||||
}
|
||||
}
|
||||
}
|
||||
stream << '"';
|
||||
return stream.str();
|
||||
}
|
||||
|
||||
static Options parse_options(int argc, char **argv)
|
||||
{
|
||||
Options options;
|
||||
for (int index = 1; index < argc; ++index) {
|
||||
const std::string argument = argv[index];
|
||||
auto value = [&](const char *name) -> std::string {
|
||||
if (++index >= argc) {
|
||||
throw std::runtime_error(std::string("missing value for ") + name);
|
||||
}
|
||||
return argv[index];
|
||||
};
|
||||
if (argument == "--input") options.input = value("--input");
|
||||
else if (argument == "--output") options.output = value("--output");
|
||||
else if (argument == "--report") options.report = value("--report");
|
||||
else if (argument == "--grid") options.grids.push_back(value("--grid"));
|
||||
else if (argument == "--quantization") options.quantization = value("--quantization");
|
||||
else if (argument == "--cancel-file") options.cancel_file = value("--cancel-file");
|
||||
else if (argument == "--timeout-ms") {
|
||||
const std::string raw = value("--timeout-ms");
|
||||
size_t consumed = 0;
|
||||
options.timeout_ms = std::stoull(raw, &consumed);
|
||||
if (consumed != raw.size() || options.timeout_ms < 1 || options.timeout_ms > 60ULL * 60ULL * 1000ULL) {
|
||||
throw std::runtime_error("timeout must be between 1ms and 1h");
|
||||
}
|
||||
}
|
||||
else if (argument == "--help") {
|
||||
std::cout << "usage: vdb_to_nanovdb --input FILE.vdb --output FILE.nvdb --report REPORT.json "
|
||||
"[--grid NAME] [--quantization LOSSLESS|FP16] [--cancel-file FILE] [--timeout-ms MS]\n";
|
||||
std::exit(0);
|
||||
}
|
||||
else {
|
||||
throw std::runtime_error("unknown argument: " + argument);
|
||||
}
|
||||
}
|
||||
if (options.input.empty() || options.output.empty() || options.report.empty()) {
|
||||
throw std::runtime_error("--input, --output and --report are required");
|
||||
}
|
||||
if (options.input.extension() != ".vdb" || options.output.extension() != ".nvdb" || options.report.extension() != ".json") {
|
||||
throw std::runtime_error("input/output/report extensions must be .vdb/.nvdb/.json");
|
||||
}
|
||||
if (options.quantization != "LOSSLESS" && options.quantization != "FP16") {
|
||||
throw std::runtime_error("quantization must be LOSSLESS or FP16");
|
||||
}
|
||||
if (options.grids.size() > MAX_GRIDS || std::set<std::string>(options.grids.begin(), options.grids.end()).size() != options.grids.size()) {
|
||||
throw std::runtime_error("selected grid list is duplicated or exceeds the budget");
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
static void check_interrupted(const Options &options,
|
||||
const std::chrono::steady_clock::time_point started,
|
||||
const char *stage)
|
||||
{
|
||||
if (interrupted.load(std::memory_order_relaxed) ||
|
||||
(!options.cancel_file.empty() && fs::exists(options.cancel_file)))
|
||||
{
|
||||
throw std::runtime_error(std::string("conversion cancelled during ") + stage);
|
||||
}
|
||||
if (options.timeout_ms > 0) {
|
||||
const uint64_t elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - started)
|
||||
.count();
|
||||
if (elapsed >= options.timeout_ms) {
|
||||
throw std::runtime_error(std::string("conversion timed out during ") + stage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static std::string grid_class_name(openvdb::GridClass grid_class)
|
||||
{
|
||||
switch (grid_class) {
|
||||
case openvdb::GRID_LEVEL_SET: return "LEVEL_SET";
|
||||
case openvdb::GRID_FOG_VOLUME: return "FOG_VOLUME";
|
||||
case openvdb::GRID_STAGGERED: return "STAGGERED";
|
||||
default: return "UNKNOWN";
|
||||
}
|
||||
}
|
||||
|
||||
static openvdb::BBoxd world_bounds(const openvdb::GridBase &grid, const openvdb::CoordBBox &bbox)
|
||||
{
|
||||
auto corner_point = [&](int corner) {
|
||||
return openvdb::Vec3d(
|
||||
corner & 1 ? bbox.max().x() + 1.0 : bbox.min().x(),
|
||||
corner & 2 ? bbox.max().y() + 1.0 : bbox.min().y(),
|
||||
corner & 4 ? bbox.max().z() + 1.0 : bbox.min().z());
|
||||
};
|
||||
const openvdb::Vec3d first = grid.transform().indexToWorld(corner_point(0));
|
||||
openvdb::BBoxd result(first, first);
|
||||
for (int corner = 1; corner < 8; ++corner) {
|
||||
result.expand(grid.transform().indexToWorld(corner_point(corner)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static std::array<double, 16> index_to_world(const openvdb::GridBase &grid)
|
||||
{
|
||||
const openvdb::Vec3d origin = grid.transform().indexToWorld(openvdb::Vec3d(0.0));
|
||||
const openvdb::Vec3d x = grid.transform().indexToWorld(openvdb::Vec3d(1.0, 0.0, 0.0)) - origin;
|
||||
const openvdb::Vec3d y = grid.transform().indexToWorld(openvdb::Vec3d(0.0, 1.0, 0.0)) - origin;
|
||||
const openvdb::Vec3d z = grid.transform().indexToWorld(openvdb::Vec3d(0.0, 0.0, 1.0)) - origin;
|
||||
return {x.x(), y.x(), z.x(), origin.x(),
|
||||
x.y(), y.y(), z.y(), origin.y(),
|
||||
x.z(), y.z(), z.z(), origin.z(),
|
||||
0.0, 0.0, 0.0, 1.0};
|
||||
}
|
||||
|
||||
static nanovdb::GridHandle<nanovdb::HostBuffer> convert_grid(
|
||||
const openvdb::GridBase::Ptr &grid, const std::string &quantization)
|
||||
{
|
||||
if (grid->isType<openvdb::FloatGrid>()) {
|
||||
auto typed = openvdb::GridBase::grid<openvdb::FloatGrid>(grid);
|
||||
if (quantization == "FP16") {
|
||||
nanovdb::tools::CreateNanoGrid<openvdb::FloatGrid> converter(*typed);
|
||||
converter.setStats(nanovdb::tools::StatsMode::All);
|
||||
converter.setChecksum(nanovdb::CheckMode::Full);
|
||||
return converter.getHandle<nanovdb::Fp16>();
|
||||
}
|
||||
}
|
||||
else if (!grid->isType<openvdb::Vec3SGrid>()) {
|
||||
throw std::runtime_error("unsupported OpenVDB grid type for " + grid->getName() + ": " + grid->valueType());
|
||||
}
|
||||
if (quantization != "LOSSLESS") {
|
||||
throw std::runtime_error("FP16 is supported only for FloatGrid: " + grid->getName());
|
||||
}
|
||||
return nanovdb::tools::openToNanoVDB(
|
||||
grid, nanovdb::tools::StatsMode::All, nanovdb::CheckMode::Full, 0);
|
||||
}
|
||||
|
||||
static void write_vec3(std::ostream &output, const openvdb::Vec3d &value)
|
||||
{
|
||||
output << '[' << value.x() << ',' << value.y() << ',' << value.z() << ']';
|
||||
}
|
||||
|
||||
static void write_coord(std::ostream &output, const openvdb::Coord &value)
|
||||
{
|
||||
output << '[' << value.x() << ',' << value.y() << ',' << value.z() << ']';
|
||||
}
|
||||
|
||||
static void write_report(const Options &options,
|
||||
const fs::path &report_path,
|
||||
const std::vector<GridReport> &grids)
|
||||
{
|
||||
std::ofstream output(report_path, std::ios::out | std::ios::trunc);
|
||||
if (!output) throw std::runtime_error("failed to create conversion report");
|
||||
char nano_version[16];
|
||||
nanovdb::toStr(nano_version, nanovdb::Version());
|
||||
output << std::setprecision(17)
|
||||
<< "{\n \"schemaVersion\":1,\n"
|
||||
<< " \"input\":" << json_string(fs::absolute(options.input).string()) << ",\n"
|
||||
<< " \"output\":" << json_string(fs::absolute(options.output).string()) << ",\n"
|
||||
<< " \"quantization\":" << json_string(options.quantization) << ",\n"
|
||||
<< " \"openVDBVersion\":" << json_string(openvdb::getLibraryVersionString()) << ",\n"
|
||||
<< " \"nanoVDBVersion\":" << json_string(nano_version) << ",\n";
|
||||
using FloatRootData = nanovdb::RootData<nanovdb::NanoUpper<float>>;
|
||||
using FloatUpperData = nanovdb::InternalData<nanovdb::NanoLower<float>, 5>;
|
||||
using FloatLowerData = nanovdb::InternalData<nanovdb::NanoLeaf<float>, 4>;
|
||||
using FloatLeafData = nanovdb::LeafData<float, nanovdb::Coord, nanovdb::Mask, 3>;
|
||||
using Vec3RootData = nanovdb::RootData<nanovdb::NanoUpper<nanovdb::Vec3f>>;
|
||||
using Vec3UpperData = nanovdb::InternalData<nanovdb::NanoLower<nanovdb::Vec3f>, 5>;
|
||||
using Vec3LowerData = nanovdb::InternalData<nanovdb::NanoLeaf<nanovdb::Vec3f>, 4>;
|
||||
using Vec3LeafData = nanovdb::LeafData<nanovdb::Vec3f, nanovdb::Coord, nanovdb::Mask, 3>;
|
||||
output << " \"float32TreeLayout\":{"
|
||||
<< "\"gridDataBytes\":" << sizeof(nanovdb::GridData)
|
||||
<< ",\"treeDataBytes\":" << sizeof(nanovdb::TreeData)
|
||||
<< ",\"treeRootOffsetOffset\":" << offsetof(nanovdb::TreeData, mNodeOffset[3])
|
||||
<< ",\"rootDataBytes\":" << sizeof(FloatRootData)
|
||||
<< ",\"rootTableSizeOffset\":" << offsetof(FloatRootData, mTableSize)
|
||||
<< ",\"rootTileBytes\":" << sizeof(FloatRootData::Tile)
|
||||
<< ",\"rootTileKeyOffset\":" << offsetof(FloatRootData::Tile, key)
|
||||
<< ",\"rootTileChildOffset\":" << offsetof(FloatRootData::Tile, child)
|
||||
<< ",\"rootTileStateOffset\":" << offsetof(FloatRootData::Tile, state)
|
||||
<< ",\"rootTileValueOffset\":" << offsetof(FloatRootData::Tile, value)
|
||||
<< ",\"upperNodeBytes\":" << sizeof(FloatUpperData)
|
||||
<< ",\"upperValueMaskOffset\":" << offsetof(FloatUpperData, mValueMask)
|
||||
<< ",\"upperChildMaskOffset\":" << offsetof(FloatUpperData, mChildMask)
|
||||
<< ",\"upperTableOffset\":" << offsetof(FloatUpperData, mTable)
|
||||
<< ",\"lowerNodeBytes\":" << sizeof(FloatLowerData)
|
||||
<< ",\"lowerValueMaskOffset\":" << offsetof(FloatLowerData, mValueMask)
|
||||
<< ",\"lowerChildMaskOffset\":" << offsetof(FloatLowerData, mChildMask)
|
||||
<< ",\"lowerTableOffset\":" << offsetof(FloatLowerData, mTable)
|
||||
<< ",\"leafNodeBytes\":" << sizeof(FloatLeafData)
|
||||
<< ",\"leafValueMaskOffset\":" << offsetof(FloatLeafData, mValueMask)
|
||||
<< ",\"leafValuesOffset\":" << offsetof(FloatLeafData, mValues)
|
||||
<< "},\n"
|
||||
<< " \"vec3fTreeLayout\":{"
|
||||
<< "\"gridDataBytes\":" << sizeof(nanovdb::GridData)
|
||||
<< ",\"treeDataBytes\":" << sizeof(nanovdb::TreeData)
|
||||
<< ",\"treeRootOffsetOffset\":" << offsetof(nanovdb::TreeData, mNodeOffset[3])
|
||||
<< ",\"rootDataBytes\":" << sizeof(Vec3RootData)
|
||||
<< ",\"rootTableSizeOffset\":" << offsetof(Vec3RootData, mTableSize)
|
||||
<< ",\"rootTileBytes\":" << sizeof(Vec3RootData::Tile)
|
||||
<< ",\"rootTileKeyOffset\":" << offsetof(Vec3RootData::Tile, key)
|
||||
<< ",\"rootTileChildOffset\":" << offsetof(Vec3RootData::Tile, child)
|
||||
<< ",\"rootTileStateOffset\":" << offsetof(Vec3RootData::Tile, state)
|
||||
<< ",\"rootTileValueOffset\":" << offsetof(Vec3RootData::Tile, value)
|
||||
<< ",\"upperNodeBytes\":" << sizeof(Vec3UpperData)
|
||||
<< ",\"upperValueMaskOffset\":" << offsetof(Vec3UpperData, mValueMask)
|
||||
<< ",\"upperChildMaskOffset\":" << offsetof(Vec3UpperData, mChildMask)
|
||||
<< ",\"upperTableOffset\":" << offsetof(Vec3UpperData, mTable)
|
||||
<< ",\"lowerNodeBytes\":" << sizeof(Vec3LowerData)
|
||||
<< ",\"lowerValueMaskOffset\":" << offsetof(Vec3LowerData, mValueMask)
|
||||
<< ",\"lowerChildMaskOffset\":" << offsetof(Vec3LowerData, mChildMask)
|
||||
<< ",\"lowerTableOffset\":" << offsetof(Vec3LowerData, mTable)
|
||||
<< ",\"leafNodeBytes\":" << sizeof(Vec3LeafData)
|
||||
<< ",\"leafValueMaskOffset\":" << offsetof(Vec3LeafData, mValueMask)
|
||||
<< ",\"leafValuesOffset\":" << offsetof(Vec3LeafData, mValues)
|
||||
<< "},\n"
|
||||
<< " \"grids\":[\n";
|
||||
for (size_t index = 0; index < grids.size(); ++index) {
|
||||
const GridReport &grid = grids[index];
|
||||
output << " {\"name\":" << json_string(grid.name)
|
||||
<< ",\"sourceType\":" << json_string(grid.source_type)
|
||||
<< ",\"valueType\":" << json_string(grid.value_type)
|
||||
<< ",\"gridClass\":" << json_string(grid.grid_class)
|
||||
<< ",\"activeVoxelCount\":" << grid.active_voxels
|
||||
<< ",\"segmentByteOffset\":" << grid.segment_offset
|
||||
<< ",\"segmentByteLength\":" << grid.segment_length
|
||||
<< ",\"byteOffset\":" << grid.grid_offset
|
||||
<< ",\"byteLength\":" << grid.grid_length
|
||||
<< ",\"indexBounds\":{\"min\":";
|
||||
write_coord(output, grid.index_bounds.min());
|
||||
output << ",\"max\":";
|
||||
write_coord(output, grid.index_bounds.max());
|
||||
output << "},\"worldBounds\":{\"min\":";
|
||||
write_vec3(output, grid.world_bounds.min());
|
||||
output << ",\"max\":";
|
||||
write_vec3(output, grid.world_bounds.max());
|
||||
output << "},\"voxelSize\":";
|
||||
write_vec3(output, grid.voxel_size);
|
||||
output << ",\"indexToWorld\":[";
|
||||
for (size_t matrix_index = 0; matrix_index < grid.index_to_world.size(); ++matrix_index) {
|
||||
if (matrix_index) output << ',';
|
||||
output << grid.index_to_world[matrix_index];
|
||||
}
|
||||
output << ']';
|
||||
if (!grid.scalar_samples.empty()) {
|
||||
output << ",\"scalarSamples\":[";
|
||||
for (size_t sample_index = 0; sample_index < grid.scalar_samples.size(); ++sample_index) {
|
||||
const auto &sample = grid.scalar_samples[sample_index];
|
||||
if (sample_index) output << ',';
|
||||
output << "{\"coord\":";
|
||||
write_coord(output, sample.coord);
|
||||
output << ",\"value\":" << sample.value << ",\"active\":" << (sample.active ? "true" : "false") << '}';
|
||||
}
|
||||
output << ']';
|
||||
}
|
||||
if (!grid.vector_samples.empty()) {
|
||||
output << ",\"vectorSamples\":[";
|
||||
for (size_t sample_index = 0; sample_index < grid.vector_samples.size(); ++sample_index) {
|
||||
const auto &sample = grid.vector_samples[sample_index];
|
||||
if (sample_index) output << ',';
|
||||
output << "{\"coord\":";
|
||||
write_coord(output, sample.coord);
|
||||
output << ",\"value\":[" << sample.value[0] << ',' << sample.value[1] << ',' << sample.value[2]
|
||||
<< "],\"active\":" << (sample.active ? "true" : "false") << '}';
|
||||
}
|
||||
output << ']';
|
||||
}
|
||||
output << '}' << (index + 1 == grids.size() ? "\n" : ",\n");
|
||||
}
|
||||
output << " ]\n}\n";
|
||||
if (!output) throw std::runtime_error("failed to write conversion report");
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
fs::path staged_output;
|
||||
fs::path staged_report;
|
||||
try {
|
||||
const Options options = parse_options(argc, argv);
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
std::signal(SIGINT, request_interrupt);
|
||||
std::signal(SIGTERM, request_interrupt);
|
||||
check_interrupted(options, started, "startup");
|
||||
if (!fs::is_regular_file(options.input)) throw std::runtime_error("input VDB does not exist");
|
||||
const uint64_t source_size = fs::file_size(options.input);
|
||||
if (source_size == 0 || source_size > MAX_SOURCE_BYTES) throw std::runtime_error("input VDB exceeds the source byte budget");
|
||||
fs::create_directories(fs::absolute(options.output).parent_path());
|
||||
fs::create_directories(fs::absolute(options.report).parent_path());
|
||||
const std::string stage_suffix = "." + std::to_string(static_cast<uint64_t>(getpid())) + ".stage";
|
||||
staged_output = options.output.string() + stage_suffix;
|
||||
staged_report = options.report.string() + stage_suffix;
|
||||
fs::remove(staged_output);
|
||||
fs::remove(staged_report);
|
||||
|
||||
openvdb::initialize();
|
||||
openvdb::io::File input(options.input.string());
|
||||
input.open(false);
|
||||
check_interrupted(options, started, "OpenVDB inventory");
|
||||
openvdb::GridPtrVecPtr source_grids = input.getGrids();
|
||||
if (!source_grids || source_grids->empty() || source_grids->size() > MAX_GRIDS) throw std::runtime_error("VDB grid count exceeds the budget");
|
||||
|
||||
const std::set<std::string> selected(options.grids.begin(), options.grids.end());
|
||||
std::set<std::string> found;
|
||||
uint64_t total_active_voxels = 0;
|
||||
std::ofstream output(staged_output, std::ios::binary | std::ios::trunc);
|
||||
if (!output) throw std::runtime_error("failed to create NanoVDB output");
|
||||
std::vector<GridReport> report;
|
||||
for (const openvdb::GridBase::Ptr &grid : *source_grids) {
|
||||
check_interrupted(options, started, "grid inventory");
|
||||
if (!selected.empty() && !selected.count(grid->getName())) continue;
|
||||
if (!found.insert(grid->getName()).second) throw std::runtime_error("duplicate source grid name: " + grid->getName());
|
||||
total_active_voxels += grid->activeVoxelCount();
|
||||
if (total_active_voxels > MAX_ACTIVE_VOXELS) throw std::runtime_error("active voxel budget exceeded");
|
||||
|
||||
auto handle = convert_grid(grid, options.quantization);
|
||||
check_interrupted(options, started, "NanoVDB conversion");
|
||||
const uint64_t segment_offset = static_cast<uint64_t>(output.tellp());
|
||||
nanovdb::io::writeGrid(output, handle, nanovdb::io::Codec::NONE);
|
||||
const uint64_t segment_end = static_cast<uint64_t>(output.tellp());
|
||||
const uint64_t name_size = grid->getName().size() + 1;
|
||||
const uint64_t grid_offset = segment_offset + sizeof(nanovdb::io::FileHeader) + sizeof(nanovdb::io::FileMetaData) + name_size;
|
||||
if (grid_offset + handle.gridSize() != segment_end) throw std::runtime_error("unexpected NanoVDB segment layout");
|
||||
|
||||
GridReport item;
|
||||
item.name = grid->getName();
|
||||
item.source_type = grid->valueType();
|
||||
item.value_type = options.quantization == "FP16" ? "FLOAT16" : grid->isType<openvdb::FloatGrid>() ? "FLOAT32" : "VEC3F32";
|
||||
item.grid_class = grid_class_name(grid->getGridClass());
|
||||
item.active_voxels = grid->activeVoxelCount();
|
||||
item.segment_offset = segment_offset;
|
||||
item.segment_length = segment_end - segment_offset;
|
||||
item.grid_offset = grid_offset;
|
||||
item.grid_length = handle.gridSize();
|
||||
item.index_bounds = grid->evalActiveVoxelBoundingBox();
|
||||
item.world_bounds = world_bounds(*grid, item.index_bounds);
|
||||
item.voxel_size = grid->voxelSize();
|
||||
item.index_to_world = index_to_world(*grid);
|
||||
if (options.quantization == "LOSSLESS" && grid->isType<openvdb::FloatGrid>()) {
|
||||
const nanovdb::NanoGrid<float> *nano_grid = handle.grid<float>();
|
||||
if (!nano_grid) throw std::runtime_error("NanoVDB Float32 grid payload is unavailable");
|
||||
const std::array<openvdb::Coord, 5> sample_coords = {
|
||||
item.index_bounds.min(), openvdb::Coord(0, 0, 0), item.index_bounds.max(),
|
||||
openvdb::Coord(item.index_bounds.min().x() - 1, 0, 0),
|
||||
openvdb::Coord(item.index_bounds.max().x() + 1, 0, 0)};
|
||||
for (const openvdb::Coord &coord : sample_coords) {
|
||||
float value = 0.0f;
|
||||
const bool active = nano_grid->tree().probeValue(nanovdb::Coord(coord.x(), coord.y(), coord.z()), value);
|
||||
item.scalar_samples.push_back({coord, value, active});
|
||||
}
|
||||
}
|
||||
else if (options.quantization == "LOSSLESS" && grid->isType<openvdb::Vec3SGrid>()) {
|
||||
const nanovdb::NanoGrid<nanovdb::Vec3f> *nano_grid = handle.grid<nanovdb::Vec3f>();
|
||||
if (!nano_grid) throw std::runtime_error("NanoVDB Vec3f grid payload is unavailable");
|
||||
const std::array<openvdb::Coord, 5> sample_coords = {
|
||||
item.index_bounds.min(), openvdb::Coord(0, 0, 0), item.index_bounds.max(),
|
||||
openvdb::Coord(item.index_bounds.min().x() - 1, 0, 0),
|
||||
openvdb::Coord(item.index_bounds.max().x() + 1, 0, 0)};
|
||||
for (const openvdb::Coord &coord : sample_coords) {
|
||||
nanovdb::Vec3f value(0.0f);
|
||||
const bool active = nano_grid->tree().probeValue(nanovdb::Coord(coord.x(), coord.y(), coord.z()), value);
|
||||
item.vector_samples.push_back({coord, {value[0], value[1], value[2]}, active});
|
||||
}
|
||||
}
|
||||
report.push_back(std::move(item));
|
||||
|
||||
if (segment_end > MAX_OUTPUT_BYTES) throw std::runtime_error("NanoVDB output exceeds the byte budget");
|
||||
}
|
||||
input.close();
|
||||
output.close();
|
||||
check_interrupted(options, started, "artifact commit");
|
||||
if (!selected.empty() && found != selected) throw std::runtime_error("one or more selected grids were not found");
|
||||
if (report.empty()) throw std::runtime_error("no supported grids were selected");
|
||||
write_report(options, staged_report, report);
|
||||
check_interrupted(options, started, "report commit");
|
||||
fs::rename(staged_output, options.output);
|
||||
fs::rename(staged_report, options.report);
|
||||
openvdb::uninitialize();
|
||||
std::cout << "vdb-to-nanovdb-ok input=" << options.input.string()
|
||||
<< " output=" << options.output.string()
|
||||
<< " grids=" << report.size()
|
||||
<< " bytes=" << fs::file_size(options.output) << '\n';
|
||||
return 0;
|
||||
}
|
||||
catch (const std::exception &error) {
|
||||
if (!staged_output.empty()) fs::remove(staged_output);
|
||||
if (!staged_report.empty()) fs::remove(staged_report);
|
||||
std::cerr << "VDB_CONVERSION_FAILED: " << error.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user