100 lines
4.9 KiB
TypeScript
100 lines
4.9 KiB
TypeScript
import { expect, test } from "@playwright/test";
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
|
|
const basicBlend = path.resolve(import.meta.dirname, "../../../tests/files/web/basic_scene.blend");
|
|
|
|
test("recovers deterministically from WASM, OPFS, GPU and NanoVDB allocation faults", async ({ page }) => {
|
|
test.setTimeout(120_000);
|
|
await page.goto("/");
|
|
const bytes = fs.readFileSync(basicBlend);
|
|
const result = await page.evaluate(async (input) => {
|
|
const { runOOMRecoveryScenarios, parseOOMRecoverySuite } = await import("/src/testing/oom-recovery-scenarios.ts");
|
|
const reports = await runOOMRecoveryScenarios(Uint8Array.from(input).buffer);
|
|
let rejectsMappingDrift = false;
|
|
try {
|
|
const invalid = structuredClone(reports);
|
|
invalid[0].faults[0].code = "INVALID_ARGUMENT";
|
|
parseOOMRecoverySuite(invalid);
|
|
}
|
|
catch {
|
|
rejectsMappingDrift = true;
|
|
}
|
|
return { reports, rejectsMappingDrift };
|
|
}, Array.from(bytes));
|
|
|
|
expect(result.rejectsMappingDrift).toBe(true);
|
|
expect(result.reports.map((report) => report.scenario)).toEqual([
|
|
"WASM_MAIN",
|
|
"OPFS_STAGING",
|
|
"GPU_RESOURCES",
|
|
"NANOVDB_RESIDENT",
|
|
]);
|
|
|
|
for (const report of result.reports) {
|
|
expect(report.schemaVersion).toBe(1);
|
|
expect(report.faults.length).toBeGreaterThan(0);
|
|
expect(report.memory.beforeBytes).toBeGreaterThanOrEqual(0);
|
|
expect(report.memory.peakBytes).toBeGreaterThanOrEqual(report.memory.beforeBytes);
|
|
expect(report.memory.peakBytes).toBeGreaterThanOrEqual(report.memory.afterBytes);
|
|
expect(report.state.temporaryResourcesAfter).toBe(0);
|
|
expect(report.recovery.recovered).toBe(true);
|
|
expect(report.recovery.tokenIsolated).toBe(true);
|
|
expect(report.recovery.sameSession || report.recovery.restartedSession).toBe(true);
|
|
expect(report.checks.length).toBeGreaterThanOrEqual(4);
|
|
}
|
|
|
|
const wasm = result.reports.find((report) => report.scenario === "WASM_MAIN")!;
|
|
expect(wasm.faults.map((fault) => [fault.point, fault.code, fault.stage])).toEqual([
|
|
["WASM_MAIN_OPEN_INPUT", "WASM_OUT_OF_MEMORY", "WASM_OPEN_INPUT"],
|
|
["WASM_MAIN_EDIT_COMMAND", "WASM_OUT_OF_MEMORY", "MAIN_EDIT_COMMAND"],
|
|
["WASM_MAIN_SAVE_RESULT", "WASM_OUT_OF_MEMORY", "WASM_SAVE_RESULT"],
|
|
]);
|
|
expect(wasm.state.revisionAfter).toBe(wasm.state.revisionBefore);
|
|
expect(wasm.state.hashAfter).toBe(wasm.state.hashBefore);
|
|
expect(wasm.memory.releasedBytes).toBeGreaterThan(0);
|
|
expect(wasm.recovery.sameSession).toBe(true);
|
|
expect(wasm.recovery.restartedSession).toBe(true);
|
|
expect(wasm.checks).toEqual(expect.arrayContaining(["undo-redo-stable", "worker-reopen"]));
|
|
|
|
const opfs = result.reports.find((report) => report.scenario === "OPFS_STAGING")!;
|
|
expect(opfs.faults[0]).toMatchObject({ point: "OPFS_STAGING_WRITE", code: "STORAGE_QUOTA", stage: "OPFS_STAGING_WRITE" });
|
|
expect(opfs.state.revisionBefore).toBe(7);
|
|
expect(opfs.state.revisionAfter).toBe(7);
|
|
expect(opfs.state.hashAfter).toBe(opfs.state.hashBefore);
|
|
expect(opfs.memory.releasedBytes).toBeGreaterThan(0);
|
|
expect(opfs.state.temporaryResourcesPeak).toBeGreaterThan(0);
|
|
expect(opfs.checks).toEqual(expect.arrayContaining(["journal-not-committed", "temporary-files-removed", "next-save-succeeds"]));
|
|
|
|
const gpu = result.reports.find((report) => report.scenario === "GPU_RESOURCES")!;
|
|
expect(gpu.faults[0]).toMatchObject({ point: "GPU_TEXTURE_UPLOAD", code: "GPU_TEXTURE_BUDGET_EXCEEDED", stage: "GPU_TEXTURE_UPLOAD" });
|
|
expect(gpu.state.hashAfter).toBe(gpu.state.hashBefore);
|
|
expect(gpu.memory.releasedBytes).toBeGreaterThan(0);
|
|
expect(gpu.state.temporaryResourcesPeak).toBe(1);
|
|
expect(gpu.checks).toEqual(expect.arrayContaining(["partial-object-not-published", "small-scene-renders"]));
|
|
|
|
const nanoVdb = result.reports.find((report) => report.scenario === "NANOVDB_RESIDENT")!;
|
|
expect(nanoVdb.faults[0]).toMatchObject({ point: "NANOVDB_PAGE_TABLE", code: "NANOVDB_GPU_BUDGET_EXCEEDED", stage: "NANOVDB_PAGE_TABLE" });
|
|
expect(nanoVdb.faults[1]).toMatchObject({ point: "NANOVDB_FEEDBACK_BUFFER", code: "NANOVDB_GPU_BUDGET_EXCEEDED", stage: "NANOVDB_FEEDBACK_BUFFER" });
|
|
expect(nanoVdb.memory.releasedBytes).toBe(2 * 64 * 1024 + 8 + 32);
|
|
expect(nanoVdb.state.temporaryResourcesPeak).toBe(3);
|
|
expect(nanoVdb.checks).toEqual(expect.arrayContaining([
|
|
"resident-buffer-destroyed-once",
|
|
"page-table-destroyed-once",
|
|
"feedback-buffer-destroyed-once",
|
|
"resource-group-dispose-idempotent",
|
|
"same-device-recovers",
|
|
"lru-eviction-recovers",
|
|
]));
|
|
|
|
console.log("oom-recovery", JSON.stringify(result.reports.map((report) => ({
|
|
scenario: report.scenario,
|
|
errors: report.faults.map((fault) => `${fault.code}@${fault.stage}`),
|
|
revision: [report.state.revisionBefore, report.state.revisionAfter],
|
|
hashStable: report.state.hashBefore === report.state.hashAfter,
|
|
releasedBytes: report.memory.releasedBytes,
|
|
temporaryResourcesAfter: report.state.temporaryResourcesAfter,
|
|
recovered: report.recovery.recovered,
|
|
}))));
|
|
});
|