86 lines
14 KiB
JavaScript
86 lines
14 KiB
JavaScript
import { readFile, readdir } from 'node:fs/promises'
|
|
import { resolve } from 'node:path'
|
|
|
|
const root = resolve(new URL('..', import.meta.url).pathname)
|
|
const load = (path) => readFile(resolve(root, path), 'utf8').then(JSON.parse)
|
|
const [
|
|
fcstdFuzz, geometryFuzz, sketchFuzz, successFixtures, failureFixtures, supplementalSuccessFixtures, supplementalFailureFixtures,
|
|
sketchOracle, partdesignBase, partdesignLoft, partdesignDressup, partdesignTransform, partdesignStructure, attachmentModes, attachmentCombinations, xlinkRelink, partdesignFailures, partdesignRevolutionGroove, partBuilders,
|
|
fcstdRoundTrip, app, partdesignReferenceLifecycle, browserMatrix, performance, fault, opfsMigration, security, qa08, addon, script,
|
|
] = await Promise.all([
|
|
load('config/qa04-fcstd-fuzz-verification.json'),
|
|
load('config/qa04-geometry-fuzz-verification.json'),
|
|
load('config/qa04-sketch-fuzz-verification.json'),
|
|
load('fixtures/freecad-golden/manifest.json'),
|
|
load('fixtures/freecad-golden/failures/manifest.json'),
|
|
load('fixtures/freecad-golden/feature-families/manifest.json'),
|
|
load('fixtures/freecad-golden/feature-families/failures/manifest.json'),
|
|
load('config/freecad-sketcher-constraint-oracle.json'),
|
|
load('config/freecad-partdesign-base-oracle.json'),
|
|
load('config/freecad-partdesign-loft-oracle.json'),
|
|
load('config/freecad-partdesign-dressup-oracle.json'),
|
|
load('config/freecad-partdesign-transform-oracle.json'),
|
|
load('config/freecad-partdesign-structure-oracle.json'),
|
|
load('config/freecad-attachment-mode-oracle.json'),
|
|
load('config/freecad-attachment-combination-oracle.json'),
|
|
load('config/freecad-xlink-relink-oracle.json'),
|
|
load('config/freecad-partdesign-failure-oracle.json'),
|
|
load('config/freecad-partdesign-revolution-groove-oracle.json'),
|
|
load('config/freecad-part-builders-oracle.json'),
|
|
load('config/freecad-fcstd-roundtrip-verification.json'),
|
|
load('config/chrome-app-e2e-verification.json'),
|
|
load('config/chrome-partdesign-reference-lifecycle-verification.json'),
|
|
load('config/browser-matrix-verification.json'),
|
|
load('config/chrome-performance-verification.json'),
|
|
load('config/chrome-fault-injection-verification.json'),
|
|
load('config/chrome-opfs-migration-verification.json'),
|
|
load('config/chrome-security-verification.json'),
|
|
load('config/chrome-qa08-verification.json'),
|
|
load('config/chrome-addon-verification.json'),
|
|
load('config/chrome-script-verification.json'),
|
|
])
|
|
|
|
const testFiles = (await readdir(resolve(root, 'tests'))).filter((name) => /\.test\.(?:ts|mjs)$/.test(name)).sort()
|
|
const testSources = await Promise.all(testFiles.map((name) => readFile(resolve(root, 'tests', name), 'utf8')))
|
|
const testCases = testSources.reduce((total, source) => total + (source.match(/(?:^|\n)test\(/g)?.length ?? 0), 0)
|
|
if (testFiles.length < 24 || testCases < 217 || !testFiles.includes('facade.test.ts') || !testFiles.includes('goldenReplay.test.mjs')) throw new Error('QA-01 unit/invariant suite inventory is incomplete.')
|
|
|
|
if (successFixtures.scenarios?.length !== 100 || failureFixtures.failures?.length !== 51 || supplementalSuccessFixtures.scenarios?.length !== 5 || supplementalFailureFixtures.failures?.length !== 5) throw new Error('QA-02 FreeCAD success/failure golden inventory is incomplete.')
|
|
if (sketchOracle.status !== 'pass' || sketchOracle.summary?.constraintTypes !== 19 || sketchOracle.summary.successPassed !== 19 || sketchOracle.summary.failurePassed !== 19) throw new Error('QA-02 Sketcher oracle evidence is incomplete.')
|
|
const partdesignCases = [partdesignBase, partdesignLoft, partdesignDressup, partdesignTransform].reduce((total, report) => total + (report.summary?.passed ?? 0), 0)
|
|
if (partdesignCases !== 23 || [partdesignBase, partdesignLoft, partdesignDressup, partdesignTransform].some((report) => report.status !== 'pass' || report.freecadVersion !== '1.1.1')) throw new Error('QA-02 PartDesign oracle evidence is incomplete.')
|
|
if (partdesignStructure.status !== 'pass' || partdesignStructure.freecadVersion !== '1.1.1' || Object.keys(partdesignStructure.initial?.bodyGroups ?? {}).length !== 4 || partdesignStructure.crossDocument?.initial?.shapeBinderExternalLink?.accepted !== false || partdesignStructure.crossDocument?.initial?.subShapeBinderSupportType !== 'App::PropertyXLinkSubList' || partdesignStructure.crossDocument?.edited?.subShapeBinderArea !== 48 || partdesignStructure.crossDocument?.deleted?.subShapeBinderCachedArea !== 48) throw new Error('QA-02 PartDesign structure and cross-document evidence is incomplete.')
|
|
if (attachmentModes.status !== 'pass' || attachmentModes.freecadVersion !== '1.1.1' || attachmentModes.registry?.modeCount !== 55 || attachmentModes.registry?.implementedUnionCount !== 50 || attachmentModes.engines?.plane?.implementedModes?.length !== 23 || attachmentModes.engines?.line?.implementedModes?.length !== 18 || attachmentModes.engines?.point?.implementedModes?.length !== 9 || attachmentModes.success?.roundtrip?.point?.placement?.position?.[2] !== 9 || Object.keys(attachmentModes.failures?.roundtrip ?? {}).length !== 3) throw new Error('QA-02 Attachment mode registry, execution or round-trip evidence is incomplete.')
|
|
if (attachmentCombinations.status !== 'pass' || attachmentCombinations.freecadVersion !== '1.1.1' || attachmentCombinations.caseCount !== 236 || attachmentCombinations.cases?.length !== 236 || attachmentCombinations.cases.some((entry) => entry.initial?.status !== 'Valid' || entry.mutated?.status !== 'Valid' || entry.roundtrip?.status !== 'Valid')) throw new Error('QA-02 exhaustive Attachment mode/reference combination evidence is incomplete.')
|
|
if (xlinkRelink.status !== 'pass' || xlinkRelink.freecadVersion !== '1.1.1' || xlinkRelink.propertyTypes?.length !== 4 || xlinkRelink.relinkAfterMissing?.holder?.xLinkList?.length !== 2 || xlinkRelink.edited?.binder?.area !== 54 || xlinkRelink.finalRoundtrip?.binder?.area !== 54) throw new Error('QA-02 native XLink close, missing-file relink, edit or FCStd evidence is incomplete.')
|
|
if (partdesignFailures.status !== 'pass' || partdesignFailures.freecadVersion !== '1.1.1' || partdesignFailures.summary?.cases !== 17 || partdesignFailures.summary.passed !== 17 || partdesignFailures.summary.rejected !== 13 || partdesignFailures.summary.acceptedEmpty !== 4 || partdesignFailures.summary.accepted !== 0) throw new Error('QA-02 PartDesign failure oracle evidence is incomplete.')
|
|
if (partdesignRevolutionGroove.status !== 'pass' || partdesignRevolutionGroove.freecadVersion !== '1.1.1' || partdesignRevolutionGroove.summary?.cases !== 2 || partdesignRevolutionGroove.summary.passed !== 2 || partdesignRevolutionGroove.cases?.some((fixture) => fixture.shapeNull !== false || fixture.shapeValid !== true || fixture.solids !== 1)) throw new Error('QA-02 PartDesign Revolution/Groove oracle evidence is incomplete.')
|
|
if (partBuilders.status !== 'pass' || partBuilders.freecadVersion !== '1.1.1' || partBuilders.summary?.successCases !== 6 || partBuilders.summary.successPassed !== 6 || partBuilders.summary.failureCases !== 6 || partBuilders.summary.failurePassed !== 6 || partBuilders.summary.rejected !== 6 || partBuilders.summary.acceptedEmpty !== 0) throw new Error('QA-02 Part builders oracle evidence is incomplete.')
|
|
if (fcstdRoundTrip.status !== 'verified' || !Number.isSafeInteger(fcstdRoundTrip.scenarioCount) || fcstdRoundTrip.scenarioCount < 23 || fcstdRoundTrip.scenarios?.length !== fcstdRoundTrip.scenarioCount || fcstdRoundTrip.scenarios?.some((scenario) => scenario.status !== 'pass' || scenario.differences?.length !== 0)) throw new Error('QA-02 FCStd oracle evidence is incomplete.')
|
|
|
|
const requiredAppWorkflows = ['start-to-workspace', 'freecad-menus', 'command-search-dialog', 'preferences-about-dialogs', 'model-tree-context-menu', 'workbench-change', 'create-sketch-command', 'loft-section-task', 'drawer-collapse', 'cam-workbench-job-flow', 'opfs-save-reopen', 'import-format-selection', 'export-format-selection']
|
|
if (app.status !== 'pass' || requiredAppWorkflows.some((workflow) => !app.workflows?.includes(workflow)) || app.pageErrors?.length !== 0 || app.desktop?.kernelPreviewSource !== 'bitbybit-occt' || app.desktop.canvas?.uniqueColors < 4 || app.desktop?.commandGroups < 4 || app.desktop?.selectionView !== true || app.desktop?.statusBar !== true || app.uiParity?.commandDialog?.modal !== true || app.uiParity?.preferencesDialog?.modal !== true || app.uiParity?.aboutDialog?.modal !== true || app.uiParity?.contextMenu?.items?.length !== 4 || app.uiParity?.camMenu?.items !== 60 || app.cam?.ui?.generated !== true || app.cam.ui.toolbarCommands !== 60 || app.cam.ui.toolpathPoints < 5 || app.mobile?.viewport?.width !== 390 || Object.keys(app.screenshots ?? {}).length !== 3) throw new Error('QA-03 production Chrome E2E evidence is incomplete.')
|
|
if (partdesignReferenceLifecycle.status !== 'pass' || partdesignReferenceLifecycle.crossOriginIsolated !== true || partdesignReferenceLifecycle.initial?.properties?.XLinkSubList?.length !== 2 || partdesignReferenceLifecycle.missing?.binder?.cacheState !== 'cached' || partdesignReferenceLifecycle.relinkedAfterMissing?.binder?.cacheState !== 'linked' || partdesignReferenceLifecycle.reopenedUndo?.binder?.area !== 30 || partdesignReferenceLifecycle.reopenedRedo?.binder?.area !== 54 || partdesignReferenceLifecycle.released?.shapeCount !== 0 || partdesignReferenceLifecycle.released?.kernelReferenceCount !== 0) throw new Error('QA-03 Chrome PartDesign external reference lifecycle evidence is incomplete.')
|
|
if (browserMatrix.status !== 'pass' || browserMatrix.browsers?.length !== 2 || !['firefox', 'webkit'].every((engine) => browserMatrix.browsers.some((entry) => entry.engine === engine && entry.status === 'pass' && entry.pageErrors?.length === 0 && entry.canvas?.geometrySource === 'bitbybit-occt'))) throw new Error('QA-03 non-Chrome browser matrix evidence is incomplete.')
|
|
|
|
if (fcstdFuzz.status !== 'fcstd-parser-fuzz-pass' || fcstdFuzz.cases !== 1000 || fcstdFuzz.counts?.accepted + fcstdFuzz.counts?.rejected !== 1000 || Object.keys(fcstdFuzz.categories ?? {}).length !== 13 || fcstdFuzz.timing?.p95Ms >= 50) throw new Error('QA-04 FCStd fuzz evidence is incomplete.')
|
|
if (geometryFuzz.status !== 'geometry-input-fuzz-pass' || geometryFuzz.cases !== 2000 || geometryFuzz.counts?.accepted + geometryFuzz.counts?.rejected !== 2000 || Object.keys(geometryFuzz.categories ?? {}).length !== 30 || geometryFuzz.timing?.p95Ms >= 10) throw new Error('QA-04 geometry fuzz evidence is incomplete.')
|
|
if (sketchFuzz.status !== 'sketch-solver-fuzz-pass' || sketchFuzz.models !== 2000 || sketchFuzz.solves !== 4000 || Object.keys(sketchFuzz.categories ?? {}).length !== 10 || sketchFuzz.timing?.p95Ms >= 10) throw new Error('QA-04 solver fuzz evidence is incomplete.')
|
|
|
|
if (performance.status !== 'pass' || performance.benchmark?.objects !== 1000 || performance.benchmark.triangles !== 1_000_000 || performance.benchmark.tableCells !== 100_000 || performance.benchmark.pass !== true || performance.afterRelease?.shapeCount !== 0) throw new Error('QA-05 performance and ownership evidence is incomplete.')
|
|
if (fault.status !== 'pass' || fault.lifecycle?.completed !== 'completed' || fault.lifecycle.cancelled !== 'cancelled' || fault.lifecycle.crashInjected !== true || fault.lifecycle.recovered !== 'completed' || fault.lifecycle.stale !== 'stale' || fault.opfs?.markerRemoved !== true) throw new Error('QA-06 Worker/WASM fault evidence is incomplete.')
|
|
if (opfsMigration.status !== 'pass' || opfsMigration.migration?.migratedSchemaVersion !== 7 || opfsMigration.migration.reopenedNativeObjectTag !== 17 || opfsMigration.migration.recoveryIntegrity !== 'ok' || opfsMigration.rollback?.rejected !== true) throw new Error('QA-06 OPFS recovery evidence is incomplete.')
|
|
if (security.status !== 'pass' || security.checks?.rejectedCases !== 4 || security.checks.pass !== true || addon.security?.forgedRejected !== true || addon.security.permissionRejected !== true || script.security?.noDynamicExecution !== true || script.security.deniedCapabilities?.length !== 4) throw new Error('QA-07 security evidence is incomplete.')
|
|
if (qa08.status !== 'pass' || qa08.locales?.supported !== 3 || qa08.accessibility?.pass !== true || app.keyboard?.named !== 8 || app.screenReader?.namedControls !== app.screenReader?.controls || app.screenReader?.unnamedControls !== 0 || app.mobile?.bodyHorizontalOverflow > 1) throw new Error('QA-08 accessibility/i18n evidence is incomplete.')
|
|
|
|
console.log(JSON.stringify({
|
|
status: 'quality-closure-pass',
|
|
tasks: 8,
|
|
unitSuites: testFiles.length,
|
|
unitTests: testCases,
|
|
freecad: { successFixtures: 105, failureFixtures: 56, primarySuccessFixtures: 100, primaryFailureFixtures: 51, supplementalSuccessFixtures: 5, supplementalFailureFixtures: 5, sketchConstraints: 19, partdesignCases, partdesignStructureBodies: Object.keys(partdesignStructure.initial.bodyGroups).length, attachmentModes: attachmentModes.registry.modeCount, attachmentImplementedUnion: attachmentModes.registry.implementedUnionCount, attachmentCombinationCases: attachmentCombinations.caseCount, xlinkPropertyKinds: xlinkRelink.propertyTypes.length, partdesignFailureCases: partdesignFailures.summary.cases, partdesignRejectedFailures: partdesignFailures.summary.rejected, partdesignAcceptedEmpty: partdesignFailures.summary.acceptedEmpty, partdesignRevolutionGrooveCases: partdesignRevolutionGroove.summary.cases, partBuilderSuccessCases: partBuilders.summary.successCases, partBuilderFailureCases: partBuilders.summary.failureCases, fcstdRoundTrips: fcstdRoundTrip.scenarioCount },
|
|
fuzz: { fcstd: fcstdFuzz.cases, geometry: geometryFuzz.cases, solverModels: sketchFuzz.models },
|
|
chrome: { workflows: app.workflows.length, screenshots: Object.keys(app.screenshots).length, accessibilityNodes: app.screenReader.nodes, partdesignExternalReferenceLifecycle: true },
|
|
browsers: Object.fromEntries(browserMatrix.browsers.map((entry) => [entry.engine, { status: entry.status, persistence: entry.persistence, geometrySource: entry.canvas.geometrySource }])),
|
|
}, null, 2))
|