#!/usr/bin/env node import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { coverageBaseline, caseEvidence, canonicalPath } from './coverage-baseline.mjs'; import { atomicJSON } from './workflow-case-lib.mjs'; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const abs = (p) => path.resolve(root, p); const argv = new Set(process.argv.slice(2)); const jsonOnly = argv.has("--json"); const markdownOnly = argv.has("--markdown"); const strict = argv.has("--strict"); const inputStamps = new Map(); const stamp = p => { if (!fs.existsSync(abs(p))) return null; const s = fs.statSync(abs(p), { bigint: true }); return `${s.dev}:${s.ino}:${s.size}:${s.mtimeNs}`; }; const readJson = (p, fallback = {}) => { if (!fs.existsSync(abs(p))) return fallback; const before = stamp(p), value = JSON.parse(fs.readFileSync(abs(p), 'utf8')); if (stamp(p) !== before) throw new Error(`Input changed while parsing: ${p}`); inputStamps.set(p, before); return value; }; const exists = (p) => fs.existsSync(abs(p)); const frozenPath = 'tests/golden/WBV2/completion-baseline-B0.json'; if (!argv.has('--help') && !exists(frozenPath) && exists('tests/golden/WBV2/completion-evaluation.json')) fs.copyFileSync(abs('tests/golden/WBV2/completion-evaluation.json'), abs(frozenPath), fs.constants.COPYFILE_EXCL); const sha = p => { const before = stamp(p); if (inputStamps.has(p) && inputStamps.get(p) !== before) throw new Error(`Input changed: ${p}`); const result = before === null ? null : crypto.createHash('sha256').update(fs.readFileSync(abs(p))).digest('hex'); if (stamp(p) !== before) throw new Error(`Input changed while hashing: ${p}`); inputStamps.set(p, before); return result; }; const ratio = (n, d) => d ? Number((n / d * 100).toFixed(1)) : null; if (argv.has("--help")) { console.log("Usage: node tools/web/evaluate-wbv2-completion.mjs [--json|--markdown] [--strict] [--case-manifest ]"); console.log("Reads repository evidence only; writes tests/golden/WBV2/completion-evaluation.{json,md}."); process.exit(0); } const queueText = fs.readFileSync(abs("docs/EXECUTION_QUEUE.md"), "utf8"); const task = queueText.match(/\| 当前任务 \| `([^`]+)` \|/u)?.[1] ?? null; const parentManifestPath = queueText.match(/\| parent manifest \| `([^`]+)` \|/u)?.[1] ?? null; const taskManifest = task ? readJson(`tests/golden/${task}/manifest.json`) : {}; const latestValidation = () => { const base = task ? `tests/golden/${task}` : ""; const candidates = base && exists(base) ? fs.readdirSync(abs(base), { withFileTypes: true }).filter((e) => e.isDirectory() && e.name.startsWith("validation-")).map((e) => `${base}/${e.name}/workflow-result.json`).filter(exists).sort() : []; return candidates.at(-1) ?? `${base}/validation-20261017-compile/workflow-result.json`; }; const files = { completion: `tests/golden/${task}/desktop-completion-manifest.json`, delivery: `tests/golden/${task}/desktop-delivery-manifest.json`, product: `tests/golden/${task}/product-implementation-report.json`, catalog: "web/app/src/capabilities/blender-feature-route-catalog.json", dynamic: "tests/golden/WBV2/dynamic-coverage-manifest.json", inventory: 'tests/golden/WBV2/dynamic-inventory.json', baseline: frozenPath, parity: "tests/golden/WBV2/full-source-parity-validation-report.json", audit: "tests/golden/WBV2/feature-completeness-audit.json", deployed: `tests/golden/${task}/deployed-audit/completion-audit.json`, chromium: `tests/golden/${task}/deployed-audit/chromium-report.json`, workflow: latestValidation(), integrity: `tests/golden/${task}/deployed-audit/evidence-integrity.json`, }; const data = Object.fromEntries(Object.entries(files).map(([k, p]) => [k, readJson(p)])); const caseIndex = process.argv.indexOf('--case-manifest'); if (caseIndex >= 0) { if (!process.argv[caseIndex + 1]) throw new Error('--case-manifest requires a path'); files.cases = canonicalPath(root, process.argv[caseIndex + 1]); data.cases = readJson(files.cases); } const inputs = Object.fromEntries(Object.entries(files).map(([k, p]) => [k, { path: p, present: exists(p), sha256: sha(p) }])); const hashCache = new Map(); const observedHash = file => { const key = canonicalPath(root, file); if (!hashCache.has(key)) hashCache.set(key, sha(key)); return hashCache.get(key); }; const baseline = coverageBaseline(data.catalog, data.inventory, data.integrity, observedHash); const selectedCases = data.cases ? caseEvidence(data.cases, observedHash) : null; for (const file of ['tools/web/evaluate-wbv2-completion.mjs', 'tools/web/coverage-baseline.mjs', 'tools/web/workflow-case-lib.mjs']) inputs[file] = { path: file, present: exists(file), sha256: sha(file) }; const count = (items, predicate) => items.filter(predicate).length; const workflows = data.product.workflowModules ?? []; const productTasks = data.product.implementationTasks ?? []; const packages = data.completion.packages ?? []; const stageGapItems = (data.delivery.stages ?? []).flatMap((stage) => Object.values(stage.gapItems ?? {}).flatMap((v) => Array.isArray(v) ? v : [])); const gaps = [...stageGapItems, ...Object.values(data.delivery.gapItems ?? {}).flatMap((v) => Array.isArray(v) ? v : [])]; const entries = data.catalog.entries ?? []; const steps = data.chromium.steps ?? []; const integrity = baseline.integrity; const routeTotal = baseline.unitCount; const routeClassified = count(entries, e => ['WEB_MAIN', 'SERVER_NATIVE', 'BROWSER_EQUIVALENT'].includes(e.route)); const verified = baseline.verifiedUnits; const unverified = baseline.unverifiedUnits; const smoke = data.audit.smoke ?? data.product.fullE2ESmoke ?? {}; const historicalWorkflowPassed = Array.isArray(data.workflow.passed) ? data.workflow.passed.length : 0; const workflowBound = data.workflow.sourceTreeSha256 === baseline.sourceTreeSha256 && typeof data.workflow.candidate === 'string' && typeof data.workflow.candidateSha256 === 'string' && observedHash(data.workflow.candidate) === data.workflow.candidateSha256; const workflowPassed = workflowBound ? historicalWorkflowPassed : 0; const workflowRemaining = Array.isArray(data.workflow.remaining) ? data.workflow.remaining : []; const historicalChromiumPassed = count(steps, s => s.status === 'PASS'); const chromiumBound = data.chromium.sourceTreeSha256 === baseline.sourceTreeSha256 && typeof data.chromium.candidate === 'string' && typeof data.chromium.candidateSha256 === 'string' && observedHash(data.chromium.candidate) === data.chromium.candidateSha256; const chromiumPassed = chromiumBound ? historicalChromiumPassed : 0; const chromiumFailed = count(steps, (s) => s.status === "FAIL"); const implementedWorkflows = count(workflows, (x) => x.status === "implemented"); const implementedTasks = count(productTasks, (x) => x.status === "implemented"); const gapStatuses = Object.fromEntries([...new Set(gaps.map((x) => x.status ?? "UNKNOWN"))].sort().map((s) => [s, count(gaps, (x) => (x.status ?? "UNKNOWN") === s)])); const wasmModules = data.audit.wasmLifecycle ?? []; const wasmComplete = count(wasmModules, (x) => x.lifecycle === "complete"); const historicalRuntimeRelease = Number((((ratio(wasmComplete, wasmModules.length) ?? 0) + (data.parity.webProductStatus === "READY" ? 100 : 0) + (data.parity.releaseStatus === "READY" ? 100 : 0)) / 3).toFixed(1)); const runtimeRelease = data.parity.sourceTreeSha256 === baseline.sourceTreeSha256 && data.audit.sourceTreeSha256 === baseline.sourceTreeSha256 ? historicalRuntimeRelease : 0; const dimensions = { implementation: { score: ratio(implementedWorkflows + implementedTasks, workflows.length + productTasks.length), numerator: implementedWorkflows + implementedTasks, denominator: workflows.length + productTasks.length, evidence: "product-implementation-report" }, routeCoverage: { score: ratio(routeClassified, routeTotal), numerator: routeClassified, denominator: routeTotal, evidence: "blender-feature-route-catalog" }, behavioralEvidence: { score: ratio(verified, routeTotal), numerator: verified, denominator: routeTotal, evidence: "catalog.behavioralEvidence" }, workflowValidation: { score: ratio(workflowPassed, historicalWorkflowPassed + workflowRemaining.length), numerator: workflowPassed, denominator: historicalWorkflowPassed + workflowRemaining.length, evidence: "source/candidate-bound validation workflow" }, chromiumSmoke: { score: ratio(chromiumPassed, steps.length), numerator: chromiumPassed, denominator: steps.length, evidence: "deployed chromium report" }, evidenceIntegrity: { score: ratio(Number(integrity.matchingArtifacts ?? 0), Number(integrity.referencedArtifacts ?? 0)), numerator: Number(integrity.matchingArtifacts ?? 0), denominator: Number(integrity.referencedArtifacts ?? 0), evidence: "evidence-integrity" }, runtimeRelease: { score: runtimeRelease, numerator: wasmComplete, denominator: wasmModules.length, evidence: "WASM lifecycle + parity release" }, }; const weighted = { implementation: 0.15, routeCoverage: 0.1, behavioralEvidence: 0.3, workflowValidation: 0.15, chromiumSmoke: 0.1, evidenceIntegrity: 0.1, runtimeRelease: 0.1 }; const score = Number(Object.entries(weighted).reduce((sum, [k, w]) => sum + (dimensions[k].score ?? 0) * w, 0).toFixed(1)); const blockers = []; const block = (id, message, evidence) => blockers.push({ id, message, evidence }); if (selectedCases && (selectedCases.cases.length !== 2 || selectedCases.cases.some(item => !item.eligible))) block('CASE_EVIDENCE_INCOMPLETE', '所选案例未全部通过当前输入和产物绑定;保留有效的局部事实。', [files.cases]); for (const item of Object.values(inputs)) if (!item.present) block('INPUT_MISSING', `输入缺失:${item.path}`, [item.path]); if (taskManifest.status !== "done" || data.completion.status !== "done") block("TASK_OPEN", "当前任务链或桌面完成清单仍为 in_progress。", [files.completion, parentManifestPath]); if (data.completion.acceptance !== "CLAIMED" || data.product.acceptance !== "CLAIMED") block("ACCEPTANCE_UNCLAIMED", "验收仍是 NOT_CLAIMED,不能宣称项目完成。", [files.completion, files.product]); if (unverified > 0) block("BEHAVIOR_UNVERIFIED", `${unverified}/${routeTotal} 个源单元没有精确行为回执。`, [files.catalog, files.dynamic]); if (data.dynamic.coverage?.unitCount !== routeTotal) block("COVERAGE_DENOMINATOR_DRIFT", `历史覆盖快照仍为 ${data.dynamic.coverage?.unitCount ?? 'unknown'};当前目录/inventory 按 ID 统一到 ${routeTotal}。旧快照保留,不能作为当前通过依据。`, [files.catalog, files.dynamic, files.inventory]); if (baseline.reconciliation.catalogOnly.length || baseline.reconciliation.inventoryOnly.length) block('COVERAGE_UNIT_MISMATCH', '目录与 inventory 的单元集合不一致;差异单元已保留在分母及残余中。', [files.catalog, files.inventory]); if (baseline.source.changedFiles || !baseline.source.inventorySnapshotMatches) block('SOURCE_SNAPSHOT_DRIFT', `声明源码范围中 ${baseline.source.changedFiles} 个文件已变化;历史源码回执需重绑定。`, [files.inventory]); if (baseline.history.recordedVerifiedUnits > verified) block('HISTORICAL_EVIDENCE_UNBOUND', `${baseline.history.recordedVerifiedUnits - verified} 个历史通过记录不满足当前引用/源码版本条件。`, [files.inventory, files.integrity]); if (data.workflow.validation !== "PASS" && data.workflow.status !== "PASS") block("LATEST_WORKFLOW_FAILED", `最新工作流验证为 ${data.workflow.validation ?? data.workflow.status ?? "UNKNOWN"}。`, [files.workflow]); if (data.parity.remaining?.status === "PARTIAL") block("SOURCE_PARITY_PARTIAL", data.parity.remaining.reason ?? "源码行为覆盖仍为 PARTIAL。", [files.parity]); if ((data.deployed.findings ?? []).some((x) => x.priority === "high")) block("DEPLOYED_HIGH_FINDING", "部署审计仍有 high 优先级缺口。", [files.deployed]); if (Number(integrity.driftedArtifacts ?? 0) > 0) block("EVIDENCE_DRIFT", `${integrity.driftedArtifacts} 个证据构件发生 hash 漂移。`, [files.integrity]); if (integrity.missingArtifacts > 0) block('EVIDENCE_MISSING', `${integrity.missingArtifacts} 个引用构件缺失。`, [files.integrity]); if (!workflowBound || !chromiumBound) block('RUNTIME_EVIDENCE_UNBOUND', '历史工作流/Chromium 报告缺少当前源码范围与候选绑定;保留历史结果,不计作当前通过。', [files.workflow, files.chromium]); if (wasmComplete < wasmModules.length) block("WASM_LIFECYCLE_INCOMPLETE", `${wasmModules.length - wasmComplete}/${wasmModules.length} 个 WASM 生命周期仍非 complete。`, [files.audit]); for (const item of workflowRemaining) block("WORKFLOW_REMAINING", item, [files.workflow]); const status = blockers.length === 0 && score >= 95 ? "COMPLETE" : blockers.some((x) => ["TASK_OPEN", "ACCEPTANCE_UNCLAIMED", "LATEST_WORKFLOW_FAILED", "SOURCE_PARITY_PARTIAL"].includes(x.id)) ? "PARTIAL_BLOCKED" : "PARTIAL"; const overall = { scorePolicy: 'current-bound-evidence-v2; historical score retained in frozen baseline', status, score, scoreMeaning: "加权进度分;门禁未通过时不能作为完成率验收结论。", implementationState: implementedTasks === productTasks.length && implementedWorkflows === workflows.length ? "IMPLEMENTED_BUT_UNVERIFIED" : "IMPLEMENTATION_INCOMPLETE", blockers: blockers.length, generatedAt: new Date().toISOString(), }; const inputSummarySha256 = crypto.createHash('sha256').update(JSON.stringify({ inputs, baseline, observations: [...hashCache.entries()].sort(([a], [b]) => a.localeCompare(b)) })).digest('hex'); const baselineBytes = `${JSON.stringify(baseline, null, 2)}\n`; const baselineSha256 = crypto.createHash('sha256').update(baselineBytes).digest('hex'); const currentBaselinePath = `tests/golden/WBV2/coverage-baselines/${baselineSha256}.json`; const report = { inputSummarySha256, baseline: { path: currentBaselinePath, sha256: crypto.createHash('sha256').update(baselineBytes).digest('hex'), sourceScope: baseline.source.scope, originalReport: inputs.baseline }, progressLines: { coreWorkflows: { currentChecks: workflowPassed, reportedChecks: historicalWorkflowPassed, remaining: workflowRemaining.length }, exactBehavior: { current: verified, total: routeTotal }, sourceParity: { current: verified, total: routeTotal, sourceTreeSha256: baseline.sourceTreeSha256 } }, selectedCases, schemaVersion: 1, tool: "evaluate-wbv2-completion", project: "desktop Blender 5.2 -> Web Blender", generatedAt: overall.generatedAt, currentTask: task, parentManifest: parentManifestPath, taskManifest: { status: taskManifest.status ?? null, nextTask: taskManifest.nextTask ?? null }, overall, dimensions, implementation: { workflows: { implemented: implementedWorkflows, total: workflows.length }, productTasks: { implemented: implementedTasks, total: productTasks.length }, desktopPackages: { implementing: count(packages, (x) => x.status === "implementing_gap"), planned: count(packages, (x) => x.status === "planned_gap_closure"), validationNotRun: count(packages, (x) => x.validation === "NOT_RUN") }, desktopStages: (data.delivery.stages ?? []).map((x) => ({ id: x.id, packages: x.completionWorkPackages ?? [], gaps: Object.values(x.gapItems ?? {}).reduce((n, v) => n + (Array.isArray(v) ? v.length : 0), 0) })), gapStatuses }, coverage: { routeTotal, routeClassified, routeUnclassified: routeTotal - routeClassified, routeKinds: data.catalog.counts ?? {}, verified, unverified, sourceTreeSha256: baseline.sourceTreeSha256, reconciliation: baseline.reconciliation, source: baseline.source }, historical: { ...baseline.history, workflowPassed: historicalWorkflowPassed, chromiumPassed: historicalChromiumPassed, runtimeRelease: historicalRuntimeRelease, dynamicCoverage: data.dynamic.coverage, frozenReport: inputs.baseline }, runtimeRelease: { wasmModules: wasmModules.length, wasmComplete, webProductStatus: data.parity.webProductStatus ?? null, releaseStatus: data.parity.releaseStatus ?? null }, validation: { latest: data.workflow.validation ?? data.workflow.status ?? null, passedChecks: workflowPassed, remaining: workflowRemaining, chromium: { passed: chromiumPassed, failed: chromiumFailed, total: steps.length, errors: data.chromium.errors ?? [], badResponses: data.chromium.badResponses ?? [] }, smoke: { passed: smoke.passed ?? smoke.testsPassed ?? null, total: smoke.totalFeatures ?? smoke.total ?? null }, fullE2E: data.product.fullE2ESmoke ?? {}, realChromiumEvidence: data.product.realChromiumEvidence ?? {} }, integrity: { referencedArtifacts: Number(integrity.referencedArtifacts ?? 0), matchingArtifacts: Number(integrity.matchingArtifacts ?? 0), driftedArtifacts: Number(integrity.driftedArtifacts ?? 0), missingArtifacts: Number(integrity.missingArtifacts ?? 0) }, blockers, inputs, }; const pct = (x) => x == null ? "n/a" : `${x}%`; const markdown = [ `# Web Blender 完成度评估\n`, `生成时间:${report.generatedAt} | 当前任务:${task ?? "unknown"}`, `输入摘要:${inputSummarySha256};基线:${currentBaselinePath}\n当前分母:${routeTotal};旧快照:${data.dynamic.coverage?.unitCount ?? 'unknown'};历史通过:${baseline.history.recordedVerifiedUnits};当前有效:${verified}\n源码范围:${baseline.source.scope};当前范围 hash:${baseline.sourceTreeSha256}\n原始评分:${data.baseline.overall?.score ?? 'unknown'}(新旧计分条件不同,不作为功能退步/进步比较)`, `历史 hash 漂移:${baseline.history.oldDriftedArtifacts ?? 'unknown'};本次漂移:${integrity.driftedArtifacts};本次缺失:${integrity.missingArtifacts}\n历史工作流/Chromium 通过:${historicalWorkflowPassed}/${historicalChromiumPassed};当前绑定通过:${workflowPassed}/${chromiumPassed}`, ...(selectedCases ? [`所选案例当前有效检查:${selectedCases.cases.reduce((sum, item) => sum + item.currentSelectedTests, 0)};未完成门禁:${JSON.stringify(selectedCases.remainingGates)};acceptance=NOT_CLAIMED`] : []), `\n## 结论\n\n- 状态:**${overall.status}**\n- 进度分:**${overall.score}/100**(${overall.scoreMeaning})\n- 实现状态:**${overall.implementationState}**\n- 阻塞项:**${overall.blockers}**\n`, "## 维度\n\n| 维度 | 分数 | 证据 |\n| --- | ---: | --- |\n" + Object.entries(dimensions).map(([k, v]) => `| ${k} | ${pct(v.score)} | ${v.numerator}/${v.denominator} · ${v.evidence} |`).join("\n"), `\n\n## 实现与覆盖\n\n- 工作流:${implementedWorkflows}/${workflows.length} implemented\n- 产品任务:${implementedTasks}/${productTasks.length} implemented\n- 桌面包:${packages.length},${count(packages, (x) => x.status === "implementing_gap")} implementing,${count(packages, (x) => x.validation === "NOT_RUN")} NOT_RUN\n- 路由分类:${routeClassified}/${routeTotal}\n- 行为回执:${verified}/${routeTotal} verified,${unverified} unverified\n- 路由类型:${Object.entries(data.catalog.counts ?? {}).map(([k, v]) => `${k}=${v}`).join(",") || "n/a"}\n- 缺口状态:${Object.entries(gapStatuses).map(([k, v]) => `${k}=${v}`).join(",") || "n/a"}\n`, `## 验证与证据\n\n- 最新工作流:${report.validation.latest};通过检查 ${workflowPassed},剩余 ${workflowRemaining.length}\n- Chromium 聚焦报告:${chromiumPassed}/${steps.length} PASS,${chromiumFailed} FAIL\n- 历史 full E2E:${data.product.fullE2ESmoke?.testsPassed ?? data.product.fullE2ESmoke?.passed ?? "n/a"}/${data.product.fullE2ESmoke?.testsTotal ?? data.product.fullE2ESmoke?.total ?? "n/a"},状态 ${data.product.fullE2ESmoke?.status ?? "n/a"}\n- 产品 Chromium 模块证据:${data.product.realChromiumEvidence?.status ?? "n/a"}\n- WASM 生命周期:${wasmComplete}/${wasmModules.length} complete;Web 产品 ${data.parity.webProductStatus ?? "n/a"};发布 ${data.parity.releaseStatus ?? "n/a"}\n- 证据 hash:${integrity.matchingArtifacts ?? 0}/${integrity.referencedArtifacts ?? 0} MATCH,漂移 ${integrity.driftedArtifacts ?? 0}\n`, "## 阻塞项\n\n" + (blockers.length ? blockers.map((b, i) => `${i + 1}. **${b.id}**:${b.message}(${b.evidence.join(", ")})`).join("\n") : "无"), "\n\n## 输入构件\n\n" + Object.values(inputs).map((x) => `- ${x.path} · ${x.present ? x.sha256 : "MISSING"}`).join("\n") + "\n", ].join("\n"); const jsonPath = "tests/golden/WBV2/completion-evaluation.json"; const markdownPath = "tests/golden/WBV2/completion-evaluation.md"; for (const [file, observed] of inputStamps) if (stamp(file) !== observed) throw new Error(`Input changed before publication: ${file}`); if (exists(currentBaselinePath) && sha(currentBaselinePath) !== baselineSha256) throw new Error(`Stored baseline hash mismatch: ${currentBaselinePath}`); if (!exists(currentBaselinePath)) atomicJSON(abs(currentBaselinePath), baseline); atomicJSON(abs(jsonPath), report); fs.writeFileSync(abs(`${markdownPath}.${process.pid}.tmp`), `${markdown}\n`); fs.renameSync(abs(`${markdownPath}.${process.pid}.tmp`), abs(markdownPath)); if (jsonOnly) process.stdout.write(`${JSON.stringify(report)}\n`); else if (markdownOnly) process.stdout.write(markdown); else process.stdout.write(`wbv2-completion status=${overall.status} score=${overall.score} blockers=${overall.blockers} verified=${verified}/${routeTotal} workflow=${report.validation.latest} currentCasesEligible=${selectedCases ? `${selectedCases.cases.filter(item => item.eligible).length}/${selectedCases.cases.length}` : 'NOT_SUPPLIED'}\n`); if (strict && overall.status !== "COMPLETE") process.exitCode = 2;