FreeCADWasmDev/tools/summarize_user_manual_tests.mjs

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const directory = path.join(root, 'docs/current/user-tests-20260915');
const runs = fs.readdirSync(directory,{withFileTypes:true}).filter(n=>n.isDirectory()&&n.name.startsWith('manual-')&&fs.existsSync(path.join(directory,n.name,'results.json'))).map(n=>n.name).sort();
const latest = new Map();
for (const run of runs) {
  const data = JSON.parse(fs.readFileSync(path.join(directory,run,'results.json')));
  for (const result of data.results) latest.set(result.id,{...result,run});
}
const text = fs.readFileSync(path.join(root,'docs/current/web-freecad-user-validation-manual.md'),'utf8');
const manual = [...text.matchAll(/^### ([A-Z]+-\d+) (.+)$/gm)].map(([,id,title])=>({id,title}));
const registry = JSON.parse(fs.readFileSync(path.join(root,'docs/current/feature-capability-registry.json')));
const modules = registry.modules.map(m=>({module:m.module,result:latest.get(`UI-03-${m.module}`)}));
const coverage = manual.map(item=>{
  if(item.id==='UI-03') return {...item,status:modules.every(m=>m.result?.status==='passed')?'passed':'failed',
    detail:`${modules.filter(m=>m.result?.status==='passed').length}/${modules.length} 模块入口通过；入口检查不等于模块业务验收`, evidence:modules.map(m=>`${m.result?.run}/report.md`)};
  const r=latest.get(item.id);
  return {...item,status:r?.status??'not_run',detail:r?.error??'仅限实际执行的步骤和断言',
    evidence:r?[`${r.run}/report.md`,...r.steps.filter(s=>s.evidence).map(s=>`${r.run}/${s.evidence.screenshot}`)]:[]};
});
const counts=Object.fromEntries(['passed','failed','blocked','not_run'].map(s=>[s,coverage.filter(c=>c.status===s).length]));
fs.writeFileSync(path.join(directory,'manual-coverage.json'),JSON.stringify({scope:'21 manual items, not 1452 command acceptance',counts,coverage,modules},null,2)+'\n');
const escapeCsv = s=>'"'+String(s??'').replaceAll('"','""')+'"';
fs.writeFileSync(path.join(directory,'command-coverage.csv'),'\uFEFF'+[
  ['capability_id','module','command','registered_implementation','user_command_test','module_entry_test'],
  ...registry.records.map(r=>[r.id,r.module,r.command,r.implementationStatus,'not_individually_tested',latest.get(`UI-03-${r.module}`)?.status??'not_run'])
].map(r=>r.map(escapeCsv).join(',')).join('\n')+'\n');
const rows=[...latest.values()];
const fail=rows.filter(r=>r.status==='failed');
const sourceErrors=fail.filter(r=>r.error?.includes('WebMacroDebugTask'));
const defect=(id,title,severity,reason,steps,evidence)=>({id,title,severity,reason,steps,evidence});
const evidenceFor = id => {const r=latest.get(id);return r?.steps.filter(s=>s.evidence).map(s=>`${r.run}/${s.evidence.screenshot}`)??[];};
const defects=[
  defect('UT-001','工作台渲染崩溃','P1','浏览器抛出 ReferenceError: WebMacroDebugTask is not defined；原生模块登记不能使 React 组件自动导入。',
    ['打开首页。','点击任一工作台导航，或访问 /workbench.html?module=Part。','观察只有导航而无工作台业务界面，Console 抛出 ReferenceError。'],evidenceFor('UI-03-Part')),
  defect('UT-002','分组创建缺少 Python json 模块','P1','主工作区 Create group 返回 native_view_failed: No module named json。',
    ['首页创建 Box。','展开 New Group。','点击 Create group。','记录错误消息以及树中未新增 Group。'],evidenceFor('DOC-02')),
  defect('UT-003','对象颜色保存重开丢失','P2','颜色编辑为 #cc3344 后保存 FCStd，重开显示默认 #73c6d4；Label 已保留。',
    ['创建 Box 并修改 Label。','将 Object Color 改为 #cc3344。','保存 FCStd 到 OPFS。','刷新后从 OPFS 打开，比较 Object Color。'],evidenceFor('DOC-01')),
  defect('UT-004','窗口宿主初始化协议及原生登记遗漏','P1','工作台能够渲染但窗口命令报 invalid_request / Native view command returned no result。窗口命令被套用几何契约，且原生处理函数未加入标准命令分发表。',
    ['打开工作台并等待窗口工具栏初始化。','检查顶部错误提示。','展开面板菜单并切换工具栏锁定。'],evidenceFor('UI-03-Part')),
  defect('UT-005','IFC 缺少真实 IfcOpenShell WASM 绑定','P1','真实 IFC 样本无法导入；当前 BIM 初始化仅有 declaration-only IfcOpenShell 替身，PartBox 无 IFC 原生导入器。不能将显式报错算作 IFC 支持。',
    ['打开 Web / Std_Open。','选择本机 AC14-FZK-Haus.ifc。','检查 file_open_failed 和 No native import handler is registered for .ifc。'],evidenceFor('FILE-02')),
];
const resolutionChecks = {
  'UT-001': modules.map(m=>`UI-03-${m.module}`),
  'UT-002': ['FIX-GROUP'],
  'UT-003': ['DOC-01'],
  'UT-004': [...modules.map(m=>`UI-03-${m.module}`),'FIX-WINDOW'],
  'UT-005': ['FILE-02','FIX-IFC-REOPEN'],
};
for (const defect of defects) {
  const checks = resolutionChecks[defect.id];
  defect.status = checks.every(id=>latest.get(id)?.status==='passed'
    && (defect.id!=='UT-004'||id==='FIX-WINDOW'||latest.get(id)?.steps.some(s=>s.name==='窗口宿主初始化及启动错误'&&s.status==='passed'))) ? 'resolved_by_retest' : 'open';
  defect.retestCases = checks;
  if (defect.id === 'UT-005') defect.resolutionScope = 'IfcOpenShellNative real IFC2X3/IFC4 import and FCStd persistence only; full ifcopenshell Python authoring/export API remains outside this closure.';
  defect.retestEvidence = checks.flatMap(evidenceFor);
  // Keep the original failure separate from newer passing screenshots.
  defect.originalEvidence = [];
  for (const run of runs) {
    const data = JSON.parse(fs.readFileSync(path.join(directory,run,'results.json')));
    const originalId = ['UT-001','UT-004'].includes(defect.id)?'UI-03-Part':defect.id==='UT-002'?'DOC-02':defect.id==='UT-005'?'FILE-02':'DOC-01';
    const record = data.results.find(r=>r.id===originalId&&r.status==='failed'
      && (defect.id!=='UT-004'||r.steps.some(s=>s.name==='窗口宿主初始化及启动错误'))
      && (defect.id!=='UT-005'||r.error?.includes('IFC native import')));
    if (record) { defect.originalEvidence = record.steps.filter(s=>s.evidence).map(s=>`${run}/${s.evidence.screenshot}`); break; }
  }
  defect.evidence = defect.originalEvidence;
}
fs.writeFileSync(path.join(directory,'defects.json'),JSON.stringify({defects,workbenchExceptionCount:sourceErrors.length},null,2)+'\n');
const short = s=>String(s??'').replaceAll('|','/').replaceAll('\n',' ').slice(0,220);
const lines=['# 用户手册测试总报告','',`生成时间：${new Date().toISOString()}`,'',
  `手册 ${manual.length} 项：通过 ${counts.passed}；失败 ${counts.failed}；阻塞/覆盖不足 ${counts.blocked}；未运行 ${counts.not_run}。`,
  '',`实际工作台入口 ${modules.length} 个，通过 ${modules.filter(m=>m.result?.status==='passed').length} 个。不得将路由登记 1452/1452 或独立演示页烟测作为上线验收。`,
  '', '## 上线结论','', `**当前不具备完整上线放行依据。** 登记缺陷 ${defects.length} 个，${defects.filter(d=>d.status==='resolved_by_retest').length} 个已通过针对性复测。手册尚未通过：${coverage.filter(c=>c.status!=='passed').map(c=>c.id+' '+c.status).join('；')||'无'}。本机性能样本通过不等于生产容量认证，入口可用不代表全部业务命令已经验收。IFC 导入通过仅指 IfcOpenShellNative 真实导入绑定，不代表完整 BIM 编辑/导出 API。`,
  '', '## 手册逐项覆盖','', '| 编号 | 项目 | 状态 | 依据 |','|---|---|---|---|',
  ...coverage.map(c=>`| ${c.id} | ${c.title} | ${c.status} | ${short(c.detail)} ${c.evidence[0]?`[过程](${c.evidence[0]})`:''} |`),
  '', '## 缺陷与修复复测','',...defects.flatMap(d=>[`### ${d.id} ${d.title} (${d.severity})`,'',`当前状态：${d.status}`,'',`原始问题：${d.reason}`,'',d.resolutionScope??'',...d.steps.map((s,i)=>`${i+1}. ${s}`),'',...d.originalEvidence.map((p,i)=>`[原始证据 ${i+1}](${p})`),'',...d.retestEvidence.map((p,i)=>`[修复复测 ${i+1}](${p})`),'']),
  '## 批次与耗时','',...runs.map(run=>{const r=JSON.parse(fs.readFileSync(path.join(directory,run,'results.json')));return `- [${run}](${run}/report.md)：${r.results.length} 条案例记录，累计案例耗时 ${r.results.reduce((n,x)=>n+(x.seconds??0),0).toFixed(1)} 秒。`; }),
  '', '## 解释与限制','',
  '- 同一手册项目以最新批次为准，旧失败和复测证据均保留。旧 summary.tsv 不纳入本报告。',
  '- 首批 FILE-03 在关闭完成前点击打开，属于测试时序错误；补充等待后单独复测。',
  '- 首批后段 Chrome GPU 进程退出并报临时空间不足，超时不能直接归为业务缺陷。受影响的入口使用新浏览器复测。',
  '- PERF-02 以最新批次为准：最初仅有 Box 循环，后续扩展为 20 轮真实 STEP 导入、修改、保存、刷新重开及进程 RSS 采样。',
  '- PERF-01 的 small/medium/large 是所提供桌面示例的相对分级，最大约 577 KiB；帧回调频率不是 CAD 绘制帧率。本轮不证明更大生产模型的性能。',
  '- 软件 WebGL、单一 Chrome 测试，不代表 Edge/Firefox/移动设备兼容。',
  '- command-coverage.csv 列出 1452 条登记命令，单命令验收仍为 not_individually_tested；不从入口检查推导业务通过。',
  '- 原 PDF 中文字体异常，本次严格使用每批 manual-snapshot.md；不沿用 PDF 已可交付的早期错误结论。',
  '', '## 后续顺序','',
  '1. 按缺陷状态处理仍开放的问题；已修复问题保留原始失败和复测证据。',
  '2. 正式构建必须通过严格 TypeScript 检查；检查 repair-checks-*/results.json 中的当前记录。',
  '3. 对失败入口及其依赖的文件、链接、结构命令补做真实业务测试；不需要重跑无关已通过项。',
  '4. 补充真实分级样本和浏览器/硬件矩阵，再进行性能及上线签署。'];
fs.writeFileSync(path.join(directory,'TEST-REPORT.md'),lines.join('\n')+'\n');
const esc=s=>String(s??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
fs.writeFileSync(path.join(directory,'index.html'),`<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>Web FreeCAD 用户测试证据</title><style>body{font:15px/1.6 sans-serif;margin:32px auto;max-width:1100px;padding:0 20px;color:#20252b}table{border-collapse:collapse;width:100%}td,th{padding:8px;border:1px solid #ccc;text-align:left}img{max-width:100%;height:auto;border:1px solid #ccc}a{color:#126d84}h2{margin-top:32px}.failed{color:#b21e2f}.passed{color:#246842}.blocked{color:#725407}</style><h1>Web FreeCAD 用户测试证据</h1><p>手册 ${manual.length} 项：通过 ${counts.passed}，失败 ${counts.failed}，阻塞 ${counts.blocked}。当前不可据此上线放行。</p><p><a href="TEST-REPORT.md">总报告</a> · <a href="manual-coverage.json">覆盖矩阵</a> · <a href="command-coverage.csv">逐命令未测清单</a></p><table><tr><th>编号</th><th>项目</th><th>状态</th><th>过程</th></tr>${coverage.map(c=>`<tr><td>${esc(c.id)}</td><td>${esc(c.title)}</td><td class="${esc(c.status)}">${esc(c.status)}</td><td>${c.evidence[0]?`<a href="${esc(c.evidence[0])}">执行记录</a>`:''}</td></tr>`).join('')}</table><h2>缺陷截图</h2>${defects.map(d=>`<h3>${esc(d.id+' '+d.title+' '+d.severity)}</h3><p>${esc(d.reason)}</p>${d.evidence.slice(-1).map(p=>`<a href="${esc(p)}"><img loading="lazy" src="${esc(p)}" alt="${esc(d.title)}"></a>`).join('')}`).join('')}<h2>全部案例截图</h2>${rows.map(r=>`<details><summary>${esc(r.id+' '+r.title+' '+r.status)}</summary>${r.steps.filter(s=>s.evidence).map(s=>`<p>${esc(s.number+'. '+s.name+' / '+s.status)}</p><a href="${esc(r.run+'/'+s.evidence.screenshot)}"><img loading="lazy" src="${esc(r.run+'/'+s.evidence.screenshot)}" alt="${esc(s.name)}"></a>`).join('')}</details>`).join('')}</html>`);
console.log(JSON.stringify({directory,manualItems:manual.length,counts,moduleEntries:modules.length,moduleEntriesPassed:modules.filter(m=>m.result?.status==='passed').length}));