perf(web): reuse incremental build and status caches

.gitignore

@@ -6,6 +6,7 @@ __pycache__/
 *.py[cod]
 *.log
 mylog.txt
+/tests/golden/WBV2/wasm-module-status-cache.json
 
 # Build and generated output
 /build_*/

docs/WEB_BUILD_EXECUTION_OPTIMIZATION.md

@@ -0,0 +1,31 @@
+# Web Blender 编译与无人值守执行优化
+
+## 已定位的耗时来源
+
+最近 `294` 个成功模块收据显示，单轮固定阶段的中位耗时约为：
+
+| 阶段 | 中位耗时 | 结论 |
+| --- | ---: | --- |
+| WebEngine 增量构建/安装 | 1.1s | 主要由构件复制和切分组成；保留 Ninja 增量状态后可避免无谓重新配置 |
+| TypeScript 类型检查 | 3.3s | 与 `npm run build` 内的 `tsc` 重复，启用 `.tsbuildinfo` 后后续调用复用图状态 |
+| Vite 生产打包 | 3.9s | 当前固定阶段中位数最高；依赖 Vite 自身缓存和增量输入 |
+| 聚焦 Chromium 命令 | 2.9s | 只在明确的后续验收阶段执行，不进入实现回合 |
+
+单次准备上下文和失败账本此前已低于 1 秒，因此继续压缩它们的收益小于消除重复类型检查、CMake fresh 配置和固定 `-j8` 的收益。
+
+## 已实施的快路径
+
+- `build-web-engine-variants.sh` 默认使用 CMake/Ninja 增量配置；只有设置
+  `WEB_ENGINE_FORCE_RECONFIGURE=1` 才使用 `--fresh`。
+- 新增 `tools/web/compute-build-jobs.sh`，根据 CPU 数量和
+  `MemAvailable` 计算并行度。可通过 `WEB_BUILD_JOBS` 覆盖；默认按每个
+  WebEngine 编译任务 4 GiB、原生 Blender 编译任务 6 GiB 估算，并限制最大并发。
+- Emscripten 环境、无人值守启动器和原生构建入口统一使用自适应并行度，避免在高配机器上固定 `-j8` 或 `-j2`。
+- `web/tsconfig.json` 开启增量图和 `.cache/tsconfig.tsbuildinfo`，使独立
+  `typecheck` 与后续 `build` 共享 TypeScript 解析结果。
+- 调度器状态显示为 WASM 构件增加 `bytes + mtime_ns` 指纹缓存；未变化时不再重复读取并计算 SHA-256。
+- 状态页从既有收据计算并显示当前阶段中位数最高的观测瓶颈，便于下一轮只调整真正耗时的阶段。
+
+## 运行契约
+
+优化不关闭验证、不伪造结果，也不改变 WASM 模块的完成预加载与调试模块按需加载策略。`WEB_BUILD_JOBS`、`WEB_ENGINE_FORCE_RECONFIGURE` 和 `WEB_BUILD_JOBS_CAP` 保留为显式运维覆盖项；普通无人值守运行直接复用已有构建目录、Emscripten RAM cache 和 TypeScript 增量状态。

my-tools/show_blender_web_scheduler.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 import argparse
 import hashlib
 import json
+import os
 import statistics
 import re
 import time
@@ -179,12 +180,22 @@ def project_progress() -> dict:
         batch_units = 0.0
     remaining_hours = unverified / unit_rate if unit_rate > 0 else None
     stage_durations = {}
+    stage_labels = {}
     for item in productive:
         for index, command in enumerate(item.get("commands", [])):
             duration = command.get("durationMs") if isinstance(command, dict) else None
             if isinstance(duration, (int, float)) and duration >= 0:
-                stage_durations.setdefault(str(index + 1), []).append(duration)
+                stage = str(index + 1)
+                stage_durations.setdefault(stage, []).append(duration)
+                if stage not in stage_labels and isinstance(command.get("command"), str):
+                    stage_labels[stage] = command["command"].split("&&", 1)[0].strip()[:120]
     stage_medians = {stage: round(statistics.median(values)) for stage, values in stage_durations.items() if values}
+    bottleneck_stage = max(stage_medians, key=stage_medians.get) if stage_medians else None
+    bottleneck = {
+        "stage": bottleneck_stage,
+        "label": stage_labels.get(bottleneck_stage) if bottleneck_stage else None,
+        "medianDurationMs": stage_medians.get(bottleneck_stage) if bottleneck_stage else None,
+    }
     historical = max(historical_candidates, default=("", {}), key=lambda item: item[0])[1]
     workflow_modules = product_backlog.get("workflowModules", []) if isinstance(product_backlog.get("workflowModules"), list) else []
     product_tasks = product_backlog.get("tasks", []) if isinstance(product_backlog.get("tasks"), list) else []
@@ -213,6 +224,7 @@ def project_progress() -> dict:
         "reusedBatches": sum(1 for item in receipts if item["reused"]),
         "observedExecutionMs": round(sum(item["durationMs"] for item in productive)),
         "stageMedianDurationMs": stage_medians,
+        "bottleneck": bottleneck,
         "medianIncrementalUnitsPerBatch": round(batch_units, 2),
         "medianIncrementalUnitsPerHour": round(unit_rate, 2),
         "estimatedRemainingHours": round(remaining_hours, 2) if remaining_hours is not None else None,
@@ -286,6 +298,14 @@ def wasm_module_status() -> dict:
     """Report module preload policy and artifact integrity without fetching or rebuilding."""
     manifest_path = ROOT / "web/app/public/wasm-module-manifest.json"
     manifest = _read_json(manifest_path)
+    cache_path = CONTROL_ROOT / "wasm-module-status-cache.json"
+    cache = _read_json(cache_path)
+    cached_entries = cache.get("entries", {}) if isinstance(cache.get("entries"), dict) else {}
+    next_cache = {"schemaVersion": 1, "manifestMtimeNs": None, "entries": {}}
+    try:
+        next_cache["manifestMtimeNs"] = str(manifest_path.stat().st_mtime_ns)
+    except OSError:
+        pass
     entries = []
     for entry in manifest.get("modules", []) if isinstance(manifest.get("modules"), list) else []:
         resources = entry.get("resources", {}) if isinstance(entry, dict) else {}
@@ -296,11 +316,26 @@ def wasm_module_status() -> dict:
             relative = resource["url"].lstrip("/")
             path = ROOT / "web/app/public" / relative
             actual = None
-            if path.is_file():
+            bytes_count = 0
+            mtime_ns = None
+            try:
+                stat = path.stat()
+                bytes_count = stat.st_size
+                mtime_ns = str(stat.st_mtime_ns)
+            except OSError:
+                stat = None
+            cache_key = str(path.relative_to(ROOT))
+            cached = next_cache["entries"].get(cache_key)
+            if not isinstance(cached, dict):
+                cached = cached_entries.get(cache_key, {}) if isinstance(cached_entries.get(cache_key), dict) else {}
+            if stat and cached.get("bytes") == bytes_count and cached.get("mtimeNs") == mtime_ns and isinstance(cached.get("sha256"), str):
+                actual = cached["sha256"]
+            elif stat:
                 actual = hashlib.sha256(path.read_bytes()).hexdigest()
+            next_cache["entries"][cache_key] = {"bytes": bytes_count, "mtimeNs": mtime_ns, "sha256": actual}
             variants[variant] = {
                 "url": resource["url"],
-                "bytes": path.stat().st_size if path.is_file() else 0,
+                "bytes": bytes_count,
                 "integrity": actual == resource.get("sha256") if actual else False,
             }
         entries.append({
@@ -309,6 +344,14 @@ def wasm_module_status() -> dict:
             "preload": entry.get("preload") is True,
             "variants": variants,
         })
+    try:
+        temporary = cache_path.with_suffix(f".tmp-{os.getpid()}")
+        temporary.write_text(json.dumps(next_cache, ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8")
+        temporary.replace(cache_path)
+    except OSError:
+        # Status reporting remains read-only from the caller's perspective if
+        # the control directory is unavailable; the next poll simply rehashes.
+        pass
     return {
         "manifest": "web/app/public/wasm-module-manifest.json" if manifest else None,
         "moduleCount": len(entries),
@@ -340,6 +383,7 @@ def format_progress(progress: dict, fast_path: dict | None = None) -> list[str]:
         f"  source traceability reference: {progress.get('verifiedUnits', 0)}/{progress.get('totalUnits', 0)} bound; excluded from product completion denominator",
         f"  task plan: {plan.get('done', 0)}/{plan.get('tasks', 0)} done, active={','.join(plan.get('active', [])) or 'none'}, blocked={plan.get('blocked', 0)}",
         f"  old verification estimate: remaining={estimate_text}, total={total_text} (reference only; strategy retired)",
+        f"  observed bottleneck: stage={progress.get('bottleneck', {}).get('stage') or 'n/a'} median={progress.get('bottleneck', {}).get('medianDurationMs') or 0}ms command={progress.get('bottleneck', {}).get('label') or 'n/a'}",
         f"  fast path: batch={batch.get('batchId') or 'none'} selected={batch.get('selectedRecordCount', 0)} completed={batch.get('completed', False)}",
         f"  context cache: {context_cache.get('status', 'MISSING')} task={context_cache.get('task') or 'none'} inputs={context_cache.get('inputCount', 0)}; failure ledger entries={ledger.get('entryCount', 0)} hydratedLogs={ledger.get('hydratedLogs', 0)}",
         f"  command budget: max={fast_path.get('commandBudget', {}).get('maxCommands', 48)}, read/exploration max={fast_path.get('commandBudget', {}).get('maxReadCommands', 16)}; verification={fast_path.get('verificationPolicy', 'n/a')}",

my-tools/start_unattended.sh

@@ -20,8 +20,10 @@ if test -d /dev/shm && test -w /dev/shm; then
   export EM_CACHE="${EM_CACHE:-$WEB_RAM_CACHE_ROOT/emcache}"
   export TMPDIR="${TMPDIR:-$WEB_RAM_CACHE_ROOT/tmp}"
 fi
-export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-8}"
+WEB_BUILD_JOBS="${WEB_BUILD_JOBS:-$(WEB_BUILD_MEMORY_GIB_PER_JOB=4 WEB_BUILD_JOBS_CAP=16 bash "$ROOT_DIR/tools/web/compute-build-jobs.sh")}"
-export NINJAFLAGS="${NINJAFLAGS:--j8}"
+export WEB_BUILD_JOBS
+export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-$WEB_BUILD_JOBS}"
+export NINJAFLAGS="${NINJAFLAGS:--j$WEB_BUILD_JOBS}"
 
 mkdir -p "$CONTROL_DIR"
 

tools/web/build-blender-native.sh

@@ -6,7 +6,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 source "${repo_root}/tools/web/native-build-env.sh"
 
 build_target="${BLENDER_BUILD_TARGET:-blender}"
-jobs="${BLENDER_BUILD_JOBS:-2}"
+jobs="${BLENDER_BUILD_JOBS:-$(WEB_BUILD_MEMORY_GIB_PER_JOB=6 WEB_BUILD_JOBS_CAP=12 bash "${repo_root}/tools/web/compute-build-jobs.sh")}"
 cmake --build "${BLENDER_BUILD_DIR}" --target "${build_target}" --parallel "${jobs}"
 test -x "${BLENDER_BUILD_DIR}/bin/blender"
 

tools/web/build-web-engine-variants.sh

@@ -8,9 +8,18 @@ configure_variant() {
   local name="$1"
   local threads="$2"
   local build_dir="${repo_root}/build_web-${name}"
-  emcmake cmake --fresh -S "${repo_root}/web/engine" -B "${build_dir}" -G Ninja \
+  local configure_args=(
-    -DCMAKE_BUILD_TYPE=Release -DWEB_ENGINE_THREADS="${threads}"
+    -S "${repo_root}/web/engine"
-  cmake --build "${build_dir}" --target web_engine
+    -B "${build_dir}"
+    -G Ninja
+    -DCMAKE_BUILD_TYPE=Release
+    -DWEB_ENGINE_THREADS="${threads}"
+  )
+  if [[ "${WEB_ENGINE_FORCE_RECONFIGURE:-0}" == "1" ]]; then
+    configure_args=(--fresh "${configure_args[@]}")
+  fi
+  emcmake cmake "${configure_args[@]}"
+  cmake --build "${build_dir}" --target web_engine --parallel "${WEB_BUILD_JOBS:-$(bash "${repo_root}/tools/web/compute-build-jobs.sh")}"
   test -s "${build_dir}/web_engine.js"
   test -s "${build_dir}/web_engine.wasm"
   printf '%s variant ok: %s\n' "${name}" "${build_dir}"

tools/web/compute-build-jobs.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Choose a high-throughput but memory-safe default. Callers can override with
+# WEB_BUILD_JOBS; the per-job memory estimate keeps large Blender/Emscripten
+# links from swapping while still using more than the historical -j8 default
+# on hosts with enough RAM.
+if [[ -n "${WEB_BUILD_JOBS:-}" ]]; then
+  printf '%s\n' "${WEB_BUILD_JOBS}"
+  exit 0
+fi
+
+cpu_count="$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '8')"
+memory_kib="$(awk '/MemAvailable:/ { print $2; exit }' /proc/meminfo 2>/dev/null || printf '0')"
+memory_per_job_gib="${WEB_BUILD_MEMORY_GIB_PER_JOB:-4}"
+job_cap="${WEB_BUILD_JOBS_CAP:-16}"
+
+if [[ ! "${cpu_count}" =~ ^[0-9]+$ ]] || ((cpu_count < 1)); then cpu_count=8; fi
+if [[ ! "${memory_kib}" =~ ^[0-9]+$ ]] || ((memory_kib < 1)); then memory_kib=$((8 * 1024 * 1024)); fi
+if [[ ! "${memory_per_job_gib}" =~ ^[0-9]+$ ]] || ((memory_per_job_gib < 1)); then memory_per_job_gib=4; fi
+if [[ ! "${job_cap}" =~ ^[0-9]+$ ]] || ((job_cap < 1)); then job_cap=16; fi
+
+memory_jobs=$((memory_kib / (memory_per_job_gib * 1024 * 1024)))
+if ((memory_jobs < 2)); then memory_jobs=2; fi
+jobs=$cpu_count
+if ((jobs > memory_jobs)); then jobs=$memory_jobs; fi
+if ((jobs > job_cap)); then jobs=$job_cap; fi
+if ((jobs < 2)); then jobs=2; fi
+printf '%s\n' "${jobs}"

tools/web/emscripten-env.sh

@@ -40,8 +40,10 @@ if [[ -d /dev/shm && -w /dev/shm ]]; then
 else
   export EM_CACHE="${EM_CACHE-${web_workspace_root}/.emcache}"
 fi
-export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL-8}"
+web_build_jobs="$(WEB_BUILD_MEMORY_GIB_PER_JOB=4 WEB_BUILD_JOBS_CAP=16 bash "${web_workspace_root}/tools/web/compute-build-jobs.sh")"
-export NINJAFLAGS="${NINJAFLAGS--j8}"
+export WEB_BUILD_JOBS="${WEB_BUILD_JOBS-${web_build_jobs}}"
+export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL-${WEB_BUILD_JOBS}}"
+export NINJAFLAGS="${NINJAFLAGS--j${WEB_BUILD_JOBS}}"
 export WEB_EMSCRIPTEN_CFLAGS="-sENVIRONMENT=web,worker -sMODULARIZE=1 -sEXPORT_ES6=1"
 export WEB_EMSCRIPTEN_LDFLAGS="-sALLOW_MEMORY_GROWTH=1 -sFILESYSTEM=1"
 

web/tsconfig.json

@@ -13,6 +13,8 @@
     "moduleResolution": "Bundler",
     "resolveJsonModule": true,
     "isolatedModules": true,
+    "incremental": true,
+    "tsBuildInfoFile": ".cache/tsconfig.tsbuildinfo",
     "noEmit": true,
     "jsx": "react-jsx",
     "baseUrl": "."