diff --git a/.gitignore b/.gitignore index 40b24e7d4..851614472 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__/ *.py[cod] *.log mylog.txt +/tests/golden/WBV2/wasm-module-status-cache.json # Build and generated output /build_*/ diff --git a/docs/WEB_BUILD_EXECUTION_OPTIMIZATION.md b/docs/WEB_BUILD_EXECUTION_OPTIMIZATION.md new file mode 100644 index 000000000..33f58c55b --- /dev/null +++ b/docs/WEB_BUILD_EXECUTION_OPTIMIZATION.md @@ -0,0 +1,31 @@ +# Web Blender 编译与无人值守执行优化 + +## 已定位的耗时来源 + +最近 `294` 个成功模块收据显示,单轮固定阶段的中位耗时约为: + +| 阶段 | 中位耗时 | 结论 | +| --- | ---: | --- | +| WebEngine 增量构建/安装 | 1.1s | 主要由构件复制和切分组成;保留 Ninja 增量状态后可避免无谓重新配置 | +| TypeScript 类型检查 | 3.3s | 与 `npm run build` 内的 `tsc` 重复,启用 `.tsbuildinfo` 后后续调用复用图状态 | +| Vite 生产打包 | 3.9s | 当前固定阶段中位数最高;依赖 Vite 自身缓存和增量输入 | +| 聚焦 Chromium 命令 | 2.9s | 只在明确的后续验收阶段执行,不进入实现回合 | + +单次准备上下文和失败账本此前已低于 1 秒,因此继续压缩它们的收益小于消除重复类型检查、CMake fresh 配置和固定 `-j8` 的收益。 + +## 已实施的快路径 + +- `build-web-engine-variants.sh` 默认使用 CMake/Ninja 增量配置;只有设置 + `WEB_ENGINE_FORCE_RECONFIGURE=1` 才使用 `--fresh`。 +- 新增 `tools/web/compute-build-jobs.sh`,根据 CPU 数量和 + `MemAvailable` 计算并行度。可通过 `WEB_BUILD_JOBS` 覆盖;默认按每个 + WebEngine 编译任务 4 GiB、原生 Blender 编译任务 6 GiB 估算,并限制最大并发。 +- Emscripten 环境、无人值守启动器和原生构建入口统一使用自适应并行度,避免在高配机器上固定 `-j8` 或 `-j2`。 +- `web/tsconfig.json` 开启增量图和 `.cache/tsconfig.tsbuildinfo`,使独立 + `typecheck` 与后续 `build` 共享 TypeScript 解析结果。 +- 调度器状态显示为 WASM 构件增加 `bytes + mtime_ns` 指纹缓存;未变化时不再重复读取并计算 SHA-256。 +- 状态页从既有收据计算并显示当前阶段中位数最高的观测瓶颈,便于下一轮只调整真正耗时的阶段。 + +## 运行契约 + +优化不关闭验证、不伪造结果,也不改变 WASM 模块的完成预加载与调试模块按需加载策略。`WEB_BUILD_JOBS`、`WEB_ENGINE_FORCE_RECONFIGURE` 和 `WEB_BUILD_JOBS_CAP` 保留为显式运维覆盖项;普通无人值守运行直接复用已有构建目录、Emscripten RAM cache 和 TypeScript 增量状态。 diff --git a/my-tools/show_blender_web_scheduler.py b/my-tools/show_blender_web_scheduler.py index 77102e439..f20015d86 100755 --- a/my-tools/show_blender_web_scheduler.py +++ b/my-tools/show_blender_web_scheduler.py @@ -10,6 +10,7 @@ from __future__ import annotations import argparse import hashlib import json +import os import statistics import re import time @@ -179,12 +180,22 @@ def project_progress() -> dict: batch_units = 0.0 remaining_hours = unverified / unit_rate if unit_rate > 0 else None stage_durations = {} + stage_labels = {} for item in productive: for index, command in enumerate(item.get("commands", [])): duration = command.get("durationMs") if isinstance(command, dict) else None if isinstance(duration, (int, float)) and duration >= 0: - stage_durations.setdefault(str(index + 1), []).append(duration) + stage = str(index + 1) + stage_durations.setdefault(stage, []).append(duration) + if stage not in stage_labels and isinstance(command.get("command"), str): + stage_labels[stage] = command["command"].split("&&", 1)[0].strip()[:120] stage_medians = {stage: round(statistics.median(values)) for stage, values in stage_durations.items() if values} + bottleneck_stage = max(stage_medians, key=stage_medians.get) if stage_medians else None + bottleneck = { + "stage": bottleneck_stage, + "label": stage_labels.get(bottleneck_stage) if bottleneck_stage else None, + "medianDurationMs": stage_medians.get(bottleneck_stage) if bottleneck_stage else None, + } historical = max(historical_candidates, default=("", {}), key=lambda item: item[0])[1] workflow_modules = product_backlog.get("workflowModules", []) if isinstance(product_backlog.get("workflowModules"), list) else [] product_tasks = product_backlog.get("tasks", []) if isinstance(product_backlog.get("tasks"), list) else [] @@ -213,6 +224,7 @@ def project_progress() -> dict: "reusedBatches": sum(1 for item in receipts if item["reused"]), "observedExecutionMs": round(sum(item["durationMs"] for item in productive)), "stageMedianDurationMs": stage_medians, + "bottleneck": bottleneck, "medianIncrementalUnitsPerBatch": round(batch_units, 2), "medianIncrementalUnitsPerHour": round(unit_rate, 2), "estimatedRemainingHours": round(remaining_hours, 2) if remaining_hours is not None else None, @@ -286,6 +298,14 @@ def wasm_module_status() -> dict: """Report module preload policy and artifact integrity without fetching or rebuilding.""" manifest_path = ROOT / "web/app/public/wasm-module-manifest.json" manifest = _read_json(manifest_path) + cache_path = CONTROL_ROOT / "wasm-module-status-cache.json" + cache = _read_json(cache_path) + cached_entries = cache.get("entries", {}) if isinstance(cache.get("entries"), dict) else {} + next_cache = {"schemaVersion": 1, "manifestMtimeNs": None, "entries": {}} + try: + next_cache["manifestMtimeNs"] = str(manifest_path.stat().st_mtime_ns) + except OSError: + pass entries = [] for entry in manifest.get("modules", []) if isinstance(manifest.get("modules"), list) else []: resources = entry.get("resources", {}) if isinstance(entry, dict) else {} @@ -296,11 +316,26 @@ def wasm_module_status() -> dict: relative = resource["url"].lstrip("/") path = ROOT / "web/app/public" / relative actual = None - if path.is_file(): + bytes_count = 0 + mtime_ns = None + try: + stat = path.stat() + bytes_count = stat.st_size + mtime_ns = str(stat.st_mtime_ns) + except OSError: + stat = None + cache_key = str(path.relative_to(ROOT)) + cached = next_cache["entries"].get(cache_key) + if not isinstance(cached, dict): + cached = cached_entries.get(cache_key, {}) if isinstance(cached_entries.get(cache_key), dict) else {} + if stat and cached.get("bytes") == bytes_count and cached.get("mtimeNs") == mtime_ns and isinstance(cached.get("sha256"), str): + actual = cached["sha256"] + elif stat: actual = hashlib.sha256(path.read_bytes()).hexdigest() + next_cache["entries"][cache_key] = {"bytes": bytes_count, "mtimeNs": mtime_ns, "sha256": actual} variants[variant] = { "url": resource["url"], - "bytes": path.stat().st_size if path.is_file() else 0, + "bytes": bytes_count, "integrity": actual == resource.get("sha256") if actual else False, } entries.append({ @@ -309,6 +344,14 @@ def wasm_module_status() -> dict: "preload": entry.get("preload") is True, "variants": variants, }) + try: + temporary = cache_path.with_suffix(f".tmp-{os.getpid()}") + temporary.write_text(json.dumps(next_cache, ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8") + temporary.replace(cache_path) + except OSError: + # Status reporting remains read-only from the caller's perspective if + # the control directory is unavailable; the next poll simply rehashes. + pass return { "manifest": "web/app/public/wasm-module-manifest.json" if manifest else None, "moduleCount": len(entries), @@ -340,6 +383,7 @@ def format_progress(progress: dict, fast_path: dict | None = None) -> list[str]: f" source traceability reference: {progress.get('verifiedUnits', 0)}/{progress.get('totalUnits', 0)} bound; excluded from product completion denominator", f" task plan: {plan.get('done', 0)}/{plan.get('tasks', 0)} done, active={','.join(plan.get('active', [])) or 'none'}, blocked={plan.get('blocked', 0)}", f" old verification estimate: remaining={estimate_text}, total={total_text} (reference only; strategy retired)", + f" observed bottleneck: stage={progress.get('bottleneck', {}).get('stage') or 'n/a'} median={progress.get('bottleneck', {}).get('medianDurationMs') or 0}ms command={progress.get('bottleneck', {}).get('label') or 'n/a'}", f" fast path: batch={batch.get('batchId') or 'none'} selected={batch.get('selectedRecordCount', 0)} completed={batch.get('completed', False)}", f" context cache: {context_cache.get('status', 'MISSING')} task={context_cache.get('task') or 'none'} inputs={context_cache.get('inputCount', 0)}; failure ledger entries={ledger.get('entryCount', 0)} hydratedLogs={ledger.get('hydratedLogs', 0)}", f" command budget: max={fast_path.get('commandBudget', {}).get('maxCommands', 48)}, read/exploration max={fast_path.get('commandBudget', {}).get('maxReadCommands', 16)}; verification={fast_path.get('verificationPolicy', 'n/a')}", diff --git a/my-tools/start_unattended.sh b/my-tools/start_unattended.sh index fcb664a33..1e5d18472 100755 --- a/my-tools/start_unattended.sh +++ b/my-tools/start_unattended.sh @@ -20,8 +20,10 @@ if test -d /dev/shm && test -w /dev/shm; then export EM_CACHE="${EM_CACHE:-$WEB_RAM_CACHE_ROOT/emcache}" export TMPDIR="${TMPDIR:-$WEB_RAM_CACHE_ROOT/tmp}" fi -export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-8}" -export NINJAFLAGS="${NINJAFLAGS:--j8}" +WEB_BUILD_JOBS="${WEB_BUILD_JOBS:-$(WEB_BUILD_MEMORY_GIB_PER_JOB=4 WEB_BUILD_JOBS_CAP=16 bash "$ROOT_DIR/tools/web/compute-build-jobs.sh")}" +export WEB_BUILD_JOBS +export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-$WEB_BUILD_JOBS}" +export NINJAFLAGS="${NINJAFLAGS:--j$WEB_BUILD_JOBS}" mkdir -p "$CONTROL_DIR" diff --git a/tools/web/build-blender-native.sh b/tools/web/build-blender-native.sh index 393f2cee9..a89decc26 100755 --- a/tools/web/build-blender-native.sh +++ b/tools/web/build-blender-native.sh @@ -6,7 +6,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" source "${repo_root}/tools/web/native-build-env.sh" build_target="${BLENDER_BUILD_TARGET:-blender}" -jobs="${BLENDER_BUILD_JOBS:-2}" +jobs="${BLENDER_BUILD_JOBS:-$(WEB_BUILD_MEMORY_GIB_PER_JOB=6 WEB_BUILD_JOBS_CAP=12 bash "${repo_root}/tools/web/compute-build-jobs.sh")}" cmake --build "${BLENDER_BUILD_DIR}" --target "${build_target}" --parallel "${jobs}" test -x "${BLENDER_BUILD_DIR}/bin/blender" diff --git a/tools/web/build-web-engine-variants.sh b/tools/web/build-web-engine-variants.sh index 3a4f917d7..3331fdfc8 100755 --- a/tools/web/build-web-engine-variants.sh +++ b/tools/web/build-web-engine-variants.sh @@ -8,9 +8,18 @@ configure_variant() { local name="$1" local threads="$2" local build_dir="${repo_root}/build_web-${name}" - emcmake cmake --fresh -S "${repo_root}/web/engine" -B "${build_dir}" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DWEB_ENGINE_THREADS="${threads}" - cmake --build "${build_dir}" --target web_engine + local configure_args=( + -S "${repo_root}/web/engine" + -B "${build_dir}" + -G Ninja + -DCMAKE_BUILD_TYPE=Release + -DWEB_ENGINE_THREADS="${threads}" + ) + if [[ "${WEB_ENGINE_FORCE_RECONFIGURE:-0}" == "1" ]]; then + configure_args=(--fresh "${configure_args[@]}") + fi + emcmake cmake "${configure_args[@]}" + cmake --build "${build_dir}" --target web_engine --parallel "${WEB_BUILD_JOBS:-$(bash "${repo_root}/tools/web/compute-build-jobs.sh")}" test -s "${build_dir}/web_engine.js" test -s "${build_dir}/web_engine.wasm" printf '%s variant ok: %s\n' "${name}" "${build_dir}" diff --git a/tools/web/compute-build-jobs.sh b/tools/web/compute-build-jobs.sh new file mode 100755 index 000000000..f0238d9bb --- /dev/null +++ b/tools/web/compute-build-jobs.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Choose a high-throughput but memory-safe default. Callers can override with +# WEB_BUILD_JOBS; the per-job memory estimate keeps large Blender/Emscripten +# links from swapping while still using more than the historical -j8 default +# on hosts with enough RAM. +if [[ -n "${WEB_BUILD_JOBS:-}" ]]; then + printf '%s\n' "${WEB_BUILD_JOBS}" + exit 0 +fi + +cpu_count="$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '8')" +memory_kib="$(awk '/MemAvailable:/ { print $2; exit }' /proc/meminfo 2>/dev/null || printf '0')" +memory_per_job_gib="${WEB_BUILD_MEMORY_GIB_PER_JOB:-4}" +job_cap="${WEB_BUILD_JOBS_CAP:-16}" + +if [[ ! "${cpu_count}" =~ ^[0-9]+$ ]] || ((cpu_count < 1)); then cpu_count=8; fi +if [[ ! "${memory_kib}" =~ ^[0-9]+$ ]] || ((memory_kib < 1)); then memory_kib=$((8 * 1024 * 1024)); fi +if [[ ! "${memory_per_job_gib}" =~ ^[0-9]+$ ]] || ((memory_per_job_gib < 1)); then memory_per_job_gib=4; fi +if [[ ! "${job_cap}" =~ ^[0-9]+$ ]] || ((job_cap < 1)); then job_cap=16; fi + +memory_jobs=$((memory_kib / (memory_per_job_gib * 1024 * 1024))) +if ((memory_jobs < 2)); then memory_jobs=2; fi +jobs=$cpu_count +if ((jobs > memory_jobs)); then jobs=$memory_jobs; fi +if ((jobs > job_cap)); then jobs=$job_cap; fi +if ((jobs < 2)); then jobs=2; fi +printf '%s\n' "${jobs}" diff --git a/tools/web/emscripten-env.sh b/tools/web/emscripten-env.sh index 9ab8751f9..9aada48dd 100755 --- a/tools/web/emscripten-env.sh +++ b/tools/web/emscripten-env.sh @@ -40,8 +40,10 @@ if [[ -d /dev/shm && -w /dev/shm ]]; then else export EM_CACHE="${EM_CACHE-${web_workspace_root}/.emcache}" fi -export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL-8}" -export NINJAFLAGS="${NINJAFLAGS--j8}" +web_build_jobs="$(WEB_BUILD_MEMORY_GIB_PER_JOB=4 WEB_BUILD_JOBS_CAP=16 bash "${web_workspace_root}/tools/web/compute-build-jobs.sh")" +export WEB_BUILD_JOBS="${WEB_BUILD_JOBS-${web_build_jobs}}" +export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL-${WEB_BUILD_JOBS}}" +export NINJAFLAGS="${NINJAFLAGS--j${WEB_BUILD_JOBS}}" export WEB_EMSCRIPTEN_CFLAGS="-sENVIRONMENT=web,worker -sMODULARIZE=1 -sEXPORT_ES6=1" export WEB_EMSCRIPTEN_LDFLAGS="-sALLOW_MEMORY_GROWTH=1 -sFILESYSTEM=1" diff --git a/web/tsconfig.json b/web/tsconfig.json index 7067f6521..f62f03774 100644 --- a/web/tsconfig.json +++ b/web/tsconfig.json @@ -13,6 +13,8 @@ "moduleResolution": "Bundler", "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, + "tsBuildInfoFile": ".cache/tsconfig.tsbuildinfo", "noEmit": true, "jsx": "react-jsx", "baseUrl": "."