From ca6fe460301e6ce5d56444d7b2389d961748a58a Mon Sep 17 00:00:00 2001 From: wangdequan Date: Wed, 12 Aug 2026 02:36:09 -0400 Subject: [PATCH] feat: add self-contained offline development resources --- README.md | 4 +- config/offline-resources.json | 215 +++++++++ docs/offline-development.zh-CN.md | 91 ++++ docs/project-runtime.zh-CN.md | 6 +- native/occt-history/build.sh | 18 +- native/planegcs/build.sh | 12 +- package.json | 6 + scripts/bootstrap-node.sh | 11 + scripts/build-opencamlib-wasm.mjs | 8 +- scripts/offline-resource-lib.mjs | 660 ++++++++++++++++++++++++++ scripts/offline-shell.sh | 10 + scripts/run-real-verification.mjs | 7 +- scripts/verify-freecad-source.mjs | 21 + tests/offlineResourceLibrary.test.mjs | 45 ++ 14 files changed, 1102 insertions(+), 12 deletions(-) create mode 100644 config/offline-resources.json create mode 100644 docs/offline-development.zh-CN.md create mode 100644 scripts/offline-resource-lib.mjs create mode 100755 scripts/offline-shell.sh create mode 100644 tests/offlineResourceLibrary.test.mjs diff --git a/README.md b/README.md index a4a5e0e..23ed613 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ FreeCAD-aligned web CAD frontend baseline for the BitBybit runtime boundary. +本机断网开发、资源同步、校验和恢复入口见 [离线资源库说明](docs/offline-development.zh-CN.md)。 + ## Current scope - React + Vite frontend shell @@ -25,7 +27,7 @@ The current milestone is a facade-first integration slice. Part Design document ./npmw run dev ``` -The repository uses a project-local, checksum-verified Node.js `22.23.2` runtime and npm `10.9.8`. The wrapper downloads it to the ignored `.runtime/` directory and never changes `/usr/bin/node`; this removes the engine warning caused by running npm with an older system Node. `./nodew` runs the same pinned Node directly. See [docs/project-runtime.zh-CN.md](docs/project-runtime.zh-CN.md) for recovery and CI rules. +The repository uses a project-local, checksum-verified Node.js `22.23.2` runtime and npm `10.9.8`. The wrapper restores it from the local resource library first, or downloads it when online, into the ignored `.runtime/` directory and never changes `/usr/bin/node`; this removes the engine warning caused by running npm with an older system Node. `./nodew` runs the same pinned Node directly. See [docs/project-runtime.zh-CN.md](docs/project-runtime.zh-CN.md) for recovery and CI rules. Build verification: diff --git a/config/offline-resources.json b/config/offline-resources.json new file mode 100644 index 0000000..3cde047 --- /dev/null +++ b/config/offline-resources.json @@ -0,0 +1,215 @@ +{ + "schemaVersion": 1, + "project": "bitbybit-web-cad", + "libraryDirectory": "web-freecad-bitbybit", + "platform": "linux-x64-debian13", + "node": { + "version": "22.23.2", + "npmVersion": "10.9.8", + "archive": ".runtime/downloads/node-v22.23.2-linux-x64.tar.xz", + "libraryPath": "toolchains/node/node-v22.23.2-linux-x64.tar.xz" + }, + "npmLocks": [ + "package-lock.json", + "cnc_wams_gpt6/linuxcnc-master/web/package-lock.json" + ], + "archives": [ + { + "id": "project-node-modules", + "source": "node_modules", + "restore": "node_modules", + "libraryPath": "archives/project-node-modules.tar.zst", + "required": true + }, + { + "id": "freecad-source", + "source": ".cache/freecad/FreeCAD", + "restore": ".cache/freecad/FreeCAD", + "libraryPath": "sources/freecad-1.1.1-0108fd4b.tar.zst", + "revision": "0108fd4b4850cc46e625b60e53cea7a7bbe69f8d", + "required": true + }, + { + "id": "bitbybit-source", + "source": ".cache/bitbybit/bitbybit", + "restore": ".cache/bitbybit/bitbybit", + "libraryPath": "sources/bitbybit-d3f52bd1.tar.zst", + "revision": "d3f52bd1fcf78148b60800d69ed6b42e53f4ad06", + "required": true + }, + { + "id": "occt-source", + "sourceEnvironment": "OCCT_SOURCE_DIR", + "sourceCandidates": [ + ".cache/occt/occt", + "../working/WebCam/occt" + ], + "restore": ".cache/occt/occt", + "libraryPath": "sources/occt-8-bb75017e.tar.zst", + "revision": "bb75017e915f6509a4bfa8b551f853352754c2da", + "required": true + }, + { + "id": "opencamlib-source", + "source": "OpenCAMLib", + "restore": "OpenCAMLib", + "libraryPath": "sources/opencamlib-95b036fe.tar.zst", + "revision": "95b036fe28ce6d77c97b98e5fbc337904ae49560", + "required": true + }, + { + "id": "camotics-worktree", + "source": "CAMotics", + "restore": "CAMotics", + "libraryPath": "sources/camotics-e84665f2-worktree.tar.zst", + "revision": "e84665f2fa9d1151f03282ac7e01320bc65e015b", + "preserveWorktree": true, + "required": true + }, + { + "id": "cbang-worktree", + "source": ".cache/camotics-native/cbang", + "restore": ".cache/camotics-native/cbang", + "libraryPath": "sources/cbang-9b6672a0-worktree.tar.zst", + "revision": "9b6672a0e2b800a909d799ffa3ab46774cf171b8", + "required": true + }, + { + "id": "linuxcnc-worktree", + "source": "cnc_wams_gpt6/linuxcnc-master", + "restore": "cnc_wams_gpt6/linuxcnc-master", + "libraryPath": "sources/linuxcnc-web-58f50e6d-worktree.tar.zst", + "revision": "58f50e6d66287bc31e6d906e5c33658a348658e4", + "preserveWorktree": true, + "required": true + }, + { + "id": "freecad-native-oracle", + "source": ".cache/freecad/install-native", + "restore": ".cache/freecad/install-native", + "libraryPath": "artifacts/freecad-install-native.tar.zst", + "required": true + }, + { + "id": "freecad-desktop-oracle", + "source": ".cache/freecad/install-desktop", + "restore": ".cache/freecad/install-desktop", + "libraryPath": "artifacts/freecad-install-desktop.tar.zst", + "required": true + }, + { + "id": "freecad-sysroot", + "source": ".cache/freecad/sysroot", + "restore": ".cache/freecad/sysroot", + "libraryPath": "artifacts/freecad-sysroot.tar.zst", + "required": true + }, + { + "id": "occt-history-dist", + "source": "native/occt-history/dist", + "restore": "native/occt-history/dist", + "libraryPath": "artifacts/occt-history-dist.tar.zst", + "required": true + }, + { + "id": "planegcs-dist", + "source": "native/planegcs/dist", + "restore": "native/planegcs/dist", + "libraryPath": "artifacts/planegcs-dist.tar.zst", + "required": true + }, + { + "id": "freecad-build-native", + "source": ".cache/freecad/build-native", + "restore": ".cache/freecad/build-native", + "libraryPath": "build-caches/freecad-build-native.tar.zst", + "required": false + }, + { + "id": "freecad-build-desktop", + "source": ".cache/freecad/build-desktop", + "restore": ".cache/freecad/build-desktop", + "libraryPath": "build-caches/freecad-build-desktop.tar.zst", + "required": false + }, + { + "id": "occt-history-build", + "source": ".cache/bitbybit/occt-history-build-offline", + "restore": ".cache/bitbybit/occt-history-build-offline", + "libraryPath": "build-caches/occt-history-build.tar.zst", + "required": false + }, + { + "id": "opencamlib-build", + "source": ".cache/opencamlib-wasm", + "restore": ".cache/opencamlib-wasm", + "libraryPath": "build-caches/opencamlib-wasm.tar.zst", + "required": false + }, + { + "id": "opencamlib-boost", + "source": ".cache/opencamlib-boost", + "restore": ".cache/opencamlib-boost", + "libraryPath": "build-caches/opencamlib-boost.tar.zst", + "required": false + } + ], + "mirrors": [ + { + "id": "playwright-browsers", + "sourceEnvironment": "PLAYWRIGHT_BROWSERS_PATH", + "sourceCandidates": [ + "~/.cache/ms-playwright" + ], + "libraryPath": "browsers/ms-playwright", + "required": true + }, + { + "id": "boost-headers", + "sourceCandidates": [ + "/usr/include/boost" + ], + "libraryPath": "system/headers/boost", + "restore": ".cache/offline-sysroot/include/boost", + "required": true + }, + { + "id": "eigen3-headers", + "sourceCandidates": [ + "/usr/include/eigen3" + ], + "libraryPath": "system/headers/eigen3", + "restore": ".cache/offline-sysroot/include/eigen3", + "required": true + }, + { + "id": "debian-package-cache", + "sourceCandidates": [ + "/var/cache/apt/archives" + ], + "libraryPath": "system/debian13-amd64/packages", + "includePattern": "\\.deb$", + "recursive": false, + "required": true + }, + { + "id": "freecad-package-cache", + "source": ".cache/freecad", + "libraryPath": "system/debian13-amd64/freecad-packages", + "includePattern": "\\.deb$", + "required": true + } + ], + "hostTools": { + "node": "22.23.2", + "npm": "10.9.8", + "cmake": "3.31.6", + "ninja": "1.12.1", + "emscripten": "3.1.69", + "chrome": "150.0.7871.128", + "firefox": "140.12.0esr", + "playwright": "1.61.1", + "boost": "1.83.0", + "eigen3": "3.4.0" + } +} diff --git a/docs/offline-development.zh-CN.md b/docs/offline-development.zh-CN.md new file mode 100644 index 0000000..b106c6f --- /dev/null +++ b/docs/offline-development.zh-CN.md @@ -0,0 +1,91 @@ +# 本机离线资源库 + +本项目的统一资源库默认位于： + +```text +/home/mes123456/resource-library/web-freecad-bitbybit +``` + +可通过 `WEB_FREECAD_RESOURCE_LIBRARY` 或各命令的 `--library=PATH` 改为其他位置。资源库不进入项目 Git；仓库保存资源定义，资源库自己的 `manifest.json` 保存每个归档和镜像的字节数、文件数、SHA-256、上游提交及工作树状态。 + +## 资源范围 + +`config/offline-resources.json` 固定以下资源： + +- Node 22.23.2 原始发行包、两个 `package-lock.json` 对应的最小 npm 内容缓存，以及当前项目 `node_modules` 快照； +- FreeCAD 1.1.1（含 GSL、OndselSolver、AddonManager、测试子模块）、bitbybit、OCCT 8、OpenCAMLib、CAMotics/C! 和 LinuxCNC Web 工作树； +- FreeCAD headless/desktop Oracle、FreeCAD sysroot、OCCT/PlaneGCS 原生 Worker 制品和可选增量构建目录； +- Playwright Chromium/Firefox/WebKit、Boost/Eigen 头文件，以及本机 `/var/cache/apt/archives` 中的 Debian 安装包池。 + +LinuxCNC 和 CAMotics 存在尚未合入各自上游提交的本地适配，因此保存完整工作树快照。`working_ocl/` 是上述源码的历史重复副本，不作为规范资源；构建入口实际使用的 `OpenCAMLib/`、`CAMotics/` 和 `.cache/bitbybit/bitbybit` 才进入资源库。 + +OCCT 增量构建缓存固定恢复到 `.cache/bitbybit/occt-history-build-offline`，其 CMake 缓存引用项目内 `.cache/occt/occt`，不包含旧工作区的绝对源码路径。 + +## 同步与校验 + +```bash +./npmw run offline:inventory +./npmw run offline:sync +./npmw run offline:check +./npmw run offline:smoke +``` + +`offline:sync` 只读取现有本机资源，不从互联网下载。首次同步会压缩数 GB 的源码和构建缓存；后续在上游提交、本地适配、工具链或 lockfile 变化后重新执行。 + +`offline:check` 逐项重新计算 SHA-256。`offline:smoke` 先在临时空目录用资源库 npm 缓存执行 `npm ci --offline`，再运行运行时/FreeCAD 源码/原生 Worker 检查和生产构建。npm 的 `--offline` 为强制模式，缺包时直接失败，不会访问 registry。资源库保持只读；离线 shell 按 manifest 哈希把 npm 内容初始化到项目 `.cache/offline-npm`，npm 的访问索引只写入这份工作缓存。 + +## 从资源库恢复 + +恢复操作必须显式限定资源，且默认拒绝覆盖现有目录： + +```bash +./npmw run offline:restore -- --only=project-node-modules,freecad-source,occt-source +``` + +确认目标可替换后使用： + +```bash +./npmw run offline:restore -- --only=project-node-modules --force +``` + +也可验证到独立临时工作目录： + +```bash +target_dir="$(mktemp -d)" +./npmw run offline:restore -- --target="$target_dir" --only=freecad-source +``` + +不要对工作区根目录使用递归删除；恢复工具会检查每个目标必须位于指定 `--target` 下。 + +## 离线开发环境 + +以下命令启动一个明确禁止 npm 网络回退的 shell： + +```bash +./scripts/offline-shell.sh +``` + +也可以直接运行单条命令： + +```bash +./scripts/offline-shell.sh ./npmw run dev +./scripts/offline-shell.sh ./npmw run build +./scripts/offline-shell.sh ./npmw run ci:real -- --lane=wasm +``` + +环境中会设置 `npm_config_offline=true`、`PLAYWRIGHT_BROWSERS_PATH`、`FIREFOX_BIN`、`WEBKIT_BIN`、`FREECAD_SOURCE_OFFLINE=1`、项目内 `OCCT_SOURCE_DIR` 和固定 Chrome 路径。Firefox/WebKit 使用 manifest 中实际归档的可执行 revision，不要求它恰好等于当前 Playwright 包的默认下载 revision。Node bootstrap 在项目 `.runtime/downloads` 缺少发行包时也会先查资源库；设置 `WEB_FREECAD_OFFLINE=1` 后若资源库仍缺包，会直接失败而不调用 `curl`。 + +## 系统包恢复边界 + +源码、npm、Node、浏览器缓存和项目制品可以直接恢复到普通用户目录。CMake、Ninja、Emscripten、Chrome、Firefox、Qt、V8 和 FreeCAD 原生依赖属于 Debian 系统包；资源库保留了本机 apt 包池，并生成标准 `Packages`/`Packages.gz` 索引。安装系统包仍需要管理员权限。例如先把两个本地目录登记成仅本机 apt 源： + +```bash +printf '%s\n' \ + 'deb [trusted=yes] file:/home/mes123456/resource-library/web-freecad-bitbybit/system/debian13-amd64/packages ./' \ + 'deb [trusted=yes] file:/home/mes123456/resource-library/web-freecad-bitbybit/system/debian13-amd64/freecad-packages ./' \ + | sudo tee /etc/apt/sources.list.d/web-freecad-offline.list +sudo apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/web-freecad-offline.list -o Dir::Etc::sourceparts=- +sudo apt-get --no-download install emscripten cmake ninja-build +``` + +这一步只在系统工具被卸载或重装主机时需要。日常断网开发直接使用当前已安装工具链。 diff --git a/docs/project-runtime.zh-CN.md b/docs/project-runtime.zh-CN.md index 73e5e9e..b04e64d 100644 --- a/docs/project-runtime.zh-CN.md +++ b/docs/project-runtime.zh-CN.md @@ -4,7 +4,7 @@ 本项目的 `package.json` 声明 Node.js `>=22`，而原开发环境实际使用系统 `/usr/bin/node v20.19.2` 与 `/usr/bin/npm 9.2.0`。npm 的 `EBADENGINE` 是运行 npm 的解释器版本不满足依赖声明所产生的警告；仅修改 `engines`、PATH 文档或 Vite 配置都不会改变当前解释器。 -项目现在锁定 Node.js `22.23.2` 与 npm `10.9.8`，并把官方归档下载到被 Git 忽略的 `.runtime/`。`./npmw`、`./nodew` 和 `scripts/bootstrap-node.sh` 是项目运行时的唯一启动路径，系统 Node 不会被替换或修改。 +项目现在锁定 Node.js `22.23.2` 与 npm `10.9.8`，并把官方归档恢复或下载到被 Git 忽略的 `.runtime/`。启动器优先读取本机统一资源库，在线模式仅在资源库缺失时访问官方发行目录。`./npmw`、`./nodew` 和 `scripts/bootstrap-node.sh` 是项目运行时的唯一启动路径，系统 Node 不会被替换或修改。 版本与校验值来源： @@ -51,7 +51,7 @@ ./npmw --version ``` -期望输出分别为 `v22.23.2` 和 `10.9.8`。第一次运行会下载约 30 MB 的 Node Linux x64 归档，后续运行复用校验通过的缓存。 +期望输出分别为 `v22.23.2` 和 `10.9.8`。第一次运行会优先从 `~/resource-library/web-freecad-bitbybit` 恢复约 30 MB 的 Node Linux x64 归档；在线且资源库缺失时才下载，后续运行复用校验通过的缓存。 ## 4. 启动器的安全行为 @@ -73,7 +73,7 @@ **仍出现 `EBADENGINE`**：检查命令是否以 `./npmw` 开头；检查 `./nodew --version`。直接执行系统 `npm` 在 `.npmrc` 的 `engine-strict=true` 下应失败，这是故意的防误用信号。 -**下载失败**：确认能访问 `https://nodejs.org`，删除确定的 `.runtime/downloads/node-v22.23.2-*` 缓存后重新执行 `./npmw install`。不要手工替换归档或跳过校验。 +**归档不可用**：先执行 `./npmw run offline:check` 检查统一资源库。在线模式可确认能访问 `https://nodejs.org`，删除确定的 `.runtime/downloads/node-v22.23.2-*` 缓存后重新执行 `./npmw install`。不要手工替换归档或跳过校验。 **架构不支持**：在当前开发机使用 Linux x64；新增平台时必须先把官方归档和 SHA-256 加入 `config/node-runtime.env`，并增加对应 CI job。 diff --git a/native/occt-history/build.sh b/native/occt-history/build.sh index 1b8058f..14c1ab1 100755 --- a/native/occt-history/build.sh +++ b/native/occt-history/build.sh @@ -2,8 +2,22 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -OCCT_SOURCE_DIR="${OCCT_SOURCE_DIR:-/home/mes123456/working/WebCam/occt}" -OCCT_BUILD_DIR="${OCCT_BUILD_DIR:-${ROOT_DIR}/.cache/bitbybit/occt-history-build}" +LOCAL_OCCT_SOURCE_DIR="${ROOT_DIR}/.cache/occt/occt" +LEGACY_OCCT_SOURCE_DIR="/home/mes123456/working/WebCam/occt" +if [[ -z "${OCCT_SOURCE_DIR:-}" ]]; then + if [[ -f "${LOCAL_OCCT_SOURCE_DIR}/CMakeLists.txt" ]]; then + OCCT_SOURCE_DIR="${LOCAL_OCCT_SOURCE_DIR}" + else + OCCT_SOURCE_DIR="${LEGACY_OCCT_SOURCE_DIR}" + fi +fi +if [[ -z "${OCCT_BUILD_DIR:-}" ]]; then + if [[ "${OCCT_SOURCE_DIR}" == "${LOCAL_OCCT_SOURCE_DIR}" ]]; then + OCCT_BUILD_DIR="${ROOT_DIR}/.cache/bitbybit/occt-history-build-offline" + else + OCCT_BUILD_DIR="${ROOT_DIR}/.cache/bitbybit/occt-history-build" + fi +fi DIST_DIR="${OCCT_HISTORY_DIST_DIR:-${ROOT_DIR}/native/occt-history/dist}" PUBLIC_DIR="${OCCT_HISTORY_PUBLIC_DIR:-${ROOT_DIR}/public/native/occt-history}" JOBS="${JOBS:-$(nproc)}" diff --git a/native/planegcs/build.sh b/native/planegcs/build.sh index 9e2b3b7..daa7725 100755 --- a/native/planegcs/build.sh +++ b/native/planegcs/build.sh @@ -7,6 +7,9 @@ PLANEGCS_SOURCE_DIR="${FREECAD_SOURCE_DIR}/src/Mod/Sketcher/App/planegcs" DIST_DIR="${PLANEGCS_DIST_DIR:-${ROOT_DIR}/native/planegcs/dist}" PUBLIC_DIR="${PLANEGCS_PUBLIC_DIR:-${ROOT_DIR}/public/native/planegcs}" HOST_HEADERS_DIR="${ROOT_DIR}/.cache/bitbybit/planegcs-host-headers" +OFFLINE_HEADERS_DIR="${ROOT_DIR}/.cache/offline-sysroot/include" +BOOST_INCLUDE_DIR="${BOOST_INCLUDE_DIR:-$([[ -d "${OFFLINE_HEADERS_DIR}/boost" ]] && printf '%s' "${OFFLINE_HEADERS_DIR}" || printf '%s' /usr/include)}" +EIGEN3_INCLUDE_DIR="${EIGEN3_INCLUDE_DIR:-$([[ -d "${OFFLINE_HEADERS_DIR}/eigen3" ]] && printf '%s' "${OFFLINE_HEADERS_DIR}/eigen3" || printf '%s' /usr/include/eigen3)}" if [[ ! -f "${PLANEGCS_SOURCE_DIR}/GCS.cpp" ]]; then echo "Locked FreeCAD planegcs source not found: ${PLANEGCS_SOURCE_DIR}" >&2 @@ -16,14 +19,17 @@ if ! command -v em++ >/dev/null; then echo "Emscripten em++ is required." >&2 exit 1 fi -if [[ ! -d /usr/include/eigen3 ]] || [[ ! -d /usr/include/boost ]]; then +if [[ ! -d "${EIGEN3_INCLUDE_DIR}" ]] || [[ ! -d "${BOOST_INCLUDE_DIR}/boost" ]]; then echo "Eigen3 and Boost headers are required." >&2 exit 1 fi mkdir -p "${DIST_DIR}" "${PUBLIC_DIR}" "${HOST_HEADERS_DIR}" +if [[ -L "${HOST_HEADERS_DIR}/boost" && "$(readlink "${HOST_HEADERS_DIR}/boost")" != "${BOOST_INCLUDE_DIR}/boost" ]]; then + rm "${HOST_HEADERS_DIR}/boost" +fi if [[ ! -e "${HOST_HEADERS_DIR}/boost" ]]; then - ln -s /usr/include/boost "${HOST_HEADERS_DIR}/boost" + ln -s "${BOOST_INCLUDE_DIR}/boost" "${HOST_HEADERS_DIR}/boost" fi em++ \ "${PLANEGCS_SOURCE_DIR}/Geo.cpp" \ @@ -36,7 +42,7 @@ em++ \ -I"${FREECAD_SOURCE_DIR}/src" \ -I"${PLANEGCS_SOURCE_DIR}" \ -I"${HOST_HEADERS_DIR}" \ - -I/usr/include/eigen3 \ + -I"${EIGEN3_INCLUDE_DIR}" \ -std=c++20 -O2 -DNDEBUG -DEIGEN_NO_DEBUG -fexceptions -pthread --bind \ -sMODULARIZE=1 -sEXPORT_ES6=1 -sENVIRONMENT=web,worker,node \ -sPTHREAD_POOL_SIZE=2 \ diff --git a/package.json b/package.json index 21bd069..dd673e6 100644 --- a/package.json +++ b/package.json @@ -260,6 +260,12 @@ "ci:real": "node scripts/run-real-verification.mjs", "ci:real:chrome": "node scripts/run-real-verification.mjs --lane=chrome --phase=execute", "ci:check:chrome": "node scripts/run-real-verification.mjs --lane=chrome --phase=check", + "offline:inventory": "node scripts/offline-resource-lib.mjs inventory", + "offline:sync": "node scripts/offline-resource-lib.mjs sync", + "offline:check": "node scripts/offline-resource-lib.mjs check", + "offline:restore": "node scripts/offline-resource-lib.mjs restore", + "offline:smoke": "node scripts/offline-resource-lib.mjs smoke", + "test:offline-tools": "node --test tests/offlineResourceLibrary.test.mjs", "test:browser-matrix": "node scripts/run-browser-matrix.mjs", "check:browser-matrix": "node scripts/check-browser-matrix.mjs", "check:runtime": "node scripts/check-runtime.mjs", diff --git a/scripts/bootstrap-node.sh b/scripts/bootstrap-node.sh index cbbd8d6..90f7036 100755 --- a/scripts/bootstrap-node.sh +++ b/scripts/bootstrap-node.sh @@ -24,6 +24,8 @@ esac archive_name="node-v${NODE_VERSION}-${platform}-${arch}.${archive_ext}" archive_path="$runtime_root/downloads/$archive_name" runtime_dir="$runtime_root/node-v${NODE_VERSION}-${platform}-${arch}" +resource_home="${WEB_FREECAD_RESOURCE_LIBRARY:-${HOME}/resource-library/web-freecad-bitbybit}" +resource_archive="$resource_home/toolchains/node/$archive_name" if [ -x "$runtime_dir/bin/node" ] && [ "$($runtime_dir/bin/node --version)" = "v$NODE_VERSION" ]; then printf '%s\n' "$runtime_dir" @@ -41,6 +43,15 @@ if [ -e "$archive_path" ]; then fi fi +if [ ! -e "$archive_path" ] && [ -e "$resource_archive" ]; then + cp "$resource_archive" "$archive_path" +fi + +if [ ! -e "$archive_path" ] && { [ "${WEB_FREECAD_OFFLINE:-0}" = "1" ] || [ "${npm_config_offline:-false}" = "true" ]; }; then + echo "Pinned Node archive is unavailable in the offline resource library: $resource_archive" >&2 + exit 2 +fi + if [ ! -e "$archive_path" ]; then temp_archive="$archive_path.part" rm -f "$temp_archive" diff --git a/scripts/build-opencamlib-wasm.mjs b/scripts/build-opencamlib-wasm.mjs index d93fe38..bae9737 100644 --- a/scripts/build-opencamlib-wasm.mjs +++ b/scripts/build-opencamlib-wasm.mjs @@ -1,5 +1,5 @@ import { execFileSync } from 'node:child_process' -import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { access, cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { resolve } from 'node:path' const root = resolve(new URL('..', import.meta.url).pathname) @@ -7,11 +7,15 @@ const source = resolve(root, 'OpenCAMLib') const build = resolve(root, '.cache/opencamlib-wasm') const boost = resolve(root, '.cache/opencamlib-boost') const install = resolve(root, 'public/vendor/opencamlib') +const offlineBoost = resolve(root, '.cache/offline-sysroot/include/boost') +const systemBoost = process.env.BOOST_INCLUDE_DIR + ? resolve(process.env.BOOST_INCLUDE_DIR, 'boost') + : '/usr/include/boost' const revision = execFileSync('git', ['-C', source, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() await rm(boost, { recursive: true, force: true }) await mkdir(boost, { recursive: true }) -await cp('/usr/include/boost', resolve(boost, 'boost'), { recursive: true }) +await cp(await access(offlineBoost).then(() => offlineBoost, () => systemBoost), resolve(boost, 'boost'), { recursive: true }) await mkdir(build, { recursive: true }) const configure = [ diff --git a/scripts/offline-resource-lib.mjs b/scripts/offline-resource-lib.mjs new file mode 100644 index 0000000..7212845 --- /dev/null +++ b/scripts/offline-resource-lib.mjs @@ -0,0 +1,660 @@ +import { createHash } from 'node:crypto' +import { spawn, spawnSync } from 'node:child_process' +import { + access, + cp, + lstat, + mkdir, + mkdtemp, + open, + readFile, + readdir, + readlink, + rename, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises' +import { homedir, platform, tmpdir } from 'node:os' +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' +import { fileURLToPath } from 'node:url' +import { gzipSync } from 'node:zlib' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const configPath = resolve(root, 'config/offline-resources.json') +const config = JSON.parse(await readFile(configPath, 'utf8')) +const command = process.argv[2] || 'help' +const values = new Map(process.argv.slice(3).filter((argument) => argument.startsWith('--') && argument.includes('=')).map((argument) => { + const offset = argument.indexOf('=') + return [argument.slice(2, offset), argument.slice(offset + 1)] +})) +const flags = new Set(process.argv.slice(3).filter((argument) => argument.startsWith('--') && !argument.includes('=')).map((argument) => argument.slice(2))) +const defaultLibrary = resolve(homedir(), 'resource-library', config.libraryDirectory) +const library = resolve(values.get('library') || process.env.WEB_FREECAD_RESOURCE_LIBRARY || defaultLibrary) +const manifestPath = resolve(library, 'manifest.json') +const selectedIds = new Set((values.get('only') || '').split(',').filter(Boolean)) +const isSelected = (resource) => selectedIds.size === 0 || selectedIds.has(resource.id) +const zstdLevel = values.get('compression-level') || '3' +const knownIds = new Set(['node-runtime', 'npm-cache', ...config.archives.map((entry) => entry.id), ...config.mirrors.map((entry) => entry.id)]) +for (const id of selectedIds) if (!knownIds.has(id)) fail(`Unknown offline resource id: ${id}`) + +function print(value) { + process.stdout.write(`${typeof value === 'string' ? value : JSON.stringify(value, null, 2)}\n`) +} + +function fail(message) { + throw new Error(message) +} + +async function exists(path) { + return access(path).then(() => true, () => false) +} + +function expandHome(path) { + return path === '~' ? homedir() : path.startsWith(`~${sep}`) ? resolve(homedir(), path.slice(2)) : path +} + +function resolveWorkspacePath(path) { + const expanded = expandHome(path) + return isAbsolute(expanded) ? expanded : resolve(root, expanded) +} + +async function resourceSource(resource) { + const candidates = [] + if (resource.sourceEnvironment && process.env[resource.sourceEnvironment]) candidates.push(process.env[resource.sourceEnvironment]) + if (resource.source) candidates.push(resource.source) + candidates.push(...(resource.sourceCandidates || [])) + for (const candidate of candidates) { + const path = resolveWorkspacePath(candidate) + if (await exists(path)) return path + } + return null +} + +function resourceLibraryPath(resource) { + return resolve(library, resource.libraryPath) +} + +function resourceRestorePath(resource, targetRoot = root) { + return resolve(targetRoot, resource.restore) +} + +async function run(program, args, options = {}) { + await new Promise((resolveRun, rejectRun) => { + const child = spawn(program, args, { + cwd: options.cwd || root, + env: options.env || process.env, + stdio: options.stdio || 'inherit', + }) + child.once('error', rejectRun) + child.once('exit', (code, signal) => { + if (code === 0) resolveRun() + else rejectRun(new Error(`${program} exited with ${code ?? signal}`)) + }) + }) +} + +async function sha256(path) { + const digest = createHash('sha256') + const handle = await open(path, 'r') + try { + for await (const chunk of handle.createReadStream()) digest.update(chunk) + } finally { + await handle.close().catch(() => {}) + } + return digest.digest('hex') +} + +async function describeFile(path) { + const metadata = await stat(path) + return { bytes: metadata.size, sha256: await sha256(path) } +} + +async function walkFiles(path, options = {}, prefix = '') { + const files = [] + for (const entry of await readdir(path, { withFileTypes: true })) { + const relativePath = prefix ? join(prefix, entry.name) : entry.name + const absolutePath = resolve(path, entry.name) + if (entry.isDirectory() && options.recursive !== false) files.push(...await walkFiles(absolutePath, options, relativePath)) + else if ((entry.isFile() || entry.isSymbolicLink()) && (!options.matcher || options.matcher.test(relativePath))) files.push(relativePath) + } + return files +} + +async function describeTree(path) { + const paths = (await walkFiles(path)).sort() + const digest = createHash('sha256') + let bytes = 0 + for (const relativePath of paths) { + const absolutePath = resolve(path, relativePath) + const metadata = await lstat(absolutePath) + digest.update(`${relativePath}\0${metadata.mode & 0o777}\0`) + if (metadata.isSymbolicLink()) { + digest.update(`link:${await readlink(absolutePath)}\0`) + continue + } + bytes += metadata.size + const handle = await open(absolutePath, 'r') + try { + for await (const chunk of handle.createReadStream()) digest.update(chunk) + } finally { + await handle.close().catch(() => {}) + } + digest.update('\0') + } + return { files: paths.length, bytes, sha256: digest.digest('hex') } +} + +async function gitMetadata(path) { + const top = spawnSync('git', ['-C', path, 'rev-parse', '--show-toplevel'], { encoding: 'utf8' }) + if (top.status !== 0) return null + const topLevel = resolve(top.stdout.trim()) + const revision = spawnSync('git', ['-C', path, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).stdout.trim() + const status = spawnSync('git', ['-C', path, 'status', '--short', '--untracked-files=all'], { + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }).stdout + return { + revision, + worktreeRoot: relative(root, topLevel), + sourceWithinWorktree: relative(topLevel, path) || '.', + dirty: status.length > 0, + statusSha256: createHash('sha256').update(status).digest('hex'), + statusEntries: status.split(/\r?\n/).filter(Boolean).length, + } +} + +async function replacePath(temporary, destination) { + await mkdir(dirname(destination), { recursive: true }) + await rm(destination, { recursive: true, force: true }) + await rename(temporary, destination) +} + +async function createArchive(resource) { + const source = await resourceSource(resource) + if (!source) { + if (resource.required) fail(`Required source is missing for ${resource.id}`) + print(`[offline:sync] skip optional ${resource.id}: source is missing`) + return null + } + const sourceGit = await gitMetadata(source) + if (resource.revision && sourceGit?.revision !== resource.revision) { + fail(`${resource.id} revision is ${sourceGit?.revision || 'unknown'}, expected ${resource.revision}`) + } + const destination = resourceLibraryPath(resource) + const temporary = `${destination}.part-${process.pid}` + await mkdir(dirname(destination), { recursive: true }) + await rm(temporary, { force: true }) + print(`[offline:sync] archive ${resource.id} <- ${source}`) + await run('tar', [ + '--create', + `--file=${temporary}`, + '--use-compress-program', + `zstd -T0 -${zstdLevel}`, + '--numeric-owner', + '--owner=0', + '--group=0', + '--directory', + source, + '.', + ]) + await replacePath(temporary, destination) + return { + id: resource.id, + kind: 'archive', + path: relative(library, destination), + ...await describeFile(destination), + source: sourceGit, + } +} + +async function syncMirror(resource) { + const source = await resourceSource(resource) + if (!source) { + if (resource.required) fail(`Required source is missing for ${resource.id}`) + print(`[offline:sync] skip optional ${resource.id}: source is missing`) + return null + } + const destination = resourceLibraryPath(resource) + const temporary = `${destination}.part-${process.pid}` + await rm(temporary, { recursive: true, force: true }) + await mkdir(temporary, { recursive: true }) + const files = await walkFiles(source, { + matcher: resource.includePattern ? new RegExp(resource.includePattern) : null, + recursive: resource.recursive, + }) + print(`[offline:sync] mirror ${resource.id} (${files.length} files) <- ${source}`) + for (const relativePath of files) { + const from = resolve(source, relativePath) + const to = resolve(temporary, relativePath) + const metadata = await lstat(from) + await mkdir(dirname(to), { recursive: true }) + if (metadata.isSymbolicLink()) await symlink(await readlink(from), to) + else await cp(from, to, { preserveTimestamps: true }) + } + if (resource.id === 'debian-package-cache' || resource.id === 'freecad-package-cache') { + const index = spawnSync('dpkg-scanpackages', ['.', '/dev/null'], { + cwd: temporary, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }) + if (index.status !== 0) fail(`dpkg-scanpackages failed for ${resource.id}: ${index.stderr.trim()}`) + await writeFile(resolve(temporary, 'Packages'), index.stdout) + await writeFile(resolve(temporary, 'Packages.gz'), gzipSync(index.stdout, { level: 9 })) + } + await replacePath(temporary, destination) + return { + id: resource.id, + kind: 'mirror', + path: relative(library, destination), + ...await describeTree(destination), + } +} + +function cacheIndexPath(cacheRoot, url) { + const key = `make-fetch-happen:request-cache:${url}` + const digest = createHash('sha256').update(key).digest('hex') + return resolve(cacheRoot, '_cacache/index-v5', digest.slice(0, 2), digest.slice(2, 4), digest.slice(4)) +} + +function cacheContentPath(cacheRoot, integrity) { + const separator = integrity.indexOf('-') + const algorithm = integrity.slice(0, separator) + const digest = Buffer.from(integrity.slice(separator + 1), 'base64').toString('hex') + return resolve(cacheRoot, '_cacache/content-v2', algorithm, digest.slice(0, 2), digest.slice(2, 4), digest.slice(4)) +} + +function platformMatches(constraints, value) { + if (!constraints?.length) return true + const includes = constraints.filter((entry) => !entry.startsWith('!')) + const excludes = constraints.filter((entry) => entry.startsWith('!')).map((entry) => entry.slice(1)) + return !excludes.includes(value) && (includes.length === 0 || includes.includes(value)) +} + +function packageMatchesHost(value) { + return platformMatches(value.os, 'linux') + && platformMatches(value.cpu, process.arch) + && platformMatches(value.libc, 'glibc') +} + +async function syncNpmCache() { + const sourceCache = resolve(process.env.npm_config_cache || process.env.NPM_CONFIG_CACHE || resolve(homedir(), '.npm')) + const destination = resolve(library, 'npm/cache') + const temporary = `${destination}.part-${process.pid}` + await rm(temporary, { recursive: true, force: true }) + const urls = new Set() + for (const lock of config.npmLocks) { + const path = resolve(root, lock) + if (!await exists(path)) fail(`npm lock is missing: ${lock}`) + const document = JSON.parse(await readFile(path, 'utf8')) + for (const value of Object.values(document.packages || {})) { + if (value?.resolved?.startsWith('http') && packageMatchesHost(value)) urls.add(value.resolved) + } + } + let copied = 0 + for (const url of [...urls].sort()) { + const indexSource = cacheIndexPath(sourceCache, url) + if (!await exists(indexSource)) fail(`npm cache index is missing ${url}`) + const lines = (await readFile(indexSource, 'utf8')).trim().split(/\r?\n/).filter(Boolean) + const line = lines.at(-1) + const entry = JSON.parse(line.slice(line.indexOf('\t') + 1)) + const contentSource = cacheContentPath(sourceCache, entry.integrity) + if (!await exists(contentSource)) fail(`npm cache content is missing ${url}`) + for (const source of [indexSource, contentSource]) { + const path = relative(sourceCache, source) + const target = resolve(temporary, path) + await mkdir(dirname(target), { recursive: true }) + await cp(source, target, { preserveTimestamps: true }) + } + copied += 1 + } + await replacePath(temporary, destination) + print(`[offline:sync] npm cache ${copied} locked tarballs`) + return { + id: 'npm-cache', + kind: 'npm-cache', + path: relative(library, destination), + packages: copied, + ...await describeTree(destination), + } +} + +async function syncNodeArchive() { + const source = resolve(root, config.node.archive) + if (!await exists(source)) fail(`Pinned Node archive is missing: ${source}`) + const destination = resolve(library, config.node.libraryPath) + await mkdir(dirname(destination), { recursive: true }) + await cp(source, destination, { preserveTimestamps: true }) + print(`[offline:sync] node ${config.node.version}`) + return { id: 'node-runtime', kind: 'file', path: relative(library, destination), ...await describeFile(destination) } +} + +async function hostToolVersions() { + const probes = { + node: [resolve(root, 'nodew'), ['--version']], + npm: [resolve(root, 'npmw'), ['--version']], + cmake: ['cmake', ['--version']], + ninja: ['ninja', ['--version']], + emscripten: ['emcc', ['--version']], + chrome: [process.env.CHROME_BIN || resolve(homedir(), '.local/bin/google-chrome'), ['--version']], + firefox: [process.env.FIREFOX_BIN || 'firefox', ['--version']], + } + const result = {} + for (const [name, [program, args]] of Object.entries(probes)) { + const probe = spawnSync(program, args, { encoding: 'utf8' }) + result[name] = probe.status === 0 ? `${probe.stdout}${probe.stderr}`.trim().split(/\r?\n/)[0] : null + } + return result +} + +async function browserExecutables() { + const browserRoot = resolve(library, 'browsers/ms-playwright') + const directories = (await readdir(browserRoot, { withFileTypes: true })) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() + .reverse() + const select = (prefix) => directories.find((name) => name.startsWith(`${prefix}-`)) + const chromiumDirectory = select('chromium') + const firefoxDirectory = select('firefox') + const webkitDirectory = select('webkit') + const paths = { + chromium: chromiumDirectory && resolve(browserRoot, chromiumDirectory, 'chrome-linux64/chrome'), + firefox: firefoxDirectory && resolve(browserRoot, firefoxDirectory, 'firefox/firefox'), + webkit: webkitDirectory && resolve(browserRoot, webkitDirectory, 'pw_run.sh'), + } + for (const [name, path] of Object.entries(paths)) { + if (!path || !await exists(path)) fail(`Offline ${name} executable is missing from ${browserRoot}`) + } + return Object.fromEntries(Object.entries(paths).map(([name, path]) => [name, relative(library, path)])) +} + +async function sync() { + if (platform() !== 'linux') fail(`This resource profile is Linux-specific; received ${platform()}`) + if (selectedIds.size > 0) fail('Partial sync is intentionally unsupported; synchronize one complete, internally consistent manifest') + await mkdir(library, { recursive: true }) + const entries = [await syncNodeArchive(), await syncNpmCache()] + for (const resource of config.archives) entries.push(await createArchive(resource)) + for (const resource of config.mirrors) entries.push(await syncMirror(resource)) + const manifest = { + schemaVersion: 1, + project: config.project, + generatedAt: new Date().toISOString(), + library, + configSha256: await sha256(configPath), + platform: config.platform, + hostTools: await hostToolVersions(), + browserExecutables: await browserExecutables(), + resources: entries.filter(Boolean), + } + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`) + print(`[offline:sync] wrote ${manifestPath}`) + print({ status: 'pass', library, resources: manifest.resources.length }) +} + +async function readManifest() { + if (!await exists(manifestPath)) fail(`Offline manifest is missing: ${manifestPath}. Run offline:sync first.`) + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) + if (manifest.schemaVersion !== 1 || manifest.project !== config.project) fail('Offline manifest identity is invalid') + return manifest +} + +async function check() { + const manifest = await readManifest() + if (manifest.configSha256 !== await sha256(configPath)) fail('Offline resource config changed after the library was synchronized') + const expectedIds = new Set([ + 'node-runtime', + 'npm-cache', + ...config.archives.filter((entry) => entry.required).map((entry) => entry.id), + ...config.mirrors.filter((entry) => entry.required).map((entry) => entry.id), + ]) + const records = new Map(manifest.resources.map((entry) => [entry.id, entry])) + for (const id of expectedIds) if (!records.has(id)) fail(`Offline manifest is missing required resource ${id}`) + for (const record of manifest.resources.filter(isSelected)) { + const path = resolve(library, record.path) + if (!await exists(path)) fail(`Offline resource is missing: ${record.id}`) + const actual = record.kind === 'archive' || record.kind === 'file' ? await describeFile(path) : await describeTree(path) + if (actual.bytes !== record.bytes || actual.sha256 !== record.sha256 || (record.files != null && actual.files !== record.files)) { + fail(`Offline resource checksum mismatch: ${record.id}`) + } + print(`[offline:check] ${record.id} ${record.sha256.slice(0, 12)}`) + } + print({ status: 'pass', library, resources: manifest.resources.length }) +} + +async function prepareNpmWorkCache(manifest) { + const npmRecord = manifest.resources.find((entry) => entry.id === 'npm-cache') + if (!npmRecord) fail('Offline manifest is missing npm-cache') + const destination = resolve(root, '.cache/offline-npm') + const marker = resolve(destination, '.resource-sha256') + const current = await readFile(marker, 'utf8').then((value) => value.trim(), () => '') + if (current === npmRecord.sha256) return destination + const temporary = `${destination}.part-${process.pid}` + await rm(temporary, { recursive: true, force: true }) + await cp(resolve(library, npmRecord.path), temporary, { recursive: true, preserveTimestamps: true }) + await writeFile(resolve(temporary, '.resource-sha256'), `${npmRecord.sha256}\n`) + await replacePath(temporary, destination) + return destination +} + +async function assertRestoreTarget(targetRoot, destination) { + const relativePath = relative(targetRoot, destination) + if (!relativePath || relativePath === '..' || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) { + fail(`Unsafe restore target: ${destination}`) + } + if (await exists(destination) && !flags.has('force')) fail(`Restore target already exists: ${destination}. Use --force to replace it.`) +} + +async function extractArchive(resource, record, targetRoot) { + const destination = resourceRestorePath(resource, targetRoot) + await assertRestoreTarget(targetRoot, destination) + const temporary = `${destination}.offline-part-${process.pid}` + await rm(temporary, { recursive: true, force: true }) + await mkdir(temporary, { recursive: true }) + print(`[offline:restore] ${resource.id} -> ${destination}`) + await run('tar', [ + '--extract', + `--file=${resolve(library, record.path)}`, + '--use-compress-program', + 'zstd -d', + '--directory', + temporary, + ]) + await replacePath(temporary, destination) +} + +async function restoreMirror(resource, record, targetRoot) { + if (!resource.restore) return + const destination = resourceRestorePath(resource, targetRoot) + await assertRestoreTarget(targetRoot, destination) + const temporary = `${destination}.offline-part-${process.pid}` + await rm(temporary, { recursive: true, force: true }) + await mkdir(dirname(temporary), { recursive: true }) + print(`[offline:restore] ${resource.id} -> ${destination}`) + await cp(resolve(library, record.path), temporary, { recursive: true, preserveTimestamps: true }) + await replacePath(temporary, destination) +} + +async function restore() { + const manifest = await readManifest() + const targetRoot = resolve(values.get('target') || root) + await mkdir(targetRoot, { recursive: true }) + const records = new Map(manifest.resources.map((entry) => [entry.id, entry])) + const resources = [...config.archives, ...config.mirrors.filter((entry) => entry.restore)].filter(isSelected) + if (selectedIds.size === 0) fail('Restore requires --only=ID,... so replacement scope is explicit') + for (const resource of resources) { + const record = records.get(resource.id) + if (!record) { + if (resource.required) fail(`Offline manifest is missing ${resource.id}`) + continue + } + if (record.kind === 'archive') await extractArchive(resource, record, targetRoot) + else await restoreMirror(resource, record, targetRoot) + } + if (selectedIds.has('node-runtime')) { + const record = records.get('node-runtime') + const destination = resolve(targetRoot, config.node.archive) + await assertRestoreTarget(targetRoot, destination) + await mkdir(dirname(destination), { recursive: true }) + await cp(resolve(library, record.path), destination, { preserveTimestamps: true }) + } + print({ status: 'pass', library, target: targetRoot, restored: [...selectedIds] }) +} + +async function npmOfflineSmoke() { + const directory = await mkdtemp(resolve(tmpdir(), 'web-freecad-offline-npm-')) + try { + for (const file of ['package.json', 'package-lock.json', '.npmrc']) await cp(resolve(root, file), resolve(directory, file)) + const npmCache = resolve(directory, '.npm-cache') + await cp(resolve(library, 'npm/cache'), npmCache, { recursive: true, preserveTimestamps: true }) + const runtimeArchive = resolve(library, config.node.libraryPath) + await run('tar', ['--extract', `--file=${runtimeArchive}`, '--directory', directory]) + const runtime = resolve(directory, `node-v${config.node.version}-linux-x64`) + print('[offline:smoke] npm ci --offline in an isolated directory') + await run(resolve(runtime, 'bin/npm'), [ + 'ci', + '--offline', + '--ignore-scripts', + `--cache=${npmCache}`, + ], { + cwd: directory, + env: { + ...process.env, + PATH: `${resolve(runtime, 'bin')}:${process.env.PATH}`, + npm_config_offline: 'true', + npm_config_audit: 'false', + npm_config_fund: 'false', + }, + }) + const installed = JSON.parse(await readFile(resolve(directory, 'node_modules/@bitbybit-dev/occt/package.json'), 'utf8')) + if (installed.version !== '1.1.1') fail(`Offline npm smoke installed unexpected OCCT ${installed.version}`) + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +async function linuxCncNpmOfflineSmoke() { + const directory = await mkdtemp(resolve(tmpdir(), 'web-freecad-offline-linuxcnc-npm-')) + const source = resolve(root, 'cnc_wams_gpt6/linuxcnc-master/web') + try { + for (const file of ['package.json', 'package-lock.json']) await cp(resolve(source, file), resolve(directory, file)) + const npmCache = resolve(directory, '.npm-cache') + await cp(resolve(library, 'npm/cache'), npmCache, { recursive: true, preserveTimestamps: true }) + const runtimeDirectory = await mkdtemp(resolve(tmpdir(), 'web-freecad-offline-node-')) + try { + await run('tar', ['--extract', `--file=${resolve(library, config.node.libraryPath)}`, '--directory', runtimeDirectory]) + const runtime = resolve(runtimeDirectory, `node-v${config.node.version}-linux-x64`) + print('[offline:smoke] LinuxCNC Web npm ci --offline in an isolated directory') + await run(resolve(runtime, 'bin/npm'), [ + 'ci', + '--offline', + '--ignore-scripts', + `--cache=${npmCache}`, + ], { + cwd: directory, + env: { + ...process.env, + PATH: `${resolve(runtime, 'bin')}:${process.env.PATH}`, + npm_config_offline: 'true', + npm_config_audit: 'false', + npm_config_fund: 'false', + }, + }) + const installed = JSON.parse(await readFile(resolve(directory, 'node_modules/playwright/package.json'), 'utf8')) + if (installed.version !== '1.61.1') fail(`Offline LinuxCNC npm smoke installed unexpected Playwright ${installed.version}`) + } finally { + await rm(runtimeDirectory, { recursive: true, force: true }) + } + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +async function playwrightOfflineSmoke() { + const playwrightPath = resolve(root, 'cnc_wams_gpt6/linuxcnc-master/web/node_modules/playwright/index.mjs') + const executables = await browserExecutables() + const absoluteExecutables = Object.fromEntries(Object.entries(executables).map(([name, path]) => [name, resolve(library, path)])) + const script = [ + `import { chromium, firefox, webkit } from ${JSON.stringify(new URL(`file://${playwrightPath}`).href)}`, + `for (const [name, type] of Object.entries({ chromium, firefox, webkit })) {`, + ` const executablePath = ${JSON.stringify(absoluteExecutables)}[name]`, + ` const browser = await type.launch({ headless: true, executablePath })`, + ` const page = await browser.newPage()`, + ` await page.setContent('offline')`, + ` if (await page.title() !== 'offline') throw new Error(name + ' page smoke failed')`, + ` console.log('[offline:smoke] browser ' + name + ' pass')`, + ` await browser.close()`, + `}`, + ].join('\n') + await run(resolve(root, 'nodew'), ['--input-type=module', '--eval', script], { + env: { ...process.env, PLAYWRIGHT_BROWSERS_PATH: resolve(library, 'browsers/ms-playwright') }, + }) +} + +async function smoke() { + await check() + await npmOfflineSmoke() + await linuxCncNpmOfflineSmoke() + await playwrightOfflineSmoke() + const environment = { + ...process.env, + WEB_FREECAD_RESOURCE_LIBRARY: library, + WEB_FREECAD_OFFLINE: '1', + npm_config_cache: await prepareNpmWorkCache(await readManifest()), + npm_config_offline: 'true', + PLAYWRIGHT_BROWSERS_PATH: resolve(library, 'browsers/ms-playwright'), + FREECAD_SOURCE_OFFLINE: '1', + OCCT_SOURCE_DIR: resolve(root, '.cache/occt/occt'), + } + for (const script of ['check:runtime', 'check:freecad-source', 'check:occt-history-artifact', 'test:occt-history', 'test:planegcs', 'build']) { + print(`[offline:smoke] npm run ${script}`) + await run(resolve(root, 'npmw'), ['run', script], { env: environment }) + } + print({ status: 'pass', library, networkFallback: false }) +} + +function shellQuote(value) { + return `'${String(value).replaceAll("'", "'\\''")}'` +} + +async function environment() { + const manifest = await readManifest() + const npmCache = await prepareNpmWorkCache(manifest) + const firefox = resolve(library, manifest.browserExecutables.firefox) + const webkit = resolve(library, manifest.browserExecutables.webkit) + const lines = [ + `export WEB_FREECAD_RESOURCE_LIBRARY=${shellQuote(library)}`, + 'export WEB_FREECAD_OFFLINE=1', + `export npm_config_cache=${shellQuote(npmCache)}`, + 'export npm_config_offline=true', + `export PLAYWRIGHT_BROWSERS_PATH=${shellQuote(resolve(library, 'browsers/ms-playwright'))}`, + 'export FREECAD_SOURCE_OFFLINE=1', + `export OCCT_SOURCE_DIR=${shellQuote(resolve(root, '.cache/occt/occt'))}`, + `export CHROME_BIN=${shellQuote(process.env.CHROME_BIN || resolve(homedir(), '.local/bin/google-chrome'))}`, + `export FIREFOX_BIN=${shellQuote(firefox)}`, + `export WEBKIT_BIN=${shellQuote(webkit)}`, + 'export CI_REAL_OFFLINE=1', + ] + print(lines.join('\n')) +} + +async function inventory() { + const resources = [] + for (const resource of [...config.archives, ...config.mirrors]) { + const source = await resourceSource(resource) + resources.push({ id: resource.id, source, present: Boolean(source), required: resource.required }) + } + print({ library, config: relative(root, configPath), resources, hostTools: await hostToolVersions() }) +} + +if (command === 'sync') await sync() +else if (command === 'check') await check() +else if (command === 'restore') await restore() +else if (command === 'smoke') await smoke() +else if (command === 'env') await environment() +else if (command === 'inventory') await inventory() +else { + print('Usage: node scripts/offline-resource-lib.mjs [--library=PATH] [--only=ID,...] [--target=PATH] [--force]') + if (command !== 'help') process.exitCode = 2 +} diff --git a/scripts/offline-shell.sh b/scripts/offline-shell.sh new file mode 100755 index 0000000..417d629 --- /dev/null +++ b/scripts/offline-shell.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_dir="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)" +eval "$("$repo_dir/nodew" "$repo_dir/scripts/offline-resource-lib.mjs" env)" + +if [[ $# -eq 0 ]]; then + exec "${SHELL:-/bin/bash}" --noprofile --norc +fi +exec "$@" diff --git a/scripts/run-real-verification.mjs b/scripts/run-real-verification.mjs index f29aa74..b04ef58 100644 --- a/scripts/run-real-verification.mjs +++ b/scripts/run-real-verification.mjs @@ -120,7 +120,12 @@ function runScript(name, laneEnvironment = {}) { console.log(`\n[real-verification] ${name}`) const result = spawnSync(npmw, ['run', name], { cwd: root, - env: { ...process.env, CI_REAL_VERIFICATION: '1', ...laneEnvironment }, + env: { + ...process.env, + CI_REAL_VERIFICATION: '1', + ...(process.env.CI_REAL_OFFLINE === '1' ? { npm_config_offline: 'true', FREECAD_SOURCE_OFFLINE: '1' } : {}), + ...laneEnvironment, + }, stdio: 'inherit', }) const record = { script: name, status: result.status ?? 1, durationMs: Date.now() - startedAt } diff --git a/scripts/verify-freecad-source.mjs b/scripts/verify-freecad-source.mjs index 31d2a11..b4ec56c 100644 --- a/scripts/verify-freecad-source.mjs +++ b/scripts/verify-freecad-source.mjs @@ -1,8 +1,29 @@ import { execFileSync } from 'node:child_process' import { readFile } from 'node:fs/promises' +import { resolve } from 'node:path' const toolchain = JSON.parse(await readFile(new URL('../config/freecad-toolchain.json', import.meta.url), 'utf8')) const reference = `refs/tags/${toolchain.source.tag}` +const root = resolve(new URL('..', import.meta.url).pathname) +const source = resolve(root, toolchain.sourceCheckout.defaultDirectory) +const git = (...args) => execFileSync('git', args, { encoding: 'utf8', timeout: 30000 }).trim() + +if (process.env.FREECAD_SOURCE_OFFLINE === '1') { + const revision = git('-C', source, 'rev-parse', 'HEAD') + if (revision !== toolchain.source.commit) { + console.error(`FreeCAD source mismatch: expected ${toolchain.source.commit}, received ${revision}.`) + process.exit(1) + } + const submodules = git('-C', source, 'submodule', 'status', '--recursive').split(/\r?\n/).filter(Boolean) + const incomplete = submodules.filter((line) => line.startsWith('-') || line.startsWith('+') || line.startsWith('U')) + if (submodules.length !== 4 || incomplete.length > 0) { + console.error(`FreeCAD offline source requires four clean initialized submodules; received ${JSON.stringify(submodules)}.`) + process.exit(1) + } + console.log(`FreeCAD offline source verified: ${source} @ ${revision}, submodules=${submodules.length}.`) + process.exit(0) +} + const output = execFileSync('git', ['ls-remote', toolchain.source.repository, reference], { encoding: 'utf8', timeout: 30000 }).trim() const [resolved, resolvedReference] = output.split(/\s+/) if (resolvedReference !== reference || resolved !== toolchain.source.commit) { diff --git a/tests/offlineResourceLibrary.test.mjs b/tests/offlineResourceLibrary.test.mjs new file mode 100644 index 0000000..18ebe9b --- /dev/null +++ b/tests/offlineResourceLibrary.test.mjs @@ -0,0 +1,45 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import { resolve } from 'node:path' +import test from 'node:test' + +const root = resolve(import.meta.dirname, '..') +const config = JSON.parse(await readFile(resolve(root, 'config/offline-resources.json'), 'utf8')) + +test('offline resource identifiers and destinations are unique and project-relative', () => { + assert.equal(config.schemaVersion, 1) + const resources = [...config.archives, ...config.mirrors] + assert.equal(new Set(resources.map((entry) => entry.id)).size, resources.length) + assert.equal(new Set(resources.map((entry) => entry.libraryPath)).size, resources.length) + for (const resource of resources) { + assert.ok(!resource.libraryPath.startsWith('/'), resource.id) + assert.ok(!resource.libraryPath.split('/').includes('..'), resource.id) + if (resource.restore) { + assert.ok(!resource.restore.startsWith('/'), resource.id) + assert.ok(!resource.restore.split('/').includes('..'), resource.id) + } + } +}) + +test('all pinned upstream sources declare full revisions', () => { + const pinned = config.archives.filter((entry) => entry.revision) + assert.ok(pinned.length >= 7) + for (const resource of pinned) assert.match(resource.revision, /^[0-9a-f]{40}$/, resource.id) +}) + +test('offline profile covers runtime, native sources, browsers and host recovery', () => { + const ids = new Set([...config.archives, ...config.mirrors].map((entry) => entry.id)) + for (const id of [ + 'project-node-modules', + 'freecad-source', + 'bitbybit-source', + 'occt-source', + 'opencamlib-source', + 'camotics-worktree', + 'linuxcnc-worktree', + 'occt-history-dist', + 'planegcs-dist', + 'playwright-browsers', + 'debian-package-cache', + ]) assert.ok(ids.has(id), id) +})