chore: checkpoint web Blender migration source, tools and docs
Accumulated uncommitted work: GHOST Web backend, WebGL/GPU web layer, web workflow engine and protocol, tooling, docs and task records. Large generated evidence, user-testing archives, build snapshots and imported test assets remain untracked. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
89
tools/web/build-webengine-numpy.py
Normal file
89
tools/web/build-webengine-numpy.py
Normal file
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build upstream NumPy for the existing static CPython/WASM runtime."""
|
||||
import hashlib, json, os, pathlib, subprocess, tarfile, urllib.request, zipfile
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
BASE = ROOT / 'build_web_numpy'
|
||||
SOURCE = BASE / 'numpy-2.3.4'
|
||||
BUILD = BASE / 'build'
|
||||
PYTHON = ROOT / '.deps/native/python/bin/python3.13'
|
||||
BASE.mkdir(exist_ok=True)
|
||||
archive = BASE / 'numpy-2.3.4.tar.gz'
|
||||
if not archive.exists():
|
||||
urllib.request.urlretrieve('https://github.com/numpy/numpy/releases/download/v2.3.4/numpy-2.3.4.tar.gz', archive)
|
||||
if hashlib.sha256(archive.read_bytes()).hexdigest() != 'a7d018bfedb375a8d979ac758b120ba846a7fe764911a64465fd87b8729f4a6a':
|
||||
raise RuntimeError('Pinned NumPy source hash mismatch')
|
||||
if not SOURCE.exists():
|
||||
with tarfile.open(archive) as package: package.extractall(BASE, filter='data')
|
||||
import re
|
||||
# Shared modules isolate legacy random's 32-bit ABI; static modules need namespacing.
|
||||
random_meson = SOURCE / 'numpy/random/meson.build'
|
||||
symbols = sorted(set(re.findall(r'\b(random_[A-Za-z0-9_]+)\s*\(', (SOURCE / 'numpy/_core/include/numpy/random/distributions.h').read_text())))
|
||||
symbols.append('random_geometric_inversion') # Exported helper absent from distributions.h.
|
||||
legacy_flags = "['-DNP_RANDOM_LEGACY=1'" + ''.join(f", '-D{name}=web_numpy_legacy_{name}'" for name in symbols) + ']'
|
||||
meson_source = re.sub(r"\['-DNP_RANDOM_LEGACY=1'(?:, '-Drandom_\w+=web_numpy_legacy_random_\w+')*\]", "['-DNP_RANDOM_LEGACY=1']", random_meson.read_text())
|
||||
if "['-DNP_RANDOM_LEGACY=1']" in meson_source: random_meson.write_text(meson_source.replace("['-DNP_RANDOM_LEGACY=1']", legacy_flags))
|
||||
elif legacy_flags not in meson_source: raise RuntimeError('Unexpected NumPy legacy random build declaration')
|
||||
env = dict(os.environ)
|
||||
env['PATH'] = str(PYTHON.parent) + ':' + env['PATH']
|
||||
env['LD_LIBRARY_PATH'] = str(ROOT / '.deps/native/python/lib') + ':' + env.get('LD_LIBRARY_PATH', '')
|
||||
pkgconfig = BASE / 'pkgconfig'
|
||||
pkgconfig.mkdir(exist_ok=True)
|
||||
env['PKG_CONFIG_LIBDIR'] = str(pkgconfig)
|
||||
env['PKG_CONFIG_PATH'] = ''
|
||||
(pkgconfig / 'python-3.13.pc').write_text(f'Name: Python\nDescription: WebEngine CPython\nVersion: 3.13\nLibs:\nCflags: -I{ROOT}/build_web_python313/build -I{ROOT}/build_web_python313/source/Include\n')
|
||||
cross = BASE / 'emscripten.cross'
|
||||
original = (SOURCE / 'tools/ci/emscripten/emscripten.meson.cross').read_text()
|
||||
original = original.replace('[properties]\n', f"[properties]\npkg_config_libdir = '{pkgconfig}'\n")
|
||||
binaries = f"c = 'emcc'\ncpp = 'em++'\nar = 'emar'\npython = '{PYTHON}'\ncython = '{PYTHON.parent}/cython'\n"
|
||||
cross.write_text(original.replace('[binaries]\n', '[binaries]\n' + binaries) + "\n[built-in options]\nc_args = ['-fexceptions', '-DPY_CALL_TRAMPOLINE=1']\ncpp_args = ['-fexceptions', '-DPY_CALL_TRAMPOLINE=1']\n")
|
||||
def run(name, command, **kwargs):
|
||||
with (BASE / (name + '.log')).open('w') as log:
|
||||
subprocess.run(command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, check=True, **kwargs)
|
||||
meson = [str(PYTHON), str(SOURCE / 'vendored-meson/meson/meson.py')]
|
||||
if not (BUILD / 'build.ninja').exists():
|
||||
run('configure', meson + ['setup', str(BUILD), str(SOURCE), '--cross-file=' + str(cross),
|
||||
'--buildtype=release', '-Dblas=none', '-Dlapack=none', '-Dcpu-baseline=none', '-Dcpu-dispatch=none'])
|
||||
else:
|
||||
run('reconfigure', meson + ['setup', '--reconfigure', str(BUILD), str(SOURCE)])
|
||||
commands = json.loads((BUILD / 'compile_commands.json').read_text())
|
||||
python_commands = [entry['command'] for entry in commands if '.so.p/' in entry['output'] and entry['output'].endswith('.o')]
|
||||
if not python_commands or any(f'-I{ROOT}/build_web_python313/build ' not in command for command in python_commands):
|
||||
raise RuntimeError('NumPy extension compilation must use target CPython pyconfig.h')
|
||||
plan = json.loads((BUILD / 'meson-info/intro-install_plan.json').read_text())
|
||||
modules = [pathlib.Path(p).relative_to(BUILD) for p, v in plan['targets'].items() if v['tag'] == 'runtime']
|
||||
inputs = set()
|
||||
for module in modules:
|
||||
query = subprocess.check_output(['ninja', '-C', str(BUILD), '-t', 'query', str(module)], env=env, text=True)
|
||||
inputs.update(line.strip().lstrip('| ') for line in query.split(' outputs:')[0].splitlines() if line.strip().endswith(('.o', '.a')))
|
||||
if len(modules) != 13 or not inputs:
|
||||
raise RuntimeError('Unexpected NumPy runtime module graph; inspect upstream install plan')
|
||||
(BUILD / 'webengine-static.ninja').write_text('include build.ninja\nbuild webengine_numpy_objects: phony ' + ' '.join(sorted(inputs)) + '\n')
|
||||
run('objects', ['ninja', '-C', str(BUILD), '-f', 'webengine-static.ninja', '-j4', 'webengine_numpy_objects'])
|
||||
library = BASE / 'libnumpy.a'
|
||||
mri = 'CREATE ' + str(library) + '\n'
|
||||
mri += ''.join(('ADDLIB ' if name.endswith('.a') else 'ADDMOD ') + str(BUILD / name) + '\n' for name in sorted(inputs))
|
||||
run('archive', ['emar', '-M'], input=mri + 'SAVE\nEND\n', text=True)
|
||||
files = {}
|
||||
for kind, entries in plan.items():
|
||||
if kind == 'targets': continue
|
||||
for name, entry in entries.items():
|
||||
if entry['tag'] != 'python-runtime': continue
|
||||
source = pathlib.Path(name)
|
||||
destination = entry['destination'].split('}/', 1)[1]
|
||||
if source.is_file(): files[destination] = source
|
||||
else:
|
||||
for file in sorted(source.rglob('*')):
|
||||
relative = file.relative_to(source)
|
||||
if not file.is_file() or any(part in entry.get('exclude_dirs', []) + ['__pycache__'] for part in relative.parts): continue
|
||||
if relative.as_posix() not in entry.get('exclude_files', []): files[destination + '/' + relative.as_posix()] = file
|
||||
with zipfile.ZipFile(BASE / 'numpy.zip', 'w', compression=zipfile.ZIP_STORED) as package:
|
||||
for name, file in sorted(files.items()): package.write(file, name)
|
||||
package.write(SOURCE / 'LICENSE.txt', 'numpy/LICENSE.txt')
|
||||
registrations = [('.'.join(module.parts[:-1] + (module.name.split('.')[0],)), module.name.split('.')[0]) for module in modules]
|
||||
header = '#pragma once\n#include <Python.h>\n'
|
||||
header += ''.join(f'extern "C" PyObject *PyInit_{symbol}();\n' for name, symbol in registrations)
|
||||
header += 'inline int web_engine_register_numpy() {\n' + ''.join(f' if (PyImport_AppendInittab("{name}", PyInit_{symbol}) < 0) return -1;\n' for name, symbol in registrations) + ' return 0;\n}\n'
|
||||
(BASE / 'webengine_numpy.hh').write_text(header)
|
||||
bound = [archive, pathlib.Path(__file__), cross, random_meson, ROOT / 'build_web_python313/build/pyconfig.h', ROOT / 'build_web_python313/build/libpython3.13.a', library, BASE / 'numpy.zip', BASE / 'webengine_numpy.hh']
|
||||
report = dict(status='BUILT_UNVERIFIED', target='wasm32-emscripten-cpython313-static', modules=[name for name, _ in registrations], artifacts={str(file.relative_to(ROOT)): hashlib.sha256(file.read_bytes()).hexdigest() for file in bound})
|
||||
(BASE / 'build.json').write_text(json.dumps(report, indent=2) + '\n')
|
||||
Reference in New Issue
Block a user