f3e21c6b98
Accumulated uncommitted work: GHOST Web backend, WebGL/GPU web layer, web workflow engine and protocol, tooling, docs and task records. Large generated evidence, user-testing archives, build snapshots and imported test assets remain untracked. Co-Authored-By: Claude Code <noreply@anthropic.com>
90 lines
6.8 KiB
Python
90 lines
6.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Build upstream NumPy for the existing static CPython/WASM runtime."""
|
|
import hashlib, json, os, pathlib, subprocess, tarfile, urllib.request, zipfile
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
BASE = ROOT / 'build_web_numpy'
|
|
SOURCE = BASE / 'numpy-2.3.4'
|
|
BUILD = BASE / 'build'
|
|
PYTHON = ROOT / '.deps/native/python/bin/python3.13'
|
|
BASE.mkdir(exist_ok=True)
|
|
archive = BASE / 'numpy-2.3.4.tar.gz'
|
|
if not archive.exists():
|
|
urllib.request.urlretrieve('https://github.com/numpy/numpy/releases/download/v2.3.4/numpy-2.3.4.tar.gz', archive)
|
|
if hashlib.sha256(archive.read_bytes()).hexdigest() != 'a7d018bfedb375a8d979ac758b120ba846a7fe764911a64465fd87b8729f4a6a':
|
|
raise RuntimeError('Pinned NumPy source hash mismatch')
|
|
if not SOURCE.exists():
|
|
with tarfile.open(archive) as package: package.extractall(BASE, filter='data')
|
|
import re
|
|
# Shared modules isolate legacy random's 32-bit ABI; static modules need namespacing.
|
|
random_meson = SOURCE / 'numpy/random/meson.build'
|
|
symbols = sorted(set(re.findall(r'\b(random_[A-Za-z0-9_]+)\s*\(', (SOURCE / 'numpy/_core/include/numpy/random/distributions.h').read_text())))
|
|
symbols.append('random_geometric_inversion') # Exported helper absent from distributions.h.
|
|
legacy_flags = "['-DNP_RANDOM_LEGACY=1'" + ''.join(f", '-D{name}=web_numpy_legacy_{name}'" for name in symbols) + ']'
|
|
meson_source = re.sub(r"\['-DNP_RANDOM_LEGACY=1'(?:, '-Drandom_\w+=web_numpy_legacy_random_\w+')*\]", "['-DNP_RANDOM_LEGACY=1']", random_meson.read_text())
|
|
if "['-DNP_RANDOM_LEGACY=1']" in meson_source: random_meson.write_text(meson_source.replace("['-DNP_RANDOM_LEGACY=1']", legacy_flags))
|
|
elif legacy_flags not in meson_source: raise RuntimeError('Unexpected NumPy legacy random build declaration')
|
|
env = dict(os.environ)
|
|
env['PATH'] = str(PYTHON.parent) + ':' + env['PATH']
|
|
env['LD_LIBRARY_PATH'] = str(ROOT / '.deps/native/python/lib') + ':' + env.get('LD_LIBRARY_PATH', '')
|
|
pkgconfig = BASE / 'pkgconfig'
|
|
pkgconfig.mkdir(exist_ok=True)
|
|
env['PKG_CONFIG_LIBDIR'] = str(pkgconfig)
|
|
env['PKG_CONFIG_PATH'] = ''
|
|
(pkgconfig / 'python-3.13.pc').write_text(f'Name: Python\nDescription: WebEngine CPython\nVersion: 3.13\nLibs:\nCflags: -I{ROOT}/build_web_python313/build -I{ROOT}/build_web_python313/source/Include\n')
|
|
cross = BASE / 'emscripten.cross'
|
|
original = (SOURCE / 'tools/ci/emscripten/emscripten.meson.cross').read_text()
|
|
original = original.replace('[properties]\n', f"[properties]\npkg_config_libdir = '{pkgconfig}'\n")
|
|
binaries = f"c = 'emcc'\ncpp = 'em++'\nar = 'emar'\npython = '{PYTHON}'\ncython = '{PYTHON.parent}/cython'\n"
|
|
cross.write_text(original.replace('[binaries]\n', '[binaries]\n' + binaries) + "\n[built-in options]\nc_args = ['-fexceptions', '-DPY_CALL_TRAMPOLINE=1']\ncpp_args = ['-fexceptions', '-DPY_CALL_TRAMPOLINE=1']\n")
|
|
def run(name, command, **kwargs):
|
|
with (BASE / (name + '.log')).open('w') as log:
|
|
subprocess.run(command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, check=True, **kwargs)
|
|
meson = [str(PYTHON), str(SOURCE / 'vendored-meson/meson/meson.py')]
|
|
if not (BUILD / 'build.ninja').exists():
|
|
run('configure', meson + ['setup', str(BUILD), str(SOURCE), '--cross-file=' + str(cross),
|
|
'--buildtype=release', '-Dblas=none', '-Dlapack=none', '-Dcpu-baseline=none', '-Dcpu-dispatch=none'])
|
|
else:
|
|
run('reconfigure', meson + ['setup', '--reconfigure', str(BUILD), str(SOURCE)])
|
|
commands = json.loads((BUILD / 'compile_commands.json').read_text())
|
|
python_commands = [entry['command'] for entry in commands if '.so.p/' in entry['output'] and entry['output'].endswith('.o')]
|
|
if not python_commands or any(f'-I{ROOT}/build_web_python313/build ' not in command for command in python_commands):
|
|
raise RuntimeError('NumPy extension compilation must use target CPython pyconfig.h')
|
|
plan = json.loads((BUILD / 'meson-info/intro-install_plan.json').read_text())
|
|
modules = [pathlib.Path(p).relative_to(BUILD) for p, v in plan['targets'].items() if v['tag'] == 'runtime']
|
|
inputs = set()
|
|
for module in modules:
|
|
query = subprocess.check_output(['ninja', '-C', str(BUILD), '-t', 'query', str(module)], env=env, text=True)
|
|
inputs.update(line.strip().lstrip('| ') for line in query.split(' outputs:')[0].splitlines() if line.strip().endswith(('.o', '.a')))
|
|
if len(modules) != 13 or not inputs:
|
|
raise RuntimeError('Unexpected NumPy runtime module graph; inspect upstream install plan')
|
|
(BUILD / 'webengine-static.ninja').write_text('include build.ninja\nbuild webengine_numpy_objects: phony ' + ' '.join(sorted(inputs)) + '\n')
|
|
run('objects', ['ninja', '-C', str(BUILD), '-f', 'webengine-static.ninja', '-j4', 'webengine_numpy_objects'])
|
|
library = BASE / 'libnumpy.a'
|
|
mri = 'CREATE ' + str(library) + '\n'
|
|
mri += ''.join(('ADDLIB ' if name.endswith('.a') else 'ADDMOD ') + str(BUILD / name) + '\n' for name in sorted(inputs))
|
|
run('archive', ['emar', '-M'], input=mri + 'SAVE\nEND\n', text=True)
|
|
files = {}
|
|
for kind, entries in plan.items():
|
|
if kind == 'targets': continue
|
|
for name, entry in entries.items():
|
|
if entry['tag'] != 'python-runtime': continue
|
|
source = pathlib.Path(name)
|
|
destination = entry['destination'].split('}/', 1)[1]
|
|
if source.is_file(): files[destination] = source
|
|
else:
|
|
for file in sorted(source.rglob('*')):
|
|
relative = file.relative_to(source)
|
|
if not file.is_file() or any(part in entry.get('exclude_dirs', []) + ['__pycache__'] for part in relative.parts): continue
|
|
if relative.as_posix() not in entry.get('exclude_files', []): files[destination + '/' + relative.as_posix()] = file
|
|
with zipfile.ZipFile(BASE / 'numpy.zip', 'w', compression=zipfile.ZIP_STORED) as package:
|
|
for name, file in sorted(files.items()): package.write(file, name)
|
|
package.write(SOURCE / 'LICENSE.txt', 'numpy/LICENSE.txt')
|
|
registrations = [('.'.join(module.parts[:-1] + (module.name.split('.')[0],)), module.name.split('.')[0]) for module in modules]
|
|
header = '#pragma once\n#include <Python.h>\n'
|
|
header += ''.join(f'extern "C" PyObject *PyInit_{symbol}();\n' for name, symbol in registrations)
|
|
header += 'inline int web_engine_register_numpy() {\n' + ''.join(f' if (PyImport_AppendInittab("{name}", PyInit_{symbol}) < 0) return -1;\n' for name, symbol in registrations) + ' return 0;\n}\n'
|
|
(BASE / 'webengine_numpy.hh').write_text(header)
|
|
bound = [archive, pathlib.Path(__file__), cross, random_meson, ROOT / 'build_web_python313/build/pyconfig.h', ROOT / 'build_web_python313/build/libpython3.13.a', library, BASE / 'numpy.zip', BASE / 'webengine_numpy.hh']
|
|
report = dict(status='BUILT_UNVERIFIED', target='wasm32-emscripten-cpython313-static', modules=[name for name, _ in registrations], artifacts={str(file.relative_to(ROOT)): hashlib.sha256(file.read_bytes()).hexdigest() for file in bound})
|
|
(BASE / 'build.json').write_text(json.dumps(report, indent=2) + '\n')
|