FreeCADWasmDev/tools/flush_deferred_module_builds.py

#!/usr/bin/env python3
"""Flush source-ready feature builds without reopening a Codex turn.

Deferred feature batches are intentionally coalesced while a module family is
being edited.  Once the planner has no work left, this small native-only step
closes that transaction: it builds each queued target, syncs artifacts once,
and records the original batch receipts.  Failed targets stay queued with a
bounded retry time so an idle scheduler cannot spin on the same linker error.
"""
from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path

from implementation_first import record_implementation_batch


ROOT = Path(__file__).resolve().parents[1]
PENDING_PATH = ROOT / "docs/current/module-build-pending.json"
OUTPUT_PATH = ROOT / "docs/current/module-batch-pipeline.json"
JOBS = max(1, int(os.environ.get("FREECAD_BUILD_JOBS", "8")))
RETRY_SECONDS = max(30, int(os.environ.get("FREECAD_DEFERRED_BUILD_RETRY_SECONDS", "300")))


def load_state() -> dict[str, object]:
    try:
        value = json.loads(PENDING_PATH.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        value = {}
    if not isinstance(value, dict) or value.get("schema") != "freecad-web-module-build-pending/v1":
        return {"schema": "freecad-web-module-build-pending/v1", "targets": {}}
    value.setdefault("targets", {})
    return value


def save_state(state: dict[str, object]) -> None:
    state["schema"] = "freecad-web-module-build-pending/v1"
    state["updatedAt"] = datetime.now(timezone.utc).isoformat()
    PENDING_PATH.parent.mkdir(parents=True, exist_ok=True)
    temporary = PENDING_PATH.with_suffix(".tmp")
    temporary.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    temporary.replace(PENDING_PATH)


def run(command: list[str], timeout: int) -> dict[str, object]:
    started = time.monotonic()
    environment = {key: str(value) for key, value in os.environ.items()}
    environment.update({
        "EM_CONFIG": str(ROOT / "tools/emscripten-writable-config.py"),
        "EMCC_CORES": str(JOBS),
        "CMAKE_BUILD_PARALLEL_LEVEL": str(JOBS),
        "FREECAD_WEB_FAST_LINK": "ON",
    })
    try:
        result = subprocess.run(command, cwd=ROOT, env=environment, text=True, capture_output=True, timeout=timeout, check=False)
        return {
            "command": command,
            "seconds": round(time.monotonic() - started, 3),
            "exitCode": result.returncode,
            "stdout": (result.stdout or "")[-3000:],
            "stderr": (result.stderr or "")[-3000:],
        }
    except subprocess.TimeoutExpired as error:
        return {
            "command": command,
            "seconds": round(time.monotonic() - started, 3),
            "exitCode": 124,
            "stdout": str(error.stdout or "")[-1000:],
            "stderr": str(error.stderr or "timeout")[-3000:],
        }


def save_report(payload: dict[str, object]) -> None:
    OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
    temporary = OUTPUT_PATH.with_suffix(".tmp")
    temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    temporary.replace(OUTPUT_PATH)


def main(argv: list[str] | None = None) -> int:
    global OUTPUT_PATH
    parser = argparse.ArgumentParser()
    parser.add_argument("--json", action="store_true", help="emit the compact JSON summary (default output)")
    parser.add_argument("--force", action="store_true", help="ignore retry backoff after a source-side fix")
    parser.add_argument("--exclude-target", action="append", default=[], help="keep this tree:target queued for the next coalesced batch")
    parser.add_argument("--report-path", type=Path, help="write a separate flush report without replacing current batch diagnostics")
    arguments = parser.parse_args(argv)
    if arguments.report_path:
        OUTPUT_PATH = ROOT / arguments.report_path
    state = load_state()
    targets = state.get("targets") if isinstance(state.get("targets"), dict) else {}
    if not targets:
        payload = {"status": "no_pending_builds", "targets": 0, "totalSeconds": 0.0}
        save_report(payload)
        print(json.dumps(payload, ensure_ascii=False))
        return 0
    now = time.time()
    results: list[dict[str, object]] = []
    flushed = 0
    skipped = 0
    excluded = 0
    failed = 0
    changed = False
    built: list[dict[str, object]] = []
    for cache_key in sorted(list(targets)):
        if cache_key in arguments.exclude_target:
            excluded += 1
            continue
        pending = targets.get(cache_key)
        if not isinstance(pending, list) or not pending:
            targets.pop(cache_key, None)
            changed = True
            continue
        metadata = pending[0] if isinstance(pending[0], dict) else {}
        next_attempt = float(metadata.get("nextAttemptAt", 0) or 0)
        if next_attempt > now and not arguments.force:
            skipped += 1
            results.append({"cacheKey": cache_key, "status": "retry_backoff", "retryAt": next_attempt})
            continue
        tree_value, separator, target = cache_key.rpartition(":")
        tree = ROOT / tree_value
        if not separator or not target or not tree.is_dir():
            failed += 1
            results.append({"cacheKey": cache_key, "status": "invalid_target", "tree": tree_value, "target": target})
            continue
        preflight = run([sys.executable, "tools/check_web_link_closure.py", tree_value, target, "--json"], 60)
        if preflight["exitCode"] != 0:
            attempt = dict(metadata)
            attempt["lastAttemptAt"] = datetime.now(timezone.utc).isoformat()
            attempt["nextAttemptAt"] = now + RETRY_SECONDS
            attempt["lastError"] = "link-closure-preflight"
            targets[cache_key] = [attempt, *pending[1:]]
            changed = True
            failed += 1
            results.append({"cacheKey": cache_key, "status": "preflight_failed", "preflight": preflight})
            continue
        build = run(["cmake", "--build", tree_value, "--target", target, f"-j{JOBS}"], 1800)
        runtime = tree / "bin" / f"{target}.js"
        wasm = tree / "bin" / f"{target}.wasm"
        if build["exitCode"] != 0 or not runtime.is_file() or not wasm.is_file():
            attempt = dict(metadata)
            attempt["lastAttemptAt"] = datetime.now(timezone.utc).isoformat()
            attempt["nextAttemptAt"] = now + RETRY_SECONDS
            attempt["lastError"] = "build" if build["exitCode"] != 0 else "artifact-missing"
            targets[cache_key] = [attempt, *pending[1:]]
            changed = True
            failed += 1
            results.append({"cacheKey": cache_key, "status": "build_failed", "build": build})
            continue
        built.append({
            "cacheKey": cache_key,
            "pending": pending,
            "target": target,
            "tree": tree_value,
            "artifacts": [str(path.relative_to(ROOT)) for path in (runtime, wasm)],
            "build": build,
        })
    if built:
        sync = run(["node", "web/tools/sync-assets.mjs"], 300)
    else:
        sync = None
    if sync is not None and sync["exitCode"] != 0:
        failed += len(built)
        for built_target in built:
            cache_key = str(built_target["cacheKey"])
            pending = built_target["pending"] if isinstance(built_target["pending"], list) else []
            metadata = dict(pending[0]) if pending and isinstance(pending[0], dict) else {}
            metadata.update({
                "lastAttemptAt": datetime.now(timezone.utc).isoformat(),
                "nextAttemptAt": now + RETRY_SECONDS,
                "lastError": "artifact-sync",
            })
            targets[cache_key] = [metadata, *pending[1:]]
            results.append({"cacheKey": cache_key, "status": "artifact_sync_failed", "build": built_target["build"]})
        changed = True
    else:
        for built_target in built:
            cache_key = str(built_target["cacheKey"])
            pending = built_target["pending"] if isinstance(built_target["pending"], list) else []
            for batch in pending:
                if not isinstance(batch, dict):
                    continue
                record_implementation_batch(
                    batch,
                    target=str(built_target["target"]),
                    build_tree=str(built_target["tree"]),
                    artifacts=list(built_target["artifacts"]),
                    source="deferred-build-flush",
                )
            targets.pop(cache_key, None)
            changed = True
            flushed += len(pending)
            results.append({"cacheKey": cache_key, "status": "flushed", "batchCount": len(pending), "build": built_target["build"]})
    state["targets"] = targets
    if changed:
        save_state(state)
    audit = run([sys.executable, "tools/generate_web_business_completion_audit.py"], 60) if flushed else None
    status = "flushed" if flushed and not failed else "partial" if flushed else "failed" if failed else "coalescing" if excluded and not skipped else "retry_backoff"
    payload = {
        "schema": "freecad-web-deferred-build-flush/v1",
        "generatedAt": datetime.now(timezone.utc).isoformat(),
        "status": status,
        "flushedBatches": flushed,
        "skippedTargets": skipped,
        "excludedTargets": excluded,
        "failedTargets": failed,
        "sync": sync,
        "businessAudit": audit,
        "results": results,
    }
    save_report(payload)
    print(json.dumps({"status": status, "flushedBatches": flushed, "skippedTargets": skipped, "failedTargets": failed}, ensure_ascii=False))
    return 0 if not failed else 1


if __name__ == "__main__":
    raise SystemExit(main())