744 lines
40 KiB
Python
744 lines
40 KiB
Python
#!/usr/bin/env python3
|
|
"""Run the deterministic post-edit pipeline for the current module batch."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
from web_build_lock import run_build
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from implementation_first import implementation_first_enabled, record_implementation_batch, user_validation_deferred
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PACKET_PATH = ROOT / "docs/current/module-implementation-packet.json"
|
|
BATCH_PATH = ROOT / "docs/current/module-batch-manifest.json"
|
|
OUTPUT_PATH = ROOT / "docs/current/module-batch-pipeline.json"
|
|
BUILD_CACHE_PATH = ROOT / "docs/current/module-build-reuse-cache.json"
|
|
PENDING_BUILD_PATH = ROOT / "docs/current/module-build-pending.json"
|
|
JOBS = max(1, int(os.environ.get("FREECAD_BUILD_JOBS", "8")))
|
|
FEATURE_LINK_BATCHES = max(1, min(2, int(os.environ.get("FREECAD_FEATURE_LINK_BATCHES", "2"))))
|
|
|
|
|
|
def load_build_cache() -> dict[str, object]:
|
|
try:
|
|
value = json.loads(BUILD_CACHE_PATH.read_text(encoding="utf-8"))
|
|
return value if isinstance(value, dict) else {}
|
|
except (OSError, json.JSONDecodeError):
|
|
return {}
|
|
|
|
|
|
def digest_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def file_signature(path: Path, previous: dict[str, object] | None = None) -> tuple[dict[str, object], bool]:
|
|
stat = path.stat()
|
|
if previous and previous.get("size") == stat.st_size and previous.get("mtimeNs") == stat.st_mtime_ns and previous.get("ctimeNs") == stat.st_ctime_ns and isinstance(previous.get("sha256"), str):
|
|
return dict(previous), True
|
|
return {"path": str(path.relative_to(ROOT)), "size": stat.st_size, "mtimeNs": stat.st_mtime_ns, "ctimeNs": stat.st_ctime_ns, "sha256": digest_file(path)}, False
|
|
|
|
|
|
def signature_digest(signatures: list[dict[str, object]]) -> str:
|
|
value = json.dumps([(row.get("path"), row.get("sha256")) for row in signatures], separators=(",", ":"))
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def save_build_cache(cache: dict[str, object]) -> None:
|
|
cache["schema"] = "freecad-web-module-build-reuse-cache/v1"
|
|
cache["updatedAt"] = datetime.now(timezone.utc).isoformat()
|
|
BUILD_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = BUILD_CACHE_PATH.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(cache, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(BUILD_CACHE_PATH)
|
|
|
|
|
|
def load_pending_builds() -> dict[str, object]:
|
|
try:
|
|
value = json.loads(PENDING_BUILD_PATH.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
value = {}
|
|
if not isinstance(value, dict) or value.get("schema") != "freecad-web-module-build-pending/v1":
|
|
value = {"schema": "freecad-web-module-build-pending/v1", "targets": {}}
|
|
value.setdefault("targets", {})
|
|
return value
|
|
|
|
|
|
def save_pending_builds(state: dict[str, object]) -> None:
|
|
state["schema"] = "freecad-web-module-build-pending/v1"
|
|
state["updatedAt"] = datetime.now(timezone.utc).isoformat()
|
|
PENDING_BUILD_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = PENDING_BUILD_PATH.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(PENDING_BUILD_PATH)
|
|
|
|
|
|
def feature_has_more_same_family(batch: dict[str, object]) -> bool:
|
|
selected = batch.get("selected") if isinstance(batch.get("selected"), dict) else {}
|
|
module = selected.get("module")
|
|
family = selected.get("family")
|
|
selected_count = int(selected.get("recordCount", 0) or 0)
|
|
if family == "desktop-command-mixed":
|
|
remaining = sum(int(row.get("unresolved", 0) or 0) for row in batch.get("queue", [])
|
|
if isinstance(row, dict) and row.get("module") == module and str(row.get("family", "")).startswith("desktop-command"))
|
|
return remaining > selected_count
|
|
for row in batch.get("queue", []) if isinstance(batch.get("queue"), list) else []:
|
|
if not isinstance(row, dict) or row.get("module") != module or row.get("family") != family:
|
|
continue
|
|
return int(row.get("unresolved", 0) or 0) > selected_count
|
|
return False
|
|
|
|
|
|
def should_defer_feature_build(
|
|
batch: dict[str, object],
|
|
*,
|
|
implementation_first: bool,
|
|
pending_count: int,
|
|
artifacts_exist: bool,
|
|
) -> bool:
|
|
selected = batch.get("selected") if isinstance(batch.get("selected"), dict) else {}
|
|
return bool(
|
|
implementation_first
|
|
and FEATURE_LINK_BATCHES > 1
|
|
and pending_count == 0
|
|
and artifacts_exist
|
|
and selected.get("family") != "runtime-issue"
|
|
and feature_has_more_same_family(batch)
|
|
)
|
|
|
|
|
|
def direct_link_dependencies(tree: Path, target: str) -> list[Path]:
|
|
ninja = tree / "build.ninja"
|
|
prefix = f"build bin/{target}.js:"
|
|
try:
|
|
with ninja.open(encoding="utf-8", errors="ignore") as stream:
|
|
line = next((value for value in stream if value.startswith(prefix)), "")
|
|
except OSError:
|
|
return []
|
|
dependencies: list[Path] = []
|
|
for token in line.split():
|
|
if not token.endswith(".a"):
|
|
continue
|
|
candidate = Path(token)
|
|
resolved = candidate if candidate.is_absolute() else tree / candidate
|
|
if resolved.is_file():
|
|
dependencies.append(resolved)
|
|
return list(dict.fromkeys(dependencies))
|
|
|
|
|
|
def target_source_dependencies(tree: Path, target: str) -> list[Path]:
|
|
"""Resolve only source files that compile into the selected target."""
|
|
ninja = tree / "build.ninja"
|
|
prefix = f"build bin/{target}.js:"
|
|
try:
|
|
lines = ninja.read_text(encoding="utf-8", errors="ignore").splitlines()
|
|
except OSError:
|
|
return []
|
|
link_line = next((line for line in lines if line.startswith(prefix)), "")
|
|
objects = {token for token in link_line.split() if token.endswith((".o", ".obj"))}
|
|
sources: list[Path] = []
|
|
for line in lines:
|
|
if not line.startswith("build "):
|
|
continue
|
|
output = line.split(":", 1)[0].removeprefix("build ").strip()
|
|
if output not in objects:
|
|
continue
|
|
tokens = line.split()
|
|
source = next((token for token in reversed(tokens) if token.endswith((".c", ".cc", ".cpp", ".cxx"))), None)
|
|
if not source:
|
|
continue
|
|
candidate = Path(source)
|
|
resolved = candidate if candidate.is_absolute() else tree / candidate
|
|
if resolved.is_file():
|
|
sources.append(resolved.resolve())
|
|
return list(dict.fromkeys(sources))
|
|
|
|
|
|
def compact(value: str, limit: int = 4000, *, preserve_tail: bool = False) -> str:
|
|
value = value.strip()
|
|
if len(value) <= limit:
|
|
return value
|
|
if preserve_tail:
|
|
head = min(600, limit // 4)
|
|
marker = "\n... output middle omitted; failure tail follows ...\n"
|
|
tail = max(0, limit - head - len(marker))
|
|
return value[:head] + marker + value[-tail:]
|
|
return value[: limit - 3] + "..."
|
|
|
|
|
|
def compiler_diagnostics(value: str, limit: int = 4000) -> str:
|
|
"""Keep compiler errors visible when a long Emscripten argv fills the tail."""
|
|
lines = value.splitlines()
|
|
selected: list[str] = []
|
|
diagnostic = re.compile(r"(?:FAILED:|fatal error:|\berror:|undefined symbol:|ninja: build stopped)", re.IGNORECASE)
|
|
for index, line in enumerate(lines):
|
|
if not diagnostic.search(line):
|
|
continue
|
|
# Emscripten repeats its complete multi-kilobyte compiler argv in an
|
|
# `em++: error:` line after the useful clang diagnostic. Keeping that
|
|
# line would evict the actual `file:line: error:` from the compact log.
|
|
if len(line) > 1000:
|
|
continue
|
|
selected.append(line)
|
|
selected.extend(value for value in lines[index + 1:index + 3] if len(value) < 500)
|
|
compacted = "\n".join(dict.fromkeys(selected))
|
|
return compacted[-limit:]
|
|
|
|
|
|
def business_failure_diagnostics(stdout: str) -> str:
|
|
try:
|
|
payload = json.loads(stdout)
|
|
except (ValueError, TypeError):
|
|
return ""
|
|
if not isinstance(payload, dict):
|
|
return ""
|
|
result = {}
|
|
for key in ("reason", "error", "stderr", "status", "exitCode", "remaining", "pendingIssues", "evidencePath"):
|
|
value = payload.get(key)
|
|
if value is None or value == "" or value == []:
|
|
continue
|
|
result[key] = compact(value, 1800, preserve_tail=True) if isinstance(value, str) else value
|
|
if not any(key in result for key in ("reason", "error", "stderr")):
|
|
try:
|
|
nested = json.loads(payload.get("stdout", "{}"))
|
|
except (ValueError, TypeError):
|
|
nested = {}
|
|
if isinstance(nested, dict):
|
|
errors = [{"id": row.get("capabilityId"), "error": compact(str(row["error"]), 1000, preserve_tail=True)}
|
|
for row in nested.get("results", []) if isinstance(row, dict) and row.get("error")]
|
|
if errors:
|
|
result = {"caseErrors": errors[:16], **result}
|
|
return compact(json.dumps(result, ensure_ascii=False), 3000, preserve_tail=True) if result else ""
|
|
|
|
|
|
def run_step(name: str, command: list[str], env: dict[str, str], timeout: int) -> dict[str, object]:
|
|
started = time.monotonic()
|
|
try:
|
|
cwd = ROOT / "web" if command and Path(command[0]) == ROOT / "web/node_modules/.bin/vite" else ROOT
|
|
result = run_build(command, cwd=cwd, env=env, text=True, capture_output=True, timeout=timeout, check=False)
|
|
failed = result.returncode != 0
|
|
stdout = compact(result.stdout, preserve_tail=failed)
|
|
stderr = compact(result.stderr, preserve_tail=failed)
|
|
if failed and name == "business-acceptance":
|
|
diagnostics = business_failure_diagnostics(result.stdout)
|
|
if diagnostics:
|
|
stderr = (stderr + "\n--- extracted business diagnostics ---\n" + diagnostics).strip()
|
|
if failed:
|
|
diagnostics = compiler_diagnostics((result.stdout or "") + "\n" + (result.stderr or ""))
|
|
if diagnostics:
|
|
stderr = (stderr + "\n--- extracted diagnostics ---\n" + diagnostics).strip()
|
|
return {
|
|
"name": name,
|
|
"command": command,
|
|
"seconds": round(time.monotonic() - started, 3),
|
|
"exitCode": result.returncode,
|
|
"stdout": stdout,
|
|
"stderr": stderr,
|
|
}
|
|
except subprocess.TimeoutExpired as error:
|
|
return {
|
|
"name": name,
|
|
"command": command,
|
|
"seconds": round(time.monotonic() - started, 3),
|
|
"exitCode": 124,
|
|
"stdout": compact(error.stdout or ""),
|
|
"stderr": compact(error.stderr or "pipeline step timed out"),
|
|
}
|
|
|
|
|
|
def save(payload: dict[str, object]) -> None:
|
|
try:
|
|
packet = json.loads(PACKET_PATH.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
packet = {}
|
|
if packet.get("batchId") == payload.get("batchId") and "supportFiles" in packet:
|
|
payload["supportHashes"] = {row["path"]: row.get("sha256") for row in packet["supportFiles"] if isinstance(row, dict) and row.get("path")}
|
|
OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = OUTPUT_PATH.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(OUTPUT_PATH)
|
|
|
|
|
|
def static_readiness(batch: dict[str, object], capability_payload: dict[str, object]) -> dict[str, object]:
|
|
selected_ids = {
|
|
str(row[0]) for row in batch.get("records", [])
|
|
if isinstance(row, list) and row and row[0]
|
|
}
|
|
ready_ids = {
|
|
str(row.get("id")) for row in capability_payload.get("records", [])
|
|
if isinstance(row, dict) and row.get("implementationStatus") == "routed"
|
|
}
|
|
missing_ids = sorted(selected_ids - ready_ids)
|
|
capability_by_id = {
|
|
str(row.get("id")): row for row in capability_payload.get("records", [])
|
|
if isinstance(row, dict) and row.get("id")
|
|
}
|
|
ui_containers = sum(
|
|
capability_by_id.get(record_id, {}).get("uiRole") == "container"
|
|
for record_id in selected_ids
|
|
)
|
|
selected = batch.get("selected") if isinstance(batch.get("selected"), dict) else {}
|
|
diagnostics = capability_payload.get("diagnostics") if isinstance(capability_payload.get("diagnostics"), dict) else {}
|
|
rejected_by_module = diagnostics.get("rejectedDesktopMarkers") if isinstance(diagnostics.get("rejectedDesktopMarkers"), dict) else {}
|
|
rejected_commands = set(rejected_by_module.get(str(selected.get("module") or ""), []))
|
|
rejected_markers = [
|
|
{"id": str(row[0]), "command": str(row[1]), "reason": diagnostics.get("rejectedMarkerReason")}
|
|
for row in batch.get("records", [])
|
|
if isinstance(row, list) and len(row) > 1 and str(row[0]) in missing_ids and str(row[1]) in rejected_commands
|
|
]
|
|
return {
|
|
"selected": len(selected_ids),
|
|
"ready": len(selected_ids) - len(missing_ids),
|
|
"missing": len(missing_ids),
|
|
"missingIds": missing_ids,
|
|
"rejectedMarkers": rejected_markers,
|
|
"blockingGaps": [{"id": record_id, "gaps": capability_by_id.get(record_id, {}).get("implementationGaps", [])}
|
|
for record_id in missing_ids],
|
|
"uiContainers": ui_containers,
|
|
"uiOnly": bool(selected_ids) and ui_containers == len(selected_ids),
|
|
}
|
|
|
|
|
|
def acceptance_contract_gap(batch: dict, packet: dict) -> dict | None:
|
|
value = packet.get("acceptanceContract")
|
|
if not value:
|
|
return None
|
|
selected = sorted({str(row[0]) for row in batch.get("records", []) if row})
|
|
path = (ROOT / str(value)).resolve()
|
|
reason = None
|
|
covered = set()
|
|
if not path.is_relative_to(ROOT):
|
|
reason = "acceptance contract must be inside the project"
|
|
else:
|
|
try:
|
|
contract = json.loads(path.read_text(encoding="utf-8"))
|
|
ids = contract.get("coveredIds", [])
|
|
command = contract.get("command", [])
|
|
if not isinstance(ids, list) or not all(isinstance(item, str) for item in ids):
|
|
reason = "coveredIds must be a list of business IDs"
|
|
else:
|
|
covered = set(ids)
|
|
if not isinstance(command, list) or len(command) < 2 or command[0] not in {"node", "python3"}:
|
|
reason = "acceptance contract needs an executable case command"
|
|
else:
|
|
script = (ROOT / str(command[1])).resolve()
|
|
if not script.is_relative_to(ROOT) or not script.is_file():
|
|
reason = "acceptance case script is missing or outside the project"
|
|
except (OSError, ValueError, AttributeError) as error:
|
|
reason = f"acceptance contract cannot be read: {type(error).__name__}"
|
|
missing = sorted(set(selected) - covered)
|
|
if selected and not missing and reason is None:
|
|
return None
|
|
return {"status": "acceptance_required", "contract": str(value), "selectedIds": selected,
|
|
"missingIds": missing, "coveredIds": sorted(covered),
|
|
"reason": reason or "selected business IDs have no declared cases; complete their real cases before building",
|
|
"buildRequired": False}
|
|
|
|
|
|
def main() -> int:
|
|
try:
|
|
packet = json.loads(PACKET_PATH.read_text(encoding="utf-8"))
|
|
batch = json.loads(BATCH_PATH.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
print(json.dumps({"status": "error", "error": str(error)}))
|
|
return 2
|
|
defer_validation = user_validation_deferred()
|
|
if defer_validation and batch.get("selected", {}).get("tier") in {"business-acceptance", "failure-reconciliation"}:
|
|
print(json.dumps({"status": "validation_deferred_to_user", "batchId": batch.get("batchId"), "nextAction": "replan-implementation"}))
|
|
return 0
|
|
if packet.get("batchId") != batch.get("batchId") or packet.get("outerPipelineAvailable") is not True:
|
|
print(json.dumps({"status": "unavailable", "batchId": batch.get("batchId")}))
|
|
return 78
|
|
if batch.get("selected", {}).get("tier") == "failure-reconciliation":
|
|
result = run_step("failure-reconciliation", [sys.executable, "tools/failure_reconciliation.py", "--check", "--keys", *batch.get("failureKeys", [])], dict(os.environ), 60)
|
|
save({"schema": "freecad-web-module-batch-pipeline/v1", "batchId": batch.get("batchId"),
|
|
"status": "pass" if result["exitCode"] == 0 else "reconciliation_incomplete", "steps": [result]})
|
|
print(json.dumps(result))
|
|
return 0 if result["exitCode"] in (0, 1) else int(result["exitCode"])
|
|
runtime_repair = batch.get("selected", {}).get("family") == "runtime-issue"
|
|
acceptance_turn = not defer_validation and (batch.get("selected", {}).get("tier") == "business-acceptance" or runtime_repair)
|
|
coverage_started = time.perf_counter()
|
|
coverage_gap = acceptance_contract_gap(batch, packet) if acceptance_turn else None
|
|
if coverage_gap:
|
|
step = {"name": "acceptance-contract-preflight", "exitCode": 78,
|
|
"seconds": round(time.perf_counter() - coverage_started, 6),
|
|
"stdout": json.dumps(coverage_gap), "stderr": ""}
|
|
save({"schema": "freecad-web-module-batch-pipeline/v1", "batchId": batch.get("batchId"),
|
|
"module": batch.get("selected", {}).get("module"), "status": "failed",
|
|
"failedStage": "business-acceptance", "buildRequired": False, "steps": [step],
|
|
"supportHashes": {row["path"]: row.get("sha256") for row in packet.get("supportFiles", [])
|
|
if isinstance(row, dict) and row.get("path")}})
|
|
print(json.dumps(coverage_gap))
|
|
return 78
|
|
if acceptance_turn and not packet.get("build"):
|
|
result = run_step("business-acceptance", [sys.executable, "tools/run_business_acceptance.py", "--reuse",
|
|
*(["--require-issue-closure"] if runtime_repair else [])], dict(os.environ), 200)
|
|
save({"schema": "freecad-web-module-batch-pipeline/v1", "batchId": batch.get("batchId"),
|
|
"status": "pass" if result["exitCode"] == 0 else "failed",
|
|
"failedStage": None if result["exitCode"] == 0 else "business-acceptance", "steps": [result]})
|
|
subprocess.run([sys.executable, "tools/generate_web_business_completion_audit.py"], cwd=ROOT, check=False)
|
|
print(json.dumps(result))
|
|
return int(result["exitCode"])
|
|
build = packet.get("build") if isinstance(packet.get("build"), dict) else {}
|
|
build_kind = str(build.get("kind") or "native")
|
|
target = str(build.get("target") or "")
|
|
tree = ROOT / str(build.get("tree") or "")
|
|
build_bin = ROOT / str(build.get("bin") or "")
|
|
runtime = ROOT / str(build.get("runtime") or "")
|
|
wasm = ROOT / str(build.get("wasm") or "")
|
|
artifact_paths = list(dict.fromkeys((runtime, wasm)))
|
|
gate_value = str(packet.get("gate") or "")
|
|
gate = ROOT / gate_value if gate_value else None
|
|
implementation_mode = implementation_first_enabled() or acceptance_turn
|
|
feature_implementation = bool(
|
|
implementation_mode
|
|
and not acceptance_turn
|
|
and batch.get("strategy") == "feature-first-deferred-validation"
|
|
)
|
|
readiness: dict[str, object] = {}
|
|
if not target or not tree.is_dir() or (not implementation_mode and (gate is None or not gate.is_file())):
|
|
print(json.dumps({"status": "unavailable", "error": "pipeline target/tree missing, or validation gate missing outside implementation-first mode"}))
|
|
return 78
|
|
environment = {key: str(value) for key, value in os.environ.items()}
|
|
environment["EM_CONFIG"] = str(ROOT / "tools/emscripten-writable-config.py")
|
|
environment["EMCC_CORES"] = str(JOBS)
|
|
environment["CMAKE_BUILD_PARALLEL_LEVEL"] = str(JOBS)
|
|
environment["FREECAD_WEB_FAST_LINK"] = "ON"
|
|
steps: list[dict[str, object]] = []
|
|
cmake_source = ROOT / "freecad-source/src/WebAssembly/FreeCADCoreProbe/CMakeLists.txt"
|
|
ninja_source = tree / "build.ninja"
|
|
if (
|
|
build_kind != "frontend"
|
|
and cmake_source.is_file()
|
|
and ninja_source.is_file()
|
|
and cmake_source.stat().st_mtime_ns > ninja_source.stat().st_mtime_ns
|
|
):
|
|
steps.append(run_step("cmake-regenerate", ["cmake", "--build", str(tree.relative_to(ROOT)), "--target", "rebuild_cache", f"-j{JOBS}"], environment, 300))
|
|
if steps[-1]["exitCode"] != 0:
|
|
payload = {"schema": "freecad-web-module-batch-pipeline/v1", "status": "failed", "batchId": batch.get("batchId"), "module": packet.get("module"), "target": target, "buildTree": str(tree.relative_to(ROOT)), "buildKind": build_kind, "failedStage": "cmake-regenerate", "steps": steps}
|
|
save(payload)
|
|
print(json.dumps({"status": "failed", "stage": "cmake-regenerate", "exitCode": steps[-1]["exitCode"]}))
|
|
return int(steps[-1]["exitCode"])
|
|
if feature_implementation:
|
|
# Refresh cheap generated route state before deciding whether an
|
|
# expensive native build is warranted. This is a manifest-driven
|
|
# completeness check, not a runtime validation gate.
|
|
steps.append(run_step("module-registry", [sys.executable, "tools/generate_web_module_registry.py"], environment, 60))
|
|
steps.append(run_step("capability-registry", [sys.executable, "tools/generate_feature_capability_registry.py"], environment, 60))
|
|
if any(step["exitCode"] != 0 for step in steps[-2:]):
|
|
payload = {"schema": "freecad-web-module-batch-pipeline/v1", "status": "failed", "batchId": batch.get("batchId"), "module": packet.get("module"), "target": target, "buildTree": str(tree.relative_to(ROOT)), "buildKind": build_kind, "failedStage": "capability-registry", "steps": steps}
|
|
save(payload)
|
|
print(json.dumps({"status": "failed", "stage": "capability-registry"}))
|
|
return int(next(step["exitCode"] for step in steps[-2:] if step["exitCode"] != 0))
|
|
capability_payload = json.loads((ROOT / "docs/current/feature-capability-registry.json").read_text(encoding="utf-8"))
|
|
readiness = static_readiness(batch, capability_payload)
|
|
if readiness["missing"] and not readiness["ready"]:
|
|
payload = {
|
|
"schema": "freecad-web-module-batch-pipeline/v1",
|
|
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
|
"status": "implementation_incomplete",
|
|
"implementationMode": "feature-first-deferred-validation",
|
|
"validationDeferred": True,
|
|
"batchId": batch.get("batchId"),
|
|
"module": packet.get("module"),
|
|
"target": target,
|
|
"buildTree": str(tree.relative_to(ROOT)),
|
|
"buildRequired": False,
|
|
"buildReuse": {"reason": "static-readiness-incomplete", "cacheHit": False},
|
|
"staticReadiness": readiness,
|
|
"totalSeconds": round(sum(float(step["seconds"]) for step in steps), 3),
|
|
"steps": steps,
|
|
}
|
|
save(payload)
|
|
print(json.dumps({"status": "implementation_incomplete", "batchId": batch.get("batchId"), "buildRequired": False, "staticReadiness": readiness, "totalSeconds": payload["totalSeconds"]}, ensure_ascii=False))
|
|
return 0
|
|
if readiness["uiOnly"]:
|
|
artifacts = [str(path.relative_to(ROOT)) for path in artifact_paths if path.exists()]
|
|
receipt = record_implementation_batch(
|
|
batch,
|
|
target=target,
|
|
build_tree=str(tree.relative_to(ROOT)),
|
|
artifacts=artifacts,
|
|
source="react-navigation",
|
|
)
|
|
if not defer_validation:
|
|
steps.append(run_step("business-audit", [sys.executable, "tools/generate_web_business_completion_audit.py"], environment, 60))
|
|
payload = {
|
|
"schema": "freecad-web-module-batch-pipeline/v1",
|
|
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
|
"status": "implementation_ready",
|
|
"implementationMode": "feature-first-deferred-validation",
|
|
"validationDeferred": True,
|
|
"batchId": batch.get("batchId"),
|
|
"module": packet.get("module"),
|
|
"target": target,
|
|
"buildTree": str(tree.relative_to(ROOT)),
|
|
"buildRequired": False,
|
|
"buildReuse": {"reason": "react-navigation-only", "cacheHit": True},
|
|
"staticReadiness": readiness,
|
|
"implementationReceipt": receipt,
|
|
"implementationReceipts": [receipt],
|
|
"coalescedBatchCount": 1,
|
|
"totalSeconds": round(sum(float(step["seconds"]) for step in steps), 3),
|
|
"steps": steps,
|
|
}
|
|
save(payload)
|
|
print(json.dumps({"status": "implementation_ready", "batchId": batch.get("batchId"), "buildRequired": False, "buildReuse": payload["buildReuse"], "totalSeconds": payload["totalSeconds"]}, ensure_ascii=False))
|
|
return 0
|
|
if build_kind != "frontend":
|
|
steps.append(run_step("preflight", [sys.executable, "tools/check_web_link_closure.py", str(tree.relative_to(ROOT)), target, "--json"], environment, 60))
|
|
if steps[-1]["exitCode"] != 0:
|
|
payload = {"schema": "freecad-web-module-batch-pipeline/v1", "status": "failed", "batchId": batch.get("batchId"), "module": packet.get("module"), "target": target, "buildTree": str(tree.relative_to(ROOT)), "buildKind": build_kind, "failedStage": "preflight", "steps": steps}
|
|
save(payload)
|
|
print(json.dumps({"status": "failed", "stage": "preflight", "exitCode": steps[-1]["exitCode"]}))
|
|
return int(steps[-1]["exitCode"])
|
|
artifacts_exist = bool(artifact_paths) and all(path.exists() for path in artifact_paths)
|
|
artifact_mtime = min(path.stat().st_mtime_ns for path in artifact_paths) if artifacts_exist else 0
|
|
native_sources = [
|
|
ROOT / str(row.get("path")) for row in packet.get("supportFiles", [])
|
|
if isinstance(row, dict) and str(row.get("path") or "").endswith((".cpp", ".h", ".hpp"))
|
|
]
|
|
target_sources = target_source_dependencies(tree, target)
|
|
if build_kind == "frontend":
|
|
input_paths = [
|
|
ROOT / str(row.get("path"))
|
|
for row in packet.get("supportFiles", [])
|
|
if isinstance(row, dict) and str(row.get("path") or "").endswith((".ts", ".tsx", ".css", ".html", ".json"))
|
|
]
|
|
input_paths.extend(ROOT / value for value in ("web/package.json", "web/tsconfig.json", "web/vite.config.ts"))
|
|
input_paths = [path for path in dict.fromkeys(input_paths) if path.exists()]
|
|
link_dependencies: list[Path] = []
|
|
else:
|
|
input_paths = [path for path in [*(target_sources or native_sources), cmake_source, ninja_source] if path.exists()]
|
|
link_dependencies = direct_link_dependencies(tree, target)
|
|
cache = load_build_cache()
|
|
entries = cache.get("entries") if isinstance(cache.get("entries"), dict) else {}
|
|
cache_key = f"{tree.relative_to(ROOT)}:{target}"
|
|
pending_state = load_pending_builds()
|
|
pending_targets = pending_state.get("targets") if isinstance(pending_state.get("targets"), dict) else {}
|
|
pending_batches = pending_targets.get(cache_key) if isinstance(pending_targets.get(cache_key), list) else []
|
|
can_defer_build = should_defer_feature_build(
|
|
batch,
|
|
implementation_first=feature_implementation,
|
|
pending_count=len(pending_batches),
|
|
artifacts_exist=artifacts_exist,
|
|
) and not readiness.get("missing")
|
|
if can_defer_build:
|
|
pending_targets[cache_key] = [batch]
|
|
pending_state["targets"] = pending_targets
|
|
save_pending_builds(pending_state)
|
|
pending_receipt = record_implementation_batch(
|
|
batch,
|
|
target=target,
|
|
build_tree=str(tree.relative_to(ROOT)),
|
|
artifacts=[str(path.relative_to(ROOT)) for path in artifact_paths if path.exists()],
|
|
source="source-ready-awaiting-coalesced-build",
|
|
status="implementation_pending_build",
|
|
)
|
|
if not defer_validation:
|
|
steps.append(run_step("business-audit", [sys.executable, "tools/generate_web_business_completion_audit.py"], environment, 60))
|
|
payload = {
|
|
"schema": "freecad-web-module-batch-pipeline/v1",
|
|
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
|
"status": "implementation_pending_build",
|
|
"implementationMode": "feature-first-deferred-validation",
|
|
"validationDeferred": True,
|
|
"batchId": batch.get("batchId"),
|
|
"module": packet.get("module"),
|
|
"target": target,
|
|
"buildTree": str(tree.relative_to(ROOT)),
|
|
"buildRequired": False,
|
|
"buildDeferred": True,
|
|
"deferredBatchCount": 1,
|
|
"deferReason": "same-module-family-next-batch",
|
|
"implementationReceipt": pending_receipt,
|
|
"staticReadiness": readiness,
|
|
"totalSeconds": round(sum(float(step["seconds"]) for step in steps), 3),
|
|
"steps": steps,
|
|
}
|
|
save(payload)
|
|
print(json.dumps({"status": payload["status"], "batchId": batch.get("batchId"), "buildRequired": False, "buildDeferred": True, "totalSeconds": payload["totalSeconds"]}, ensure_ascii=False))
|
|
return 0
|
|
previous_entry = entries.get(cache_key) if isinstance(entries.get(cache_key), dict) else {}
|
|
previous_inputs = {str(row.get("path")): row for row in previous_entry.get("inputs", []) if isinstance(row, dict)}
|
|
previous_artifacts = {str(row.get("path")): row for row in previous_entry.get("artifacts", []) if isinstance(row, dict)}
|
|
input_signatures: list[dict[str, object]] = []
|
|
input_hash_hits = 0
|
|
for path in input_paths:
|
|
signature, hit = file_signature(path, previous_inputs.get(str(path.relative_to(ROOT))))
|
|
input_signatures.append(signature)
|
|
input_hash_hits += int(hit)
|
|
input_digest = signature_digest(input_signatures)
|
|
artifact_signatures: list[dict[str, object]] = []
|
|
artifact_hash_hits = 0
|
|
if artifacts_exist:
|
|
for path in artifact_paths:
|
|
signature, hit = file_signature(path, previous_artifacts.get(str(path.relative_to(ROOT))))
|
|
artifact_signatures.append(signature)
|
|
artifact_hash_hits += int(hit)
|
|
cache_hit = bool(
|
|
artifact_signatures
|
|
and previous_entry.get("inputDigest") == input_digest
|
|
and previous_entry.get("artifactDigest") == signature_digest(artifact_signatures)
|
|
)
|
|
mtime_clean = bool(artifact_mtime and all(path.stat().st_mtime_ns <= artifact_mtime for path in input_paths))
|
|
dependency_dirty = any(path.stat().st_mtime_ns > artifact_mtime for path in link_dependencies) if artifact_mtime else True
|
|
# mtime is diagnostic only. The first run must establish a trusted
|
|
# build cache entry; acceptance results are tracked independently. Later
|
|
# skips require matching input and artifact content from a successful build.
|
|
build_required = not artifacts_exist or not cache_hit or dependency_dirty
|
|
reuse_reason = "content-hash" if cache_hit else "untrusted-cache-seed" if mtime_clean else "input-changed"
|
|
# Ninja must still check transitive headers and library sources: the packet
|
|
# fingerprints do not enumerate the compiler's entire dependency graph.
|
|
if build_required or build_kind != "frontend":
|
|
build_command = (
|
|
([str(ROOT / "web/node_modules/.bin/vite"), "build"]
|
|
if defer_validation else ["npm", "--prefix", "web", "run", "build"])
|
|
if build_kind == "frontend"
|
|
else ["cmake", "--build", str(tree.relative_to(ROOT)), "--target", target, f"-j{JOBS}"]
|
|
)
|
|
steps.append(run_step("build", build_command, environment, 1800))
|
|
if steps[-1]["exitCode"] != 0:
|
|
payload = {"schema": "freecad-web-module-batch-pipeline/v1", "status": "failed", "batchId": batch.get("batchId"), "module": packet.get("module"), "target": target, "buildTree": str(tree.relative_to(ROOT)), "buildKind": build_kind, "failedStage": "build", "buildRequired": True, "steps": steps}
|
|
save(payload)
|
|
print(json.dumps({"status": "failed", "stage": "build", "exitCode": steps[-1]["exitCode"]}))
|
|
return int(steps[-1]["exitCode"])
|
|
checked_artifacts = [file_signature(path, previous_artifacts.get(str(path.relative_to(ROOT))))[0] for path in artifact_paths]
|
|
artifact_changed = signature_digest(checked_artifacts) != signature_digest(artifact_signatures)
|
|
build_required = build_required or artifact_changed
|
|
if build_kind != "frontend":
|
|
steps.append(run_step("artifact-sync", ["node", "web/tools/sync-assets.mjs"], environment, 300))
|
|
if steps[-1]["exitCode"] != 0:
|
|
payload = {"schema": "freecad-web-module-batch-pipeline/v1", "status": "failed", "batchId": batch.get("batchId"), "module": packet.get("module"), "target": target, "buildTree": str(tree.relative_to(ROOT)), "buildKind": build_kind, "failedStage": "artifact-sync", "buildRequired": True, "steps": steps}
|
|
save(payload)
|
|
print(json.dumps({"status": "failed", "stage": "artifact-sync", "exitCode": steps[-1]["exitCode"]}))
|
|
return int(steps[-1]["exitCode"])
|
|
artifact_signatures = checked_artifacts
|
|
artifact_hash_hits = 0 if build_required else len(checked_artifacts)
|
|
input_signatures = [file_signature(path, previous_inputs.get(str(path.relative_to(ROOT))))[0] for path in input_paths]
|
|
input_digest = signature_digest(input_signatures)
|
|
input_hash_hits = 0 if build_required else input_hash_hits
|
|
reuse_reason = "rebuilt" if build_required else "content-hash+ninja"
|
|
if implementation_mode and artifact_signatures:
|
|
# A failed business case must not discard a successfully built target.
|
|
entries[cache_key] = {
|
|
"inputDigest": input_digest,
|
|
"artifactDigest": signature_digest(artifact_signatures),
|
|
"inputs": input_signatures,
|
|
"artifacts": artifact_signatures,
|
|
"updatedAt": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
cache["entries"] = entries
|
|
save_build_cache(cache)
|
|
receipt: dict[str, object] | None = None
|
|
receipts: list[dict[str, object]] = []
|
|
if implementation_mode:
|
|
artifacts = [str(path.relative_to(ROOT)) for path in artifact_paths if path.exists()]
|
|
for pending_batch in pending_batches:
|
|
if isinstance(pending_batch, dict) and pending_batch.get("batchId") != batch.get("batchId"):
|
|
receipts.append(record_implementation_batch(
|
|
pending_batch,
|
|
target=target,
|
|
build_tree=str(tree.relative_to(ROOT)),
|
|
artifacts=artifacts,
|
|
source="coalesced-outer-build",
|
|
))
|
|
receipt = None if acceptance_turn else record_implementation_batch(
|
|
batch,
|
|
target=target,
|
|
build_tree=str(tree.relative_to(ROOT)),
|
|
artifacts=artifacts,
|
|
source="outer-build" if feature_implementation else "module-business-build",
|
|
require_routed_capability=feature_implementation,
|
|
)
|
|
if receipt is not None:
|
|
receipts.append(receipt)
|
|
if pending_batches:
|
|
pending_targets.pop(cache_key, None)
|
|
pending_state["targets"] = pending_targets
|
|
save_pending_builds(pending_state)
|
|
status = "pass" if acceptance_turn else "implementation_ready" if int(receipt.get("remainingRecordCount", 0) or 0) == 0 else "implementation_partial"
|
|
if not defer_validation and (acceptance_turn or batch.get("selected", {}).get("family") == "runtime-issue"):
|
|
record_ids = [str(row[0]) for row in batch.get("records", []) if row]
|
|
steps.append(run_step("business-acceptance", [sys.executable, "tools/run_business_acceptance.py",
|
|
"--reuse", *(["--require-issue-closure"] if runtime_repair else []),
|
|
"--module", str(packet.get("module")), "--ids", *record_ids], environment, 210))
|
|
if steps[-1]["exitCode"] != 0:
|
|
payload = {"schema": "freecad-web-module-batch-pipeline/v1", "status": "failed",
|
|
"batchId": batch.get("batchId"), "module": packet.get("module"), "target": target,
|
|
"buildTree": str(tree.relative_to(ROOT)), "failedStage": "business-acceptance", "steps": steps,
|
|
"buildRequired": build_required, "buildReuse": {"reason": reuse_reason, "cacheHit": cache_hit},
|
|
"totalSeconds": round(sum(float(step["seconds"]) for step in steps), 3)}
|
|
save(payload)
|
|
print(json.dumps({"status": "failed", "stage": "business-acceptance", "exitCode": steps[-1]["exitCode"],
|
|
"reason": "Business completion requires executable cases; retain unfinished work until its result is verified"}))
|
|
return int(steps[-1]["exitCode"])
|
|
steps.append(run_step("business-audit", [sys.executable, "tools/generate_web_business_completion_audit.py"], environment, 60))
|
|
else:
|
|
assert gate is not None
|
|
steps.append(run_step("focused-gate", ["node", str(gate.relative_to(ROOT)), str(build_bin.relative_to(ROOT))], environment, 180))
|
|
if steps[-1]["exitCode"] == 0:
|
|
steps.append(run_step("incremental-context", [sys.executable, "tools/validate_incremental_context.py"], environment, 60))
|
|
status = "pass" if steps[-1]["exitCode"] == 0 and all(step["exitCode"] == 0 for step in steps) else "failed"
|
|
if not implementation_mode and status == "pass" and artifact_signatures:
|
|
entries[cache_key] = {
|
|
"inputDigest": input_digest,
|
|
"artifactDigest": signature_digest(artifact_signatures),
|
|
"inputs": input_signatures,
|
|
"artifacts": artifact_signatures,
|
|
"updatedAt": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
cache["entries"] = entries
|
|
save_build_cache(cache)
|
|
payload = {
|
|
"schema": "freecad-web-module-batch-pipeline/v1",
|
|
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
|
"status": status,
|
|
"implementationMode": (
|
|
"feature-first-deferred-validation" if feature_implementation
|
|
else "module-business-first-deferred-validation" if implementation_mode
|
|
else "evidence-first"
|
|
),
|
|
"validationDeferred": implementation_mode and not acceptance_turn,
|
|
"implementationReceipt": receipt,
|
|
"implementationReceipts": receipts,
|
|
"coalescedBatchCount": len(receipts),
|
|
"staticReadiness": readiness if feature_implementation else None,
|
|
"batchId": batch.get("batchId"),
|
|
"module": packet.get("module"),
|
|
"target": target,
|
|
"buildKind": build_kind,
|
|
"buildTree": str(tree.relative_to(ROOT)),
|
|
"buildRequired": build_required,
|
|
"buildReuse": {
|
|
"reason": reuse_reason,
|
|
"cacheHit": cache_hit,
|
|
"inputDigest": input_digest,
|
|
"inputHashCacheHits": input_hash_hits,
|
|
"inputFiles": len(input_signatures),
|
|
"artifactHashCacheHits": artifact_hash_hits,
|
|
"artifacts": len(artifact_signatures),
|
|
"directLinkDependencies": len(link_dependencies),
|
|
"dependencyDirty": dependency_dirty,
|
|
},
|
|
"totalSeconds": round(sum(float(step["seconds"]) for step in steps), 3),
|
|
"steps": steps,
|
|
}
|
|
save(payload)
|
|
print(json.dumps({"status": status, "batchId": batch.get("batchId"), "buildRequired": build_required, "buildReuse": payload["buildReuse"], "totalSeconds": payload["totalSeconds"], "steps": [{"name": step["name"], "seconds": step["seconds"], "exitCode": step["exitCode"]} for step in steps]}, ensure_ascii=False))
|
|
return 0 if status in {"pass", "implementation_ready", "implementation_partial", "implementation_pending_build"} else int(steps[-1]["exitCode"] or 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|