"""Run bounded validation stages and retain hash-valid successful evidence.""" import hashlib import json import os import tempfile import time from wbv2_step_cache import StepCache from wbv2_validation import prepare_validation_command def _native_build_fingerprint(cwd): """Hash native source/build metadata without workflow-specific test inputs.""" root = cwd.resolve() source = root / "blender-5.2.0" / "source" digest = hashlib.sha256() if source.is_dir(): for directory, _, names in os.walk(source): directory_path = os.path.abspath(directory) for name in sorted(names): path = os.path.join(directory_path, name) try: stat = os.stat(path) except OSError: continue relative = os.path.relpath(path, root) digest.update(f"{relative}\0{stat.st_size}\0{stat.st_mtime_ns}\0".encode()) for relative in ( ".emscripten-web", "build_web_blender6/CMakeCache.txt", "build_web_blender6/build.ninja", "my-tools/build_local_webengine.py", "tools/web/emscripten-env.sh", "tools/web/sync-web-engine-runtime.mjs", ): path = root / relative try: stat = path.stat() except OSError: digest.update(f"{relative}\0missing\0".encode()) continue digest.update(f"{relative}\0{stat.st_size}\0{stat.st_mtime_ns}\0".encode()) try: digest.update(hashlib.sha256(path.read_bytes()).digest()) except OSError: pass return digest.hexdigest() def _shared_build_failure_key(argv, cwd, environment, unset_env, fingerprints): """Identify a native build independently of its owning workflow/job.""" normalized_environment = [] for key, value in environment.items(): # Receipt output is deliberately per-attempt; it must not partition # the shared native build failure namespace. normalized = "" if key == "WBV2_RECEIPT_DIR" else str(value) normalized_environment.append((str(key), normalized)) payload = { "argv": list(argv), "cwd": str(cwd), "environment": sorted(normalized_environment), "unsetEnv": sorted(str(value) for value in unset_env), "nativeBuildFingerprint": _native_build_fingerprint(cwd), } return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() def _shared_build_failure_path(directory, key): path = directory / "shared-build-failures" path.mkdir(parents=True, exist_ok=True) return path / f"{key}.json" def _load_shared_build_failure(directory, key): path = _shared_build_failure_path(directory, key) try: record = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError, TypeError): return None if record.get("schemaVersion") != 1 or record.get("key") != key: return None if record.get("classification") not in {"command_failed", "timeout", "log_limit"}: return None return record def _store_shared_build_failure(directory, key, receipt): path = _shared_build_failure_path(directory, key) record = { "schemaVersion": 1, "key": key, "classification": "timeout" if receipt["timedOut"] else "log_limit" if receipt["logLimit"] else "command_failed", "exitCode": receipt["exitCode"], "seconds": receipt["durationSeconds"], "log": receipt["log"], "logSha256": receipt.get("logSha256"), "recordedAt": time.time(), } temporary = None try: with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", dir=path.parent, delete=False, prefix=f".{key}.", suffix=".tmp" ) as stream: json.dump(record, stream, sort_keys=True) stream.flush() os.fsync(stream.fileno()) temporary = stream.name os.replace(temporary, path) except OSError: if temporary: try: os.unlink(temporary) except OSError: pass def _shared_failure_receipt(check, record): return { "id": check["id"], "kinds": check["kinds"], "exitCode": record["exitCode"], "classification": record["classification"], "durationSeconds": 0.0, "log": record["log"], "logSha256": record.get("logSha256"), "interrupted": False, "timedOut": record["classification"] == "timeout", "logLimit": record["classification"] == "log_limit", "sharedFailureKey": record["key"], "sharedFailureRecordedAt": record.get("recordedAt"), } def execute_checks(runner, result, run_dir, inputs, supervisor_env): from wbv2_autopilot import contained, read_json, sha, write_json cache = StepCache(runner.root, runner.directory / "step-cache") receipts, failures, outcomes = [], [], {} original_cases = result["cases"] case_bindings = {check["id"]: [case for case in original_cases if case["checkId"] == check["id"]] for check in result["checks"]} cache_path = runner.directory / "step-cache" cache_path.mkdir(parents=True, exist_ok=True) for check in result["checks"]: dependencies = check.get("dependsOn", []) failed_dependencies = [identifier for identifier in dependencies if outcomes.get(identifier) != "passed"] if failed_dependencies: outcomes[check["id"]] = "blocked" runner.event("check.blocked", checkId=check["id"], dependencies=failed_dependencies) continue if runner.stop_requested or (runner.directory / "pause.request").exists(): raise InterruptedError("local validation paused; completed steps preserved") argv, cwd, check_env, unset_env = prepare_validation_command(runner.root, check["argv"], check["cwd"], max_build_jobs=runner.policy["maxBuildJobs"]) cache_argv = list(argv) if any("playwright" in argument and argument.endswith(".js") for argument in argv) and "test" in argv: output = str(run_dir / "playwright" / check["id"]) argv = [*argv, "--output", output] cache_argv = [*cache_argv, "--output", ""] environment = {**check_env, **supervisor_env} cache_environment = dict(environment) cache_environment["WBV2_RECEIPT_DIR"] = "" inherited = {key: value for key, value in os.environ.items() if key not in unset_env and key not in environment} cache_environment["inheritedEnvironmentSha256"] = hashlib.sha256(json.dumps(inherited, sort_keys=True).encode()).hexdigest() # A recipe can narrow a stage's dependencies explicitly. The default is # conservative and covers all declared workflow inputs. selected = set(check.get("cacheInputs", list(inputs))) | set(result.get("mandatoryCacheInputs", [])) fingerprints = {path: sha(contained(runner.root, path)) for path in selected} for path in result["artifacts"]: if contained(runner.root, path).is_file(): fingerprints[path] = sha(contained(runner.root, path)) fingerprints["my-tools/wbv2_validation.py"] = sha(runner.root / "my-tools/wbv2_validation.py") for path in ("my-tools/wbv2_local_execution.py", "my-tools/wbv2_step_cache.py"): if (runner.root / path).is_file(): fingerprints[path] = sha(runner.root / path) bound_cases = case_bindings[check["id"]] cache_environment["caseContractSha256"] = hashlib.sha256(json.dumps([{key: value for key, value in case.items() if key != "receipt"} for case in bound_cases], sort_keys=True).encode()).hexdigest() cache_environment["kinds"] = json.dumps(sorted(check["kinds"])) cache_environment["timeoutSeconds"] = str(min(check.get("timeoutSeconds", runner.policy["checkTimeoutSeconds"]), runner.policy["checkTimeoutSeconds"])) cache_environment["maxLogBytes"] = str(runner.policy["maxLogBytes"]) key = cache.key(runner.state.get("activeJob") or "validation", check["id"], cache_argv, str(cwd.relative_to(runner.root)) or ".", cache_environment, unset_env, fingerprints) # CMake/Ninja own the native dependency graph; always ask the incremental # builder before reusing runtime evidence for its outputs. reusable = "source_build" not in check["kinds"] with runner.timed("step_cache_lookup"): receipt = cache.load(key) if reusable else None if receipt is not None: cached_cases = receipt.get("caseBindings", []) if {case["id"] for case in cached_cases} != {case["id"] for case in bound_cases}: receipt = None if receipt is not None: replacements = {case["id"]: case for case in receipt.get("caseBindings", [])} result["cases"] = [replacements.get(case["id"], case) for case in result["cases"]] runner.event("check.reused", checkId=check["id"], originalLog=receipt["log"], avoidedSeconds=receipt["durationSeconds"], provenance=receipt["reusedFrom"]) else: shared_failure = None if "source_build" in check["kinds"]: shared_failure = _load_shared_build_failure( cache_path, _shared_build_failure_key(argv, cwd, environment, unset_env, fingerprints), ) if shared_failure is not None: receipt = _shared_failure_receipt(check, shared_failure) failure = { "checkId": check["id"], "exitCode": receipt["exitCode"], "classification": "shared_failure", "seconds": 0.0, "log": receipt["log"], "sharedFailureKey": receipt["sharedFailureKey"], "originalClassification": shared_failure["classification"], } failures.append(failure) outcomes[check["id"]] = "failed" runner.event("check.shared_failure", **failure) receipts.append(receipt) write_json(run_dir / "step-checkpoint.json", {"schemaVersion": 1, "checks": receipts, "outcomes": outcomes, "failures": failures}) if not check.get("continueOnFailure", False): break continue timeout = min(check.get("timeoutSeconds", runner.policy["checkTimeoutSeconds"]), runner.policy["checkTimeoutSeconds"]) receipt = runner.run_process(argv, cwd, run_dir / f"check-{check['id']}.log", timeout, env=environment, unset_env=unset_env, phase="validation:" + check["id"]) receipt = {**receipt, "id": check["id"], "kinds": check["kinds"]} if receipt["interrupted"]: raise InterruptedError("local check interrupted; successful checkpoints preserved") if receipt["exitCode"] != 0 or receipt["timedOut"] or receipt["logLimit"]: failure = {"checkId": check["id"], "exitCode": receipt["exitCode"], "classification": "timeout" if receipt["timedOut"] else "log_limit" if receipt["logLimit"] else "command_failed", "seconds": receipt["durationSeconds"], "log": receipt["log"]} failures.append(failure) outcomes[check["id"]] = "failed" runner.event("check.failed", **failure) if "source_build" in check["kinds"]: _store_shared_build_failure( cache_path, _shared_build_failure_key(argv, cwd, environment, unset_env, fingerprints), receipt, ) if not check.get("continueOnFailure", False): break continue if any(sha(contained(runner.root, path)) != digest for path, digest in fingerprints.items() if path in inputs): raise InterruptedError("check input changed during execution") artifacts = [] for case in bound_cases: path = contained(runner.root, case["receipt"]) if not path.is_file(): raise ValueError(f"missing runtime receipt: case={case['id']} check={check['id']} path={case['receipt']}; repair the producing test before retry") data = read_json(path) if data.get("status") != "PASS": raise ValueError("case receipt missing PASS status") if (data.get("caseId") != case["id"] or data.get("kind") != case["kind"] or set(data.get("unitIds", [])) != set(case["unitIds"])): raise ValueError(f"runtime receipt binding mismatch before cache: case={case['id']} check={check['id']} actualCaseId={data.get('caseId')!r} expectedKind={case['kind']} actualKind={data.get('kind')!r} missingUnits={sorted(set(case['unitIds']) - set(data.get('unitIds', [])))} unexpectedUnits={sorted(set(data.get('unitIds', [])) - set(case['unitIds']))}") artifacts.extend(data.get("artifacts", [])) artifacts.append({"path": case["receipt"], "sha256": sha(path)}) artifacts.extend({"path": path, "sha256": sha(contained(runner.root, path))} for path in result["artifacts"]) unique_artifacts = {} for artifact in artifacts: prior = unique_artifacts.setdefault(artifact["path"], artifact) if prior["sha256"] != artifact["sha256"]: raise ValueError("inconsistent artifact bindings across cases") receipt["caseBindings"] = bound_cases if reusable: with runner.timed("step_cache_store"): cache.store(key, receipt, list(unique_artifacts.values())) receipts.append(receipt) outcomes[check["id"]] = "passed" write_json(run_dir / "step-checkpoint.json", {"schemaVersion": 1, "checks": receipts, "outcomes": outcomes, "failures": failures}) write_json(run_dir / "step-checkpoint.json", {"schemaVersion": 1, "checks": receipts, "outcomes": outcomes, "failures": failures}) if failures: raise ValueError("local checks failed: " + ", ".join(failure["checkId"] + "=" + failure["classification"] for failure in failures)) return receipts