f3e21c6b98
Accumulated uncommitted work: GHOST Web backend, WebGL/GPU web layer, web workflow engine and protocol, tooling, docs and task records. Large generated evidence, user-testing archives, build snapshots and imported test assets remain untracked. Co-Authored-By: Claude Code <noreply@anthropic.com>
271 lines
14 KiB
Python
271 lines
14 KiB
Python
"""Run bounded validation stages and retain hash-valid successful evidence."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import time
|
|
|
|
from wbv2_step_cache import StepCache
|
|
from wbv2_validation import prepare_validation_command
|
|
|
|
|
|
def _native_build_fingerprint(cwd):
|
|
"""Hash native source/build metadata without workflow-specific test inputs."""
|
|
root = cwd.resolve()
|
|
source = root / "blender-5.2.0" / "source"
|
|
digest = hashlib.sha256()
|
|
if source.is_dir():
|
|
for directory, _, names in os.walk(source):
|
|
directory_path = os.path.abspath(directory)
|
|
for name in sorted(names):
|
|
path = os.path.join(directory_path, name)
|
|
try:
|
|
stat = os.stat(path)
|
|
except OSError:
|
|
continue
|
|
relative = os.path.relpath(path, root)
|
|
digest.update(f"{relative}\0{stat.st_size}\0{stat.st_mtime_ns}\0".encode())
|
|
for relative in (
|
|
".emscripten-web",
|
|
"build_web_blender6/CMakeCache.txt",
|
|
"build_web_blender6/build.ninja",
|
|
"my-tools/build_local_webengine.py",
|
|
"tools/web/emscripten-env.sh",
|
|
"tools/web/sync-web-engine-runtime.mjs",
|
|
):
|
|
path = root / relative
|
|
try:
|
|
stat = path.stat()
|
|
except OSError:
|
|
digest.update(f"{relative}\0missing\0".encode())
|
|
continue
|
|
digest.update(f"{relative}\0{stat.st_size}\0{stat.st_mtime_ns}\0".encode())
|
|
try:
|
|
digest.update(hashlib.sha256(path.read_bytes()).digest())
|
|
except OSError:
|
|
pass
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _shared_build_failure_key(argv, cwd, environment, unset_env, fingerprints):
|
|
"""Identify a native build independently of its owning workflow/job."""
|
|
normalized_environment = []
|
|
for key, value in environment.items():
|
|
# Receipt output is deliberately per-attempt; it must not partition
|
|
# the shared native build failure namespace.
|
|
normalized = "<attempt-receipt-dir>" if key == "WBV2_RECEIPT_DIR" else str(value)
|
|
normalized_environment.append((str(key), normalized))
|
|
payload = {
|
|
"argv": list(argv),
|
|
"cwd": str(cwd),
|
|
"environment": sorted(normalized_environment),
|
|
"unsetEnv": sorted(str(value) for value in unset_env),
|
|
"nativeBuildFingerprint": _native_build_fingerprint(cwd),
|
|
}
|
|
return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
|
|
|
|
|
|
def _shared_build_failure_path(directory, key):
|
|
path = directory / "shared-build-failures"
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
return path / f"{key}.json"
|
|
|
|
|
|
def _load_shared_build_failure(directory, key):
|
|
path = _shared_build_failure_path(directory, key)
|
|
try:
|
|
record = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError, TypeError):
|
|
return None
|
|
if record.get("schemaVersion") != 1 or record.get("key") != key:
|
|
return None
|
|
if record.get("classification") not in {"command_failed", "timeout", "log_limit"}:
|
|
return None
|
|
return record
|
|
|
|
|
|
def _store_shared_build_failure(directory, key, receipt):
|
|
path = _shared_build_failure_path(directory, key)
|
|
record = {
|
|
"schemaVersion": 1,
|
|
"key": key,
|
|
"classification": "timeout" if receipt["timedOut"] else "log_limit" if receipt["logLimit"] else "command_failed",
|
|
"exitCode": receipt["exitCode"],
|
|
"seconds": receipt["durationSeconds"],
|
|
"log": receipt["log"],
|
|
"logSha256": receipt.get("logSha256"),
|
|
"recordedAt": time.time(),
|
|
}
|
|
temporary = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="w", encoding="utf-8", dir=path.parent, delete=False,
|
|
prefix=f".{key}.", suffix=".tmp"
|
|
) as stream:
|
|
json.dump(record, stream, sort_keys=True)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
temporary = stream.name
|
|
os.replace(temporary, path)
|
|
except OSError:
|
|
if temporary:
|
|
try:
|
|
os.unlink(temporary)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _shared_failure_receipt(check, record):
|
|
return {
|
|
"id": check["id"],
|
|
"kinds": check["kinds"],
|
|
"exitCode": record["exitCode"],
|
|
"classification": record["classification"],
|
|
"durationSeconds": 0.0,
|
|
"log": record["log"],
|
|
"logSha256": record.get("logSha256"),
|
|
"interrupted": False,
|
|
"timedOut": record["classification"] == "timeout",
|
|
"logLimit": record["classification"] == "log_limit",
|
|
"sharedFailureKey": record["key"],
|
|
"sharedFailureRecordedAt": record.get("recordedAt"),
|
|
}
|
|
|
|
|
|
def execute_checks(runner, result, run_dir, inputs, supervisor_env):
|
|
from wbv2_autopilot import contained, read_json, sha, write_json
|
|
|
|
cache = StepCache(runner.root, runner.directory / "step-cache")
|
|
receipts, failures, outcomes = [], [], {}
|
|
original_cases = result["cases"]
|
|
case_bindings = {check["id"]: [case for case in original_cases if case["checkId"] == check["id"]] for check in result["checks"]}
|
|
cache_path = runner.directory / "step-cache"
|
|
cache_path.mkdir(parents=True, exist_ok=True)
|
|
for check in result["checks"]:
|
|
dependencies = check.get("dependsOn", [])
|
|
failed_dependencies = [identifier for identifier in dependencies if outcomes.get(identifier) != "passed"]
|
|
if failed_dependencies:
|
|
outcomes[check["id"]] = "blocked"
|
|
runner.event("check.blocked", checkId=check["id"], dependencies=failed_dependencies)
|
|
continue
|
|
if runner.stop_requested or (runner.directory / "pause.request").exists():
|
|
raise InterruptedError("local validation paused; completed steps preserved")
|
|
argv, cwd, check_env, unset_env = prepare_validation_command(runner.root, check["argv"], check["cwd"], max_build_jobs=runner.policy["maxBuildJobs"])
|
|
cache_argv = list(argv)
|
|
if any("playwright" in argument and argument.endswith(".js") for argument in argv) and "test" in argv:
|
|
output = str(run_dir / "playwright" / check["id"])
|
|
argv = [*argv, "--output", output]
|
|
cache_argv = [*cache_argv, "--output", "<isolated-playwright-output>"]
|
|
environment = {**check_env, **supervisor_env}
|
|
cache_environment = dict(environment)
|
|
cache_environment["WBV2_RECEIPT_DIR"] = "<fresh-attempt-directory>"
|
|
inherited = {key: value for key, value in os.environ.items() if key not in unset_env and key not in environment}
|
|
cache_environment["inheritedEnvironmentSha256"] = hashlib.sha256(json.dumps(inherited, sort_keys=True).encode()).hexdigest()
|
|
# A recipe can narrow a stage's dependencies explicitly. The default is
|
|
# conservative and covers all declared workflow inputs.
|
|
selected = set(check.get("cacheInputs", list(inputs))) | set(result.get("mandatoryCacheInputs", []))
|
|
fingerprints = {path: sha(contained(runner.root, path)) for path in selected}
|
|
for path in result["artifacts"]:
|
|
if contained(runner.root, path).is_file():
|
|
fingerprints[path] = sha(contained(runner.root, path))
|
|
fingerprints["my-tools/wbv2_validation.py"] = sha(runner.root / "my-tools/wbv2_validation.py")
|
|
for path in ("my-tools/wbv2_local_execution.py", "my-tools/wbv2_step_cache.py"):
|
|
if (runner.root / path).is_file():
|
|
fingerprints[path] = sha(runner.root / path)
|
|
bound_cases = case_bindings[check["id"]]
|
|
cache_environment["caseContractSha256"] = hashlib.sha256(json.dumps([{key: value for key, value in case.items() if key != "receipt"} for case in bound_cases], sort_keys=True).encode()).hexdigest()
|
|
cache_environment["kinds"] = json.dumps(sorted(check["kinds"]))
|
|
cache_environment["timeoutSeconds"] = str(min(check.get("timeoutSeconds", runner.policy["checkTimeoutSeconds"]), runner.policy["checkTimeoutSeconds"]))
|
|
cache_environment["maxLogBytes"] = str(runner.policy["maxLogBytes"])
|
|
key = cache.key(runner.state.get("activeJob") or "validation", check["id"], cache_argv, str(cwd.relative_to(runner.root)) or ".", cache_environment, unset_env, fingerprints)
|
|
# CMake/Ninja own the native dependency graph; always ask the incremental
|
|
# builder before reusing runtime evidence for its outputs.
|
|
reusable = "source_build" not in check["kinds"]
|
|
with runner.timed("step_cache_lookup"):
|
|
receipt = cache.load(key) if reusable else None
|
|
if receipt is not None:
|
|
cached_cases = receipt.get("caseBindings", [])
|
|
if {case["id"] for case in cached_cases} != {case["id"] for case in bound_cases}:
|
|
receipt = None
|
|
if receipt is not None:
|
|
replacements = {case["id"]: case for case in receipt.get("caseBindings", [])}
|
|
result["cases"] = [replacements.get(case["id"], case) for case in result["cases"]]
|
|
runner.event("check.reused", checkId=check["id"], originalLog=receipt["log"], avoidedSeconds=receipt["durationSeconds"], provenance=receipt["reusedFrom"])
|
|
else:
|
|
shared_failure = None
|
|
if "source_build" in check["kinds"]:
|
|
shared_failure = _load_shared_build_failure(
|
|
cache_path,
|
|
_shared_build_failure_key(argv, cwd, environment, unset_env, fingerprints),
|
|
)
|
|
if shared_failure is not None:
|
|
receipt = _shared_failure_receipt(check, shared_failure)
|
|
failure = {
|
|
"checkId": check["id"],
|
|
"exitCode": receipt["exitCode"],
|
|
"classification": "shared_failure",
|
|
"seconds": 0.0,
|
|
"log": receipt["log"],
|
|
"sharedFailureKey": receipt["sharedFailureKey"],
|
|
"originalClassification": shared_failure["classification"],
|
|
}
|
|
failures.append(failure)
|
|
outcomes[check["id"]] = "failed"
|
|
runner.event("check.shared_failure", **failure)
|
|
receipts.append(receipt)
|
|
write_json(run_dir / "step-checkpoint.json", {"schemaVersion": 1, "checks": receipts, "outcomes": outcomes, "failures": failures})
|
|
if not check.get("continueOnFailure", False):
|
|
break
|
|
continue
|
|
timeout = min(check.get("timeoutSeconds", runner.policy["checkTimeoutSeconds"]), runner.policy["checkTimeoutSeconds"])
|
|
receipt = runner.run_process(argv, cwd, run_dir / f"check-{check['id']}.log", timeout, env=environment, unset_env=unset_env, phase="validation:" + check["id"])
|
|
receipt = {**receipt, "id": check["id"], "kinds": check["kinds"]}
|
|
if receipt["interrupted"]:
|
|
raise InterruptedError("local check interrupted; successful checkpoints preserved")
|
|
if receipt["exitCode"] != 0 or receipt["timedOut"] or receipt["logLimit"]:
|
|
failure = {"checkId": check["id"], "exitCode": receipt["exitCode"], "classification": "timeout" if receipt["timedOut"] else "log_limit" if receipt["logLimit"] else "command_failed", "seconds": receipt["durationSeconds"], "log": receipt["log"]}
|
|
failures.append(failure)
|
|
outcomes[check["id"]] = "failed"
|
|
runner.event("check.failed", **failure)
|
|
if "source_build" in check["kinds"]:
|
|
_store_shared_build_failure(
|
|
cache_path,
|
|
_shared_build_failure_key(argv, cwd, environment, unset_env, fingerprints),
|
|
receipt,
|
|
)
|
|
if not check.get("continueOnFailure", False):
|
|
break
|
|
continue
|
|
if any(sha(contained(runner.root, path)) != digest for path, digest in fingerprints.items() if path in inputs):
|
|
raise InterruptedError("check input changed during execution")
|
|
artifacts = []
|
|
for case in bound_cases:
|
|
path = contained(runner.root, case["receipt"])
|
|
if not path.is_file():
|
|
raise ValueError(f"missing runtime receipt: case={case['id']} check={check['id']} path={case['receipt']}; repair the producing test before retry")
|
|
data = read_json(path)
|
|
if data.get("status") != "PASS":
|
|
raise ValueError("case receipt missing PASS status")
|
|
if (data.get("caseId") != case["id"] or data.get("kind") != case["kind"]
|
|
or set(data.get("unitIds", [])) != set(case["unitIds"])):
|
|
raise ValueError(f"runtime receipt binding mismatch before cache: case={case['id']} check={check['id']} actualCaseId={data.get('caseId')!r} expectedKind={case['kind']} actualKind={data.get('kind')!r} missingUnits={sorted(set(case['unitIds']) - set(data.get('unitIds', [])))} unexpectedUnits={sorted(set(data.get('unitIds', [])) - set(case['unitIds']))}")
|
|
artifacts.extend(data.get("artifacts", []))
|
|
artifacts.append({"path": case["receipt"], "sha256": sha(path)})
|
|
artifacts.extend({"path": path, "sha256": sha(contained(runner.root, path))} for path in result["artifacts"])
|
|
unique_artifacts = {}
|
|
for artifact in artifacts:
|
|
prior = unique_artifacts.setdefault(artifact["path"], artifact)
|
|
if prior["sha256"] != artifact["sha256"]:
|
|
raise ValueError("inconsistent artifact bindings across cases")
|
|
receipt["caseBindings"] = bound_cases
|
|
if reusable:
|
|
with runner.timed("step_cache_store"):
|
|
cache.store(key, receipt, list(unique_artifacts.values()))
|
|
receipts.append(receipt)
|
|
outcomes[check["id"]] = "passed"
|
|
write_json(run_dir / "step-checkpoint.json", {"schemaVersion": 1, "checks": receipts, "outcomes": outcomes, "failures": failures})
|
|
write_json(run_dir / "step-checkpoint.json", {"schemaVersion": 1, "checks": receipts, "outcomes": outcomes, "failures": failures})
|
|
if failures:
|
|
raise ValueError("local checks failed: " + ", ".join(failure["checkId"] + "=" + failure["classification"] for failure in failures))
|
|
return receipts
|