"""Classify historical command failures and reconcile them with business work.""" from __future__ import annotations import argparse import hashlib import json import re import shlex import tempfile from collections import Counter from datetime import datetime, timezone from pathlib import Path from scheduler_progress import record_module ROOT = Path(__file__).resolve().parents[1] SOURCE = ROOT / "docs/current/freecad-web-scheduler-failures.json" OUTPUT = ROOT / "docs/current/failure-reconciliation.json" REPORT = ROOT / "docs/current/FAILURE_RECONCILIATION.md" MODULES = ROOT / "web/public/modules/manifest.json" FINAL_STATES = {"resolved", "superseded", "not_applicable"} def read(path: Path) -> dict: try: value = json.loads(path.read_text(encoding="utf-8")) return value if isinstance(value, dict) else {} except (OSError, ValueError): return {} def write(path: Path, value: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) # Concurrent writers must not rename or overwrite each other's staging file. with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", delete=False) as stream: temporary = Path(stream.name) try: stream.write(json.dumps(value, ensure_ascii=False, indent=2) + "\n") stream.close() temporary.replace(path) finally: temporary.unlink(missing_ok=True) def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else "missing" def entries(payload: dict) -> dict[str, dict]: values = payload.get("failures", {}) if isinstance(values, dict): return {str(key): row for key, row in values.items() if isinstance(row, dict)} return {str(row["key"]): row for row in values if isinstance(row, dict) and row.get("key")} if isinstance(values, list) else {} def occurrence(entry: dict) -> str: value = {key: entry.get(key) for key in ("command", "originalCommand", "count", "lastSeenAt", "lastExitCode", "firstError", "lastError")} return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() def command_tokens(command: str) -> list[str]: try: tokens = shlex.split(command) if tokens and Path(tokens[0]).name in {"bash", "sh"}: for flag in ("-lc", "-c"): if flag in tokens and tokens.index(flag) + 1 < len(tokens): tokens = shlex.split(tokens[tokens.index(flag) + 1]) break if len(tokens) >= 3 and tokens[0] == "cd" and tokens[1] == str(ROOT) and tokens[2] == "&&": tokens = tokens[3:] return [token.removeprefix(str(ROOT) + "/") if token.startswith(str(ROOT) + "/") else token for token in tokens] except ValueError: return [command] def owners(entry: dict, names: list[str]) -> tuple[list[str], list[str]]: found = set() evidence = [] if entry.get("module") in names: found.add(entry["module"]) evidence.append("recorded execution module") aliases = {name.lower(): name for name in names} aliases["fem"] = "Fem" text = " ".join(str(entry.get(key) or "") for key in ("command", "originalCommand", "replayCommand")) patterns = (r"src/Mod/([A-Za-z]+)/", r"FreeCADWeb([A-Za-z]+)", r"(?:run_p6_|run_)([a-z]+)[_\-]", r"web/src/([a-z]+)[_\-.]") for pattern in patterns: for match in re.finditer(pattern, text): token = match.group(1).lower() matches = [alias for alias in aliases if token.startswith(alias)] if pattern.startswith("FreeCAD") else [token] if token in aliases else [] if matches: found.add(aliases[max(matches, key=len)]) evidence.append(match.group(0)) return sorted(found) or ["Shared"], sorted(set(evidence)) def classify(entry: dict) -> tuple[str, bool, str]: command = str(entry.get("command") or "") error = str(entry.get("lastError") or entry.get("firstError") or "").lower() lower = command.lower() tokens = command_tokens(command) source_search_limit = ( len(tokens) >= 2 and Path(tokens[0]).name in {"python", "python3"} and tokens[1] == "tools/fast_source_search.py" and bool(re.search(r"fast_source_search: more than 5000 source files in .+; cite a narrower --path", error)) ) diagnostic_command = ( "failure_reconciliation.py" in lower or source_search_limit or "sha256sum" in lower or ("rg --files" in lower and "git status" in lower) ) legacy = bool(re.search(r"python_api|full_parity|full-parity|module_batch_gate|tier[1-4]|object_contract|object_module|family_gate", lower)) if entry.get("lastExitCode") in {-2, -15, 130, 143} or any(value in error for value in ("servers are currently overloaded", "concurrency limit exceeded", "stream disconnected")): return "runtime_event", legacy, "runtime_recovery" if any(value in error for value in ("fatal error:", "undefined symbol", "ninja: build stopped", "cannot find -l")) or "cmake --build" in lower: return "build_link", legacy, "with_module" if (re.search(r"\bnode\b.*(?:\s-e\s|--eval\b)", lower, re.S) and "compilesourcetextmodule" in error and re.search(r"\[eval\d*\]:\d+.*?\^\s*syntaxerror:", error, re.S)): return "diagnostic_tool", legacy, "review_or_replace" if any(value in error for value in ("require is not defined", "err_module_not_found", "cannot find module", "native_protocol", "memory access out of bounds", "native dispatcher")): return "worker_runtime", legacy, "with_module" if entry.get("lastExitCode") == 124 or any(value in error for value in ("timed out", "timeout", "execution window")): return "runtime_timeout", legacy, "with_module" if any(value in error for value in ("batch manifest", "context validation", "continuation_state_invalid", "batchid does not match")): return "batch_context", legacy, "review_or_replace" if legacy else "shared_infrastructure" if diagnostic_command: return "diagnostic_tool", legacy, "review_or_replace" tokens = command_tokens(command) if (tokens and Path(tokens[0]).name in {"rg", "grep", "sed", "awk", "cat", "ls", "find", "git", "jq", "wc"} and not any(Path(token).name in {"node", "python", "python3", "cmake", "ninja", "make", "em++"} for token in tokens[1:])): return "diagnostic_tool", legacy, "review_or_replace" if any(value in error for value in ("no such file", "not found", "path mismatch")): return "missing_path_or_artifact", legacy, "with_module" if legacy: return "legacy_validation", True, "map_to_business" if any(value in lower for value in ("prepare_", "rotate_", "sync_web_task", "check_continuation", "validate_")): return "orchestration", False, "shared_infrastructure" if any(value in lower for value in ("smoke", "runtime", "facade", "kernel")): return "business_runtime", False, "with_module" return "needs_review", legacy, "classify_now" INSPECTION_FLAGS = {"--short", "--porcelain", "--branch", "--check", "--stat", "-s", "-b", "-sb", "-uno", "--untracked-files=no"} def guarded_inspection_chain(command: str) -> bool: """Recognize only a rejected Git inspection and inert read-only tails.""" if "\n" in command or "\r" in command: return False try: outer = shlex.split(command) if outer and Path(outer[0]).name in {"bash", "sh"}: flag = next((flag for flag in ("-lc", "-c") if flag in outer), None) if flag is None or outer.index(flag) + 2 != len(outer): return False command = outer[outer.index(flag) + 1] lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|<>()") lexer.whitespace_split = True lexer.commenters = "" tokens = list(lexer) except ValueError: return False if tokens[:3] == ["cd", str(ROOT), "&&"]: tokens = tokens[3:] if any(any(char in token for char in ";<>$`()\n\r") for token in tokens): return False groups = [[]] operators = [] for token in tokens: if token in {"&&", "|"}: operators.append(token) groups.append([]) elif token in {"||", "&"}: return False else: groups[-1].append(token) first = groups[0] if (len(first) < 2 or Path(first[0]).name != "git" or first[1] not in {"status", "diff"} or not all(token in INSPECTION_FLAGS for token in first[2:]) or operators and operators[0] != "&&"): return False for group in groups[1:]: if not group: return False name = Path(group[0]).name if name == "ls": continue if (name == "sed" and len(group) >= 3 and group[1] == "-n" and re.fullmatch(r"\d+(?:,\d+)?p", group[2]) and not any(token.startswith("-") for token in group[3:])): continue if name == "git" and len(group) >= 3 and group[1] in {"status", "diff"}: options = group[2:group.index("--")] if "--" in group else group[2:] if all(token in INSPECTION_FLAGS for token in options): continue return False return True def diagnostic_disposition(entry: dict) -> str | None: error = str(entry.get("lastError") or entry.get("firstError") or "") if (entry.get("lastExitCode") == 64 and "SCHEDULER_GIT_SCOPE_REJECTED" in error and guarded_inspection_chain(str(entry.get("command") or ""))): return "unscoped-git-inspection-rejected" return None def timestamp(value: object) -> float: try: return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() except ValueError: return 0 def declared_business_ids(entry: dict) -> set[str]: declared = set(entry.get("capabilityIds", [])) if declared and all(value.startswith("failure:") for value in declared): tokens = command_tokens(str(entry.get("command", ""))) if (len(tokens) >= 3 and Path(tokens[0]).name in {"python", "python3"} and tokens[1] == "tools/record_runtime_issue.py" and ":" in tokens[2] and not tokens[2].startswith(("-", "failure:"))): # The task's failure IDs describe the review batch, not the # explicitly named business for which issue registration failed. return {tokens[2]} return declared def valid_resolution(key: str, entry: dict, resolution: dict, cache: dict | None = None) -> bool: cache = {} if cache is None else cache def checksum(path: Path) -> str: checksums = cache.setdefault("digests", {}) if path not in checksums: checksums[path] = digest(path) return checksums[path] if resolution.get("disposition") not in FINAL_STATES or resolution.get("failureFingerprint") != occurrence(entry): return False path = (ROOT / str(resolution.get("evidencePath", ""))).resolve() if not path.is_relative_to(ROOT) or not path.is_file() or checksum(path) != resolution.get("evidenceSha256"): return False proofs = cache.setdefault("proofs", {}) if path not in proofs: proofs[path] = read(path) proof = proofs[path] if timestamp(proof.get("executedAt")) <= timestamp(entry.get("lastSeenAt")): return False if proof.get("schema") == "freecad-web-command-success/v1": return (resolution.get("disposition") == "resolved" and proof.get("exitCode") == 0 and proof.get("failureFingerprints", {}).get(key) == occurrence(entry) and command_tokens(str(proof.get("command", ""))) == command_tokens(str(entry.get("command", "")))) if proof.get("schema") == "freecad-web-business-cases/v1": from business_acceptance import accepted_ids, successful_result, digest as business_digest, module_inputs modules, _ = owners(entry, list(read(MODULES).get("modules", {}))) declared_ids = declared_business_ids(entry) matching = {row.get("capabilityId") for row in proof.get("results", []) if key in row.get("resolvedFailureKeys", []) and successful_result(row) and (not declared_ids or row.get("capabilityId") in declared_ids) and ("Shared" in modules or record_module(str(row.get("capabilityId", ""))) in modules)} if "acceptedIds" not in cache: business_digest.cache_clear() module_inputs.cache_clear() cache["acceptedIds"] = accepted_ids() return bool(matching & cache["acceptedIds"]) and proof.get("failureFingerprints", {}).get(key) == occurrence(entry) if proof.get("schema") == "freecad-web-diagnostic-disposition/v1": item = proof.get("failures", {}).get(key, {}) rule = diagnostic_disposition(entry) return bool(rule and item.get("rule") == rule and item.get("failureFingerprint") == occurrence(entry) and resolution.get("disposition") == "not_applicable") if proof.get("schema") == "freecad-web-failure-review/v1": item = proof.get("failures", {}).get(key, {}) return (item.get("failureFingerprint") == occurrence(entry) and item.get("disposition") == resolution.get("disposition") and bool(item.get("reason")) and bool(item.get("sources")) and all((ROOT / source).resolve().is_relative_to(ROOT) and (ROOT / source).is_file() and checksum(ROOT / source) == expected for source, expected in item["sources"].items()) and resolution.get("disposition") in {"superseded", "not_applicable"}) return False def summary_current() -> dict: source = read(SOURCE) classified = read(OUTPUT) raw = entries(source) rows = classified.get("records", {}) cache = {} pending = sum(not valid_resolution(key, row, rows.get(key, {}).get("resolution", {}), cache) for key, row in raw.items()) unclassified = sum(key not in rows or rows[key].get("failureFingerprint") != occurrence(row) for key, row in raw.items()) valid_source = (source.get("schema") == "freecad-web-scheduler-failures/v1" and isinstance(source.get("failures"), (dict, list)) and len(raw) == len(source["failures"]) and classified.get("schema") == "freecad-web-failure-reconciliation/v1") return {"total": len(raw), "pending": pending, "closed": len(raw) - pending, "unclassified": unclassified, "complete": valid_source and pending == 0 and unclassified == 0} def pending_keys(keys: list[str]) -> list[str]: source = entries(read(SOURCE)) records = read(OUTPUT).get("records", {}) cache = {} return [key for key in keys if key not in source or not valid_resolution(key, source[key], records.get(key, {}).get("resolution", {}), cache)] def pending_business_repair(keys: list[str]) -> bool: return any(set(keys) & set(issue.get("failureKeys", [])) and issue.get("status", "open") in {"open", "confirmed", "reopened"} for issue in read(ROOT / "docs/current/runtime-issue-ledger.json").get("issues", [])) def synchronize(*, force: bool = False) -> dict: source = read(SOURCE) if not source.get("schema"): raise ValueError("failure ledger is missing or unreadable") raw = entries(source) previous = read(OUTPUT) rows = previous.get("records", {}) source_hash = digest(SOURCE) generator_hash = digest(Path(__file__)) modules_hash = digest(MODULES) validation_cache = {} if (not force and previous.get("sourceSha256") == source_hash and previous.get("generatorSha256") == generator_hash and previous.get("modulesSha256") == modules_hash and set(rows) == set(raw) and all(row.get("status") == (row.get("resolution", {}).get("disposition") if valid_resolution(key, raw[key], row.get("resolution", {}), validation_cache) else "pending") for key, row in rows.items())): return previous names = list(read(MODULES).get("modules", {})) automatic = {key: {"rule": diagnostic_disposition(entry), "failureFingerprint": occurrence(entry)} for key, entry in raw.items() if diagnostic_disposition(entry) and not valid_resolution(key, entry, rows.get(key, {}).get("resolution", {}), validation_cache)} automatic_path = None if automatic: now = datetime.now(timezone.utc) automatic_path = ROOT / "docs/current/failure-evidence" / f"diagnostic-{now.strftime('%Y%m%dT%H%M%S%f')}.json" write(automatic_path, {"schema": "freecad-web-diagnostic-disposition/v1", "executedAt": now.isoformat(), "failures": automatic, "reason": "The scope guard correctly rejected a standalone read-only Git inspection; no CAD operation was attempted. Original failures and violation counts remain retained."}) for key, entry in raw.items(): old = rows.get(key, {}) fingerprint = occurrence(entry) category, legacy, handling = classify(entry) modules, module_evidence = owners(entry, names) previous_versions = list(old.get("previousOccurrences", [])) if old and old.get("failureFingerprint") != fingerprint: previous_versions.append({"failureFingerprint": old.get("failureFingerprint"), "sourceSnapshot": old.get("sourceSnapshot"), "resolution": old.get("resolution")}) resolution = old.get("resolution", {}) if key in automatic: resolution = {"disposition": "not_applicable", "failureFingerprint": fingerprint, "evidencePath": str(automatic_path.relative_to(ROOT)), "evidenceSha256": digest(automatic_path), "reason": "Read-only inspection rejected by the scope guard; no CAD business failure"} if entry.get("resolutionProof"): candidate = {"disposition": "resolved", "failureFingerprint": fingerprint, "evidencePath": entry["resolutionProof"], "evidenceSha256": digest(ROOT / entry["resolutionProof"]), "reason": "matching command completed successfully after the latest failure"} if valid_resolution(key, entry, candidate, validation_cache): resolution = candidate rows[key] = {"key": key, "failureFingerprint": fingerprint, "category": category, "modules": modules, "moduleEvidence": module_evidence, "legacyValidation": legacy, "handling": handling, "capabilityIds": entry.get("capabilityIds", []), "status": resolution.get("disposition") if valid_resolution(key, entry, resolution, validation_cache) else "pending", "sourceSnapshot": entry, "previousOccurrences": previous_versions, "resolution": resolution, "group": ":".join(["+".join(modules), category])} pending = [rows[key] for key in raw if rows[key]["status"] == "pending"] counts = dict(Counter(row["category"] for row in rows.values())) payload = {"schema": "freecad-web-failure-reconciliation/v1", "sourceSha256": source_hash, "generatorSha256": generator_hash, "modulesSha256": modules_hash, "updatedAt": datetime.now(timezone.utc).isoformat(), "records": rows, "summary": {"total": len(raw), "classified": len(raw), "pending": len(pending), "closed": len(raw) - len(pending), "byCategory": counts, "byHandling": dict(Counter(row["handling"] for row in pending)), "byModule": dict(Counter(module for row in pending for module in row["modules"])), "groups": len({row["group"] for row in pending})}} write(OUTPUT, payload) lines = ["# 失败记录分类与业务对账", "", f"分类 {len(raw)}/{len(raw)};待闭环 {len(pending)};已闭环 {len(raw) - len(pending)}。分类不代表修复,原始记录保留。", "", "处理顺序:当前业务首错立即修复;共享故障优先;历史模块故障随业务推进处理;过期检查经源码与替代证据审查后归档。禁止为清零重跑无关逐 API gate。", "", "| 类别 | 记录数 |", "|---|---:|"] lines.extend(f"| {category} | {count} |" for category, count in sorted(counts.items())) lines.extend(["", "结构化记录见 `failure-reconciliation.json`。同类错误分组仅用于合并工作,不自动认定同根因或批量关闭。", "", "最终完成要求:每条当前失败都有有效的 resolved/superseded/not_applicable 证据;新失败会使旧闭环失效。"]) REPORT.write_text("\n".join(lines) + "\n", encoding="utf-8") return payload def module_context(module: str, capability_ids: set[str], limit: int = 3) -> dict: payload = read(OUTPUT) relevant = [row for row in payload.get("records", {}).values() if row.get("status") == "pending" and module in row.get("modules", [])] relevant.sort(key=lambda row: (not bool(capability_ids & set(row.get("capabilityIds", []))), row.get("legacyValidation", False), -int(row.get("sourceSnapshot", {}).get("count", 0)))) groups = [] seen = set() for row in relevant: if row["group"] in seen: continue seen.add(row["group"]) source = row["sourceSnapshot"] groups.append({"key": row["key"], "failureFingerprint": row["failureFingerprint"], "category": row["category"], "handling": "current_business" if capability_ids & set(row.get("capabilityIds", [])) else row["handling"], "command": str(source.get("command", ""))[:500], "error": str(source.get("lastError") or source.get("firstError") or "")[:700], "relatedCount": sum(other["group"] == row["group"] for other in relevant), "instruction": "Investigate the same failure through the current module business; do not replay the historical API command blindly."}) if len(groups) >= limit: break return {"pendingForModule": len(relevant), "groups": groups, "ledger": str(OUTPUT.relative_to(ROOT))} def record_success(entry: dict, command: str, output: object = "") -> str: now = datetime.now(timezone.utc) key = str(entry["key"]) path = ROOT / "docs/current/failure-evidence" / f"{key}-{now.strftime('%Y%m%dT%H%M%S%f')}.json" write(path, {"schema": "freecad-web-command-success/v1", "executedAt": now.isoformat(), "command": command, "exitCode": 0, "failureFingerprints": {key: occurrence(entry)}, "outputSha256": hashlib.sha256(str(output).encode()).hexdigest(), "output": str(output)[-2000:]}) return str(path.relative_to(ROOT)) def adopt_business_evidence(path: Path) -> int: payload = synchronize() proof = read(path) count = 0 for result in proof.get("results", []): for key in result.get("resolvedFailureKeys", []): row = payload["records"].get(key) if not row: continue resolution = {"disposition": "resolved", "failureFingerprint": row["failureFingerprint"], "evidencePath": str(path.relative_to(ROOT)), "evidenceSha256": digest(path), "reason": f"covered by current business case {result.get('capabilityId')}"} if valid_resolution(key, row["sourceSnapshot"], resolution): row["resolution"] = resolution count += 1 write(OUTPUT, payload) synchronize() return count def review(keys: list[str], disposition: str, reason: str, sources: list[Path]) -> None: payload = synchronize() paths = [path.resolve() for path in sources] if not reason.strip() or not paths or any(not path.is_file() or not path.is_relative_to(ROOT) for path in paths): raise ValueError("review requires a reason and existing project source/evidence files") if any(key not in payload["records"] for key in keys): raise ValueError("unknown failure key") if any(payload["records"][key]["category"] not in {"runtime_event", "diagnostic_tool", "legacy_validation", "batch_context", "orchestration"} for key in keys): raise ValueError("runtime and build failures need a current fix or explicit replacement evidence") now = datetime.now(timezone.utc) path = ROOT / "docs/current/failure-evidence" / f"review-{now.strftime('%Y%m%dT%H%M%S%f')}.json" write(path, {"schema": "freecad-web-failure-review/v1", "executedAt": now.isoformat(), "failures": {key: {"failureFingerprint": payload["records"][key]["failureFingerprint"], "disposition": disposition, "reason": reason, "sources": {str(source.relative_to(ROOT)): digest(source) for source in paths}} for key in keys}}) for key in keys: payload["records"][key]["resolution"] = {"disposition": disposition, "failureFingerprint": payload["records"][key]["failureFingerprint"], "evidencePath": str(path.relative_to(ROOT)), "evidenceSha256": digest(path), "reason": reason} write(OUTPUT, payload) synchronize() def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--check", action="store_true") parser.add_argument("--force", action="store_true", help="recompute classification instead of reusing an unchanged snapshot") parser.add_argument("--module") parser.add_argument("--keys", nargs="+") parser.add_argument("--review", nargs="+") parser.add_argument("--disposition", choices=["superseded", "not_applicable"]) parser.add_argument("--reason") parser.add_argument("--source", type=Path, action="append", default=[]) options = parser.parse_args() if options.review: if not options.disposition or not options.reason: parser.error("--review requires --disposition, --reason and --source") review(options.review, options.disposition, options.reason, options.source) payload = synchronize(force=options.force) print(json.dumps(payload["summary"], ensure_ascii=False)) if options.keys: remaining = pending_keys(options.keys) print(json.dumps({"pendingKeys": remaining}, ensure_ascii=False)) return 1 if options.check and remaining else 0 if options.module: remaining = [row["key"] for row in payload["records"].values() if row["status"] == "pending" and options.module in row["modules"]] print(json.dumps({"module": options.module, "remaining": len(remaining)}, ensure_ascii=False)) return 1 if options.check and remaining else 0 return 1 if options.check and not summary_current()["complete"] else 0 if __name__ == "__main__": raise SystemExit(main())