329 lines
18 KiB
Python
329 lines
18 KiB
Python
"""Persist business progress and prevent repeated selection of stalled work."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROGRESS_PATH = ROOT / "docs/current/scheduler-progress.json"
|
|
RECOVERY_PATH = ROOT / "docs/current/scheduler-recovery-request.json"
|
|
NO_PROGRESS_LIMIT = max(2, int(os.environ.get("FREECAD_NO_PROGRESS_LIMIT", "3")))
|
|
COOLDOWN_SECONDS = max(30, int(os.environ.get("FREECAD_STALLED_RETRY_SECONDS", "300")))
|
|
MODULE_NO_PROGRESS_LIMIT = max(3, int(os.environ.get("FREECAD_MODULE_NO_PROGRESS_LIMIT", "3")))
|
|
# Feature implementation turns are expensive source/build sessions. Move a
|
|
# stalled module aside after two ineffective turns so another executable
|
|
# module can make progress while the stalled IDs remain explicitly deferred.
|
|
FEATURE_NO_PROGRESS_LIMIT = max(2, int(os.environ.get("FREECAD_FEATURE_NO_PROGRESS_LIMIT", "2")))
|
|
# A stalled module is deferred so another batch can run, but ten minutes of
|
|
# empty scheduler time is too expensive for implementation-first work. Keep a
|
|
# bounded cooldown and let the next planner re-evaluate the unchanged records.
|
|
MODULE_COOLDOWN_SECONDS = max(60, int(os.environ.get("FREECAD_MODULE_RETRY_SECONDS", "180")))
|
|
PROVIDER_BACKOFF_MAX_SECONDS = max(30, int(os.environ.get("FREECAD_PROVIDER_BACKOFF_MAX_SECONDS", "600")))
|
|
|
|
|
|
def provider_retry_delay(failures: int, retry_delay: float) -> float:
|
|
short_retry = min(60, max(1, retry_delay) * 2 ** min(max(0, failures - 1), 4))
|
|
return min(PROVIDER_BACKOFF_MAX_SECONDS, short_retry * 2 ** min(max(0, failures - 3), 4))
|
|
|
|
|
|
def advance_provider(provider: dict, failed: bool, responded: bool, now: float, retry_delay: float, reason: str) -> None:
|
|
if failed:
|
|
failures = int(provider.get("consecutiveFailures", 0)) + 1
|
|
provider.update(consecutiveFailures=failures, retryAt=now + provider_retry_delay(failures, retry_delay), reason=reason)
|
|
elif responded:
|
|
provider.update(consecutiveFailures=0, retryAt=0, reason="")
|
|
|
|
|
|
def record_provider_attempt(*, failed: bool, responded: bool, reason: str = "", retry_delay: float = 20,
|
|
now: float | None = None) -> dict:
|
|
"""Share maintenance request health without inventing a business outcome."""
|
|
state = read(PROGRESS_PATH)
|
|
provider = state.setdefault("provider", {})
|
|
if failed or responded:
|
|
advance_provider(provider, failed, responded, time.time() if now is None else now, retry_delay, reason)
|
|
save(state)
|
|
return provider
|
|
|
|
|
|
def read(path: Path) -> dict:
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
return value if isinstance(value, dict) else {}
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
|
|
def save(value: dict) -> None:
|
|
PROGRESS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = PROGRESS_PATH.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(PROGRESS_PATH)
|
|
|
|
|
|
def selected_ids(batch: dict) -> set[str]:
|
|
return {str(row[0]) for row in batch.get("records", []) if isinstance(row, list) and row}
|
|
|
|
|
|
def ineffective_feature_seconds(batch: dict, history: list[dict]) -> float:
|
|
"""Keep work history across replanning, but stop at relevant real progress."""
|
|
ids = selected_ids(batch)
|
|
if not ids or not batch.get("batchId"):
|
|
return 0.0
|
|
seconds = 0.0
|
|
seen = set()
|
|
for row in reversed(history[-100:]):
|
|
if ids.intersection(row.get("progressedIds", [])):
|
|
break
|
|
if row.get("batchId") != batch["batchId"] or row.get("tier") != "feature-workflow":
|
|
continue
|
|
if row.get("outcome") == "progress":
|
|
break
|
|
if row.get("outcome") not in {"no_progress", "source_edits_pending_outcome"}:
|
|
continue
|
|
cycle = row.get("cycle")
|
|
if cycle is not None and cycle in seen:
|
|
continue
|
|
seen.add(cycle)
|
|
seconds += max(0.0, float(row.get("seconds", 0) or 0))
|
|
return round(seconds, 3)
|
|
|
|
|
|
def snapshot(batch: dict) -> dict:
|
|
ids = selected_ids(batch)
|
|
features = {row["id"]: row for row in read(ROOT / "docs/current/feature-capability-registry.json").get("records", []) if row.get("id") in ids}
|
|
receipts = read(ROOT / "docs/current/implementation-first-registry.json").get("records", {})
|
|
acceptance = read(ROOT / "docs/current/business-acceptance.json").get("records", {})
|
|
issues = read(ROOT / "docs/current/runtime-issue-ledger.json").get("issues", [])
|
|
failure_rows = read(ROOT / "docs/current/failure-reconciliation.json").get("records", {}) if any(record_id.startswith("failure:") for record_id in ids) else {}
|
|
return {record_id: {
|
|
"route": features.get(record_id, {}).get("implementationStatus"),
|
|
"receipt": {key: receipts.get(record_id, {}).get(key) for key in ("status", "batchId")},
|
|
"acceptance": {key: acceptance.get(record_id, {}).get(key) for key in ("status", "fingerprint", "evidencePath")},
|
|
"issues": sorted((row.get("id", ""), row.get("status", "")) for row in issues if row.get("capabilityId") == record_id),
|
|
"failureResolution": failure_rows.get(record_id.rsplit(":", 1)[-1], {}).get("status") if record_id.startswith("failure:") else None,
|
|
} for record_id in ids}
|
|
|
|
|
|
def record_module(record_id: str) -> str:
|
|
parts = record_id.split(":")
|
|
return "Web" if parts[0] == "product" else parts[1] if parts[0] in {"business-format", "failure"} and len(parts) > 1 else parts[0]
|
|
|
|
|
|
def module_states(state: dict) -> dict:
|
|
modules = dict(state.get("modules", {}))
|
|
# Existing installations already have outcome history. Recover its last
|
|
# module streak so a restart cannot grant stalled work another full run.
|
|
for module in {row.get("module") for row in state.get("history", []) if row.get("module")} - modules.keys():
|
|
streak = 0
|
|
last = {}
|
|
for event in state.get("history", []):
|
|
if event.get("module") != module or event.get("outcome") == "provider_wait" or event.get("tier") == "failure-reconciliation":
|
|
continue
|
|
if event.get("evidenceRefreshOnly"):
|
|
continue
|
|
streak = 0 if event.get("outcome") == "progress" else streak + 1
|
|
last = event
|
|
modules[module] = {"consecutiveNoProgress": streak, "lastAttemptAt": last.get("at", 0),
|
|
"lastReason": last.get("reason", ""), "recoveryLevel": int(streak >= MODULE_NO_PROGRESS_LIMIT),
|
|
"retryAt": float(last.get("at", 0)) + MODULE_COOLDOWN_SECONDS if streak >= MODULE_NO_PROGRESS_LIMIT else 0}
|
|
return modules
|
|
|
|
|
|
def restore_module_recovery(now: float | None = None) -> list[str]:
|
|
"""Migrate an old outcome ledger once, preserving a diagnosed stall."""
|
|
now = time.time() if now is None else now
|
|
state = read(PROGRESS_PATH)
|
|
stored = state.get("modules", {})
|
|
modules = module_states(state)
|
|
restored = []
|
|
for module in modules.keys() - stored.keys():
|
|
entry = modules[module]
|
|
if int(entry.get("consecutiveNoProgress", 0)) >= MODULE_NO_PROGRESS_LIMIT:
|
|
entry.update(retryAt=now + MODULE_COOLDOWN_SECONDS, lastAttemptAt=now,
|
|
consecutiveNoProgress=0, recoveryAction="repair shared integration using the expanded work packet")
|
|
restored.append(module)
|
|
if modules.keys() != stored.keys():
|
|
state.update(schema="freecad-web-scheduler-progress/v1", modules=modules, updatedAt=now)
|
|
save(state)
|
|
return restored
|
|
|
|
|
|
def cooling_modules(now: float | None = None, state: dict | None = None) -> dict:
|
|
now = time.time() if now is None else now
|
|
return {module: row for module, row in module_states(read(PROGRESS_PATH) if state is None else state).items()
|
|
if float(row.get("retryAt", 0)) > now}
|
|
|
|
|
|
def cooling_ids(now: float | None = None, candidates: set[str] | None = None) -> set[str]:
|
|
now = time.time() if now is None else now
|
|
state = read(PROGRESS_PATH)
|
|
modules = cooling_modules(now, state)
|
|
records = state.get("records", {})
|
|
ids = set(records) | (candidates or set())
|
|
return {record_id for record_id in ids if float(records.get(record_id, {}).get("retryAt", 0)) > now
|
|
or record_module(record_id) in modules}
|
|
|
|
|
|
def retry_details(ids: set[str], now: float | None = None) -> dict:
|
|
now = time.time() if now is None else now
|
|
state = read(PROGRESS_PATH)
|
|
records = state.get("records", {})
|
|
modules = cooling_modules(now, state)
|
|
retry_times = {record_id: max(float(records.get(record_id, {}).get("retryAt", 0)),
|
|
float(modules.get(record_module(record_id), {}).get("retryAt", 0))) for record_id in ids}
|
|
cooling = [record_id for record_id, retry_at in retry_times.items() if retry_at > now]
|
|
if not cooling:
|
|
return {}
|
|
return {"status": "waiting_for_retry", "reason": "business records are cooling down after repeated attempts without a new outcome",
|
|
"deferredRecords": len(cooling), "deferredIds": sorted(cooling), "retryAt": min(retry_times[record_id] for record_id in cooling)}
|
|
|
|
|
|
def batch_retry_details(batch: dict, now: float | None = None) -> dict:
|
|
return retry_details(selected_ids(batch), now)
|
|
|
|
|
|
def progress_summary(state: dict, window: int = 20) -> dict:
|
|
history = state.get("history", [])[-window:]
|
|
progressed = sum(row.get("outcome") == "progress" for row in history)
|
|
no_progress = sum(row.get("outcome") in {"no_progress", "source_edits_pending_outcome"} for row in history)
|
|
measured = [row for row in history if "firstBusinessPassIds" in row]
|
|
first_passes = {value for row in measured for value in row["firstBusinessPassIds"]}
|
|
refreshes = sum(len(row.get("refreshedBusinessIds", [])) for row in measured)
|
|
return {"cycles": len(history), "progressCycles": progressed, "noProgressCycles": no_progress,
|
|
"deliveryMeasuredCycles": len(measured), "firstBusinessPasses": len(first_passes), "businessRefreshes": refreshes,
|
|
"providerWaitCycles": sum(row.get("outcome") == "provider_wait" for row in history),
|
|
"progressRate": 100 * progressed / len(history) if history else None,
|
|
"noProgressSeconds": round(sum(float(row.get("seconds", 0)) for row in history if row.get("outcome") != "progress"), 3),
|
|
"verdict": "STALLED: recent cycles produced no new business outcomes" if no_progress >= NO_PROGRESS_LIMIT and progressed == 0
|
|
else "WAITING: provider has not returned a business outcome" if history and progressed == 0 and no_progress == 0
|
|
else "PROGRESS: first business acceptance recorded" if first_passes
|
|
else "ACTIVITY: routes, receipts or evidence changed; not proof of new business delivery" if progressed
|
|
else "OBSERVING: insufficient completed cycles"}
|
|
|
|
|
|
def stalled_launch(status: dict, batch: dict, state: dict, now: float | None = None) -> dict:
|
|
"""Detect a live turn launched after its records entered cooldown."""
|
|
now = time.time() if now is None else now
|
|
if status.get("status") != "codex_turn_running" or status.get("moduleBatchId") != batch.get("batchId"):
|
|
return {}
|
|
try:
|
|
started = datetime.fromisoformat(status["turnStartedAt"].replace("Z", "+00:00")).timestamp()
|
|
except (KeyError, TypeError, ValueError):
|
|
return {}
|
|
entries = state.get("records", {})
|
|
modules = cooling_modules(now, state)
|
|
blocked = sorted(record_id for record_id in selected_ids(batch)
|
|
if (float(entries.get(record_id, {}).get("retryAt", 0)) > now
|
|
and 0 < float(entries.get(record_id, {}).get("lastAttemptAt", 0)) < started)
|
|
or (record_module(record_id) in modules and 0 < float(modules[record_module(record_id)].get("lastAttemptAt", 0)) < started))
|
|
if not blocked:
|
|
return {}
|
|
return {"reason": "cooled business records were launched again", "batchId": batch.get("batchId"),
|
|
"recordIds": blocked, "cycle": status.get("cycle")}
|
|
|
|
|
|
def recovery_requested(pid: int, batch_id: str | None = None) -> bool:
|
|
request = read(RECOVERY_PATH)
|
|
return (request.get("schedulerPid") == pid and request.get("status") == "requested"
|
|
and (batch_id is None or request.get("batchId") == batch_id))
|
|
|
|
|
|
def advanced(before: dict, after: dict) -> bool:
|
|
if after.get("failureResolution") in {"resolved", "superseded", "not_applicable"} and after.get("failureResolution") != before.get("failureResolution"):
|
|
return True
|
|
if before.get("route") != "routed" and after.get("route") == "routed":
|
|
return True
|
|
ready = {"implementation_pending_build": 1, "implemented_pending_use": 2, "implementation_ready": 2}
|
|
old_receipt, receipt = before.get("receipt", {}), after.get("receipt", {})
|
|
if receipt.get("status") in ready and ready[receipt["status"]] > ready.get(old_receipt.get("status"), 0):
|
|
return True
|
|
accepted = after.get("acceptance", {})
|
|
if accepted.get("status") == "passed" and accepted != before.get("acceptance"):
|
|
return True
|
|
previous_issues = dict(before.get("issues", []))
|
|
return any(status in {"fix_deployed_pending_use", "resolved", "closed"} and status != previous_issues.get(issue_id)
|
|
for issue_id, status in after.get("issues", []))
|
|
|
|
|
|
def substantive_progress(before: dict, after: dict) -> bool:
|
|
"""Keep valid revalidation without letting it hide an unfinished port."""
|
|
if advanced({**before, "acceptance": {}}, {**after, "acceptance": {}}):
|
|
return True
|
|
return (after.get("acceptance", {}).get("status") == "passed"
|
|
and before.get("acceptance", {}).get("status") != "passed")
|
|
|
|
|
|
def record_outcome(batch: dict, before: dict, after: dict, *, cycle: int, seconds: float = 0,
|
|
file_changes: int = 0, provider_failure: bool = False, model_response: bool = False,
|
|
retry_delay: float = 20, reason: str = "", now: float | None = None) -> dict:
|
|
now = time.time() if now is None else now
|
|
state = read(PROGRESS_PATH)
|
|
state.update(schema="freecad-web-scheduler-progress/v1", updatedAt=now)
|
|
records = state.setdefault("records", {})
|
|
provider = state.setdefault("provider", {})
|
|
modules = module_states(state)
|
|
state["modules"] = modules
|
|
module = batch.get("selected", {}).get("module")
|
|
review = batch.get("selected", {}).get("tier") == "failure-reconciliation"
|
|
module_state = (modules.get(module, {}) if review else modules.setdefault(module, {})) if module else {}
|
|
# Edits are activity, not delivery. The post-turn pipeline has already had
|
|
# its chance to build/register them; only a changed outcome resets a streak.
|
|
progressed = sorted(record_id for record_id in selected_ids(batch) if advanced(before.get(record_id, {}), after.get(record_id, {})))
|
|
substantive = [record_id for record_id in progressed
|
|
if substantive_progress(before.get(record_id, {}), after.get(record_id, {}))]
|
|
refresh_only = bool(progressed) and not substantive
|
|
first_passes, refreshed = [], []
|
|
for record_id in progressed:
|
|
old = before.get(record_id, {}).get("acceptance", {})
|
|
current = after.get(record_id, {}).get("acceptance", {})
|
|
if current.get("status") == "passed" and current != old:
|
|
(refreshed if old.get("evidencePath") or old.get("status") == "passed" else first_passes).append(record_id)
|
|
deferred = []
|
|
advance_provider(provider, provider_failure, model_response, now, retry_delay, reason)
|
|
if not provider_failure:
|
|
if not review:
|
|
module_state.update(lastAttemptAt=now, lastCycle=cycle, lastReason=reason)
|
|
if not review and substantive:
|
|
module_state.update(consecutiveNoProgress=0, recoveryLevel=0, retryAt=0, lastProgressAt=now)
|
|
elif not review and not refresh_only:
|
|
streak = int(module_state.get("consecutiveNoProgress", 0)) + 1
|
|
module_state["consecutiveNoProgress"] = streak
|
|
module_limit = FEATURE_NO_PROGRESS_LIMIT if batch.get("selected", {}).get("tier") == "feature-workflow" else MODULE_NO_PROGRESS_LIMIT
|
|
if streak >= module_limit:
|
|
level = int(module_state.get("recoveryLevel", 0)) + 1
|
|
module_state.update(consecutiveNoProgress=0, recoveryLevel=level,
|
|
retryAt=0 if batch.get("selected", {}).get("tier") == "feature-workflow" else now + min(3600, MODULE_COOLDOWN_SECONDS * 2 ** min(level - 1, 3)),
|
|
recoveryAction="repair shared business integration before repeating command batches")
|
|
for record_id in selected_ids(batch):
|
|
row = records.setdefault(record_id, {})
|
|
row.update(module=batch.get("selected", {}).get("module"), lastCycle=cycle, lastAttemptAt=now)
|
|
if record_id in progressed:
|
|
row.update(consecutiveNoProgress=0, retryAt=0, lastProgressAt=now, status="progressed")
|
|
if record_id in substantive:
|
|
row["cooldownRounds"] = 0
|
|
continue
|
|
attempts = int(row.get("consecutiveNoProgress", 0)) + 1
|
|
row.update(consecutiveNoProgress=attempts, status="pending", lastReason=reason or "no new route, implementation receipt or business result")
|
|
record_limit = FEATURE_NO_PROGRESS_LIMIT if batch.get("selected", {}).get("tier") == "feature-workflow" else NO_PROGRESS_LIMIT
|
|
if attempts >= (1 if review else record_limit):
|
|
rounds = int(row.get("cooldownRounds", 0)) + 1
|
|
row.update(status="cooldown", retryAt=now + min(3600, COOLDOWN_SECONDS * 2 ** min(rounds - 1, 4)),
|
|
cooldownRounds=rounds, consecutiveNoProgress=0)
|
|
deferred.append(record_id)
|
|
outcome = "provider_wait" if provider_failure else "progress" if progressed else "source_edits_pending_outcome" if file_changes else "no_progress"
|
|
event = {"cycle": cycle, "batchId": batch.get("batchId"), "module": batch.get("selected", {}).get("module"),
|
|
"tier": batch.get("selected", {}).get("tier"),
|
|
"businessLane": batch.get("businessLane", "acceptance"),
|
|
"outcome": outcome, "seconds": round(seconds, 3), "fileChanges": file_changes,
|
|
"progressedIds": progressed, "firstBusinessPassIds": first_passes, "refreshedBusinessIds": refreshed,
|
|
"substantiveProgressIds": substantive, "evidenceRefreshOnly": refresh_only,
|
|
"deferredIds": deferred, "reason": reason, "at": now}
|
|
history = [row for row in state.get("history", []) if row.get("cycle") != cycle]
|
|
state["history"] = (history + [event])[-100:]
|
|
state["lastOutcome"] = event
|
|
save(state)
|
|
return {**event, "provider": provider, "moduleRecovery": module_state}
|