304 lines
20 KiB
Python
304 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Persist periodic unattended optimization, executed between business turns."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from implementation_first import user_validation_deferred
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
STATE_PATH = ROOT / "docs/current/scheduler-maintenance.json"
|
|
DEFAULT_INTERVAL = 1800
|
|
|
|
|
|
def read(path: Path) -> dict:
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
return value if isinstance(value, dict) else {}
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
|
|
def timestamp(value: float) -> str:
|
|
return datetime.fromtimestamp(value, timezone.utc).isoformat()
|
|
|
|
|
|
def write(path: Path, value: dict) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(path)
|
|
|
|
|
|
@contextmanager
|
|
def locked_state():
|
|
STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
with STATE_PATH.with_suffix(".lock").open("a") as lock:
|
|
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
|
|
yield read(STATE_PATH)
|
|
|
|
|
|
def configure(enabled: bool, interval: int = DEFAULT_INTERVAL, *, now: float | None = None, run_now: bool = False) -> dict:
|
|
if interval < 60:
|
|
raise ValueError("optimization interval must be at least 60 seconds")
|
|
now = time.time() if now is None else now
|
|
with locked_state() as state:
|
|
state.setdefault("runCount", 0)
|
|
if enabled and (not state.get("enabled") or state.get("intervalSeconds") != interval or run_now):
|
|
state["nextRunAt"] = now if run_now else now + interval
|
|
state.update(schema="freecad-web-scheduler-maintenance/v1", enabled=enabled, intervalSeconds=interval,
|
|
configuredAt=timestamp(now), mode="between-business-turns")
|
|
if state.get("status") != "running":
|
|
state["status"] = "scheduled" if enabled else "disabled"
|
|
write(STATE_PATH, state)
|
|
return state
|
|
|
|
|
|
def advance_slot(due: float, interval: int, now: float) -> tuple[float, int]:
|
|
slots = max(1, int((now - due) // interval) + 1)
|
|
return due + slots * interval, slots - 1
|
|
|
|
|
|
def active_child(state: dict | None = None) -> int | None:
|
|
state = read(STATE_PATH) if state is None else state
|
|
pid = state.get("childPid")
|
|
if state.get("status") != "running" or not isinstance(pid, int) or pid <= 0:
|
|
return None
|
|
try:
|
|
command = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace")
|
|
return pid if "codex" in command and "exec" in command and str(ROOT) in command else None
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def track_child(run_id: str, pid: int | None) -> None:
|
|
with locked_state() as state:
|
|
if state.get("status") == "running" and state.get("runId") == run_id:
|
|
state["childPid"] = pid
|
|
write(STATE_PATH, state)
|
|
|
|
|
|
def claim_due(*, now: float | None = None) -> dict | None:
|
|
"""The caller must hold the scheduler lock and have no active business turn."""
|
|
if user_validation_deferred():
|
|
return None
|
|
now = time.time() if now is None else now
|
|
with locked_state() as state:
|
|
if not state.get("enabled"):
|
|
return None
|
|
if state.get("status") == "running":
|
|
# A surviving worker must be drained before a replacement scheduler
|
|
# may run. A completed scheduler turn records its outcome in finish.
|
|
owner = state.get("schedulerPid")
|
|
if active_child(state) or (owner and Path(f"/proc/{owner}").exists()):
|
|
return None
|
|
state.update(status="scheduled", lastOutcome="interrupted")
|
|
write(STATE_PATH, state)
|
|
due = float(state.get("nextRunAt", now + DEFAULT_INTERVAL))
|
|
if now < due:
|
|
return None
|
|
interval = int(state.get("intervalSeconds", DEFAULT_INTERVAL))
|
|
next_due, skipped = advance_slot(due, interval, now)
|
|
count = int(state.get("runCount", 0)) + 1
|
|
state.update(status="running", schedulerPid=os.getpid(), runCount=count,
|
|
runId=f"optimization-{count:06d}", scheduledAt=timestamp(due), startedAt=timestamp(now),
|
|
nextRunAt=next_due, lateBySeconds=round(max(0, now - due), 3),
|
|
skippedIntervals=int(state.get("skippedIntervals", 0)) + skipped)
|
|
write(STATE_PATH, state)
|
|
return dict(state)
|
|
|
|
|
|
def diagnostic_context() -> dict:
|
|
current = ROOT / "docs/current"
|
|
status = read(current / "freecad-web-scheduler-status.json")
|
|
progress = read(current / "scheduler-progress.json")
|
|
history = progress.get("history", [])[-20:]
|
|
issues = read(current / "runtime-issue-ledger.json").get("issues", [])
|
|
pending_issues = [row for row in issues if row.get("status") not in {"resolved", "closed"}]
|
|
pipeline = read(current / "module-batch-pipeline.json")
|
|
audit = read(current / "web-business-completion-audit.json")
|
|
timing = read(current / "freecad-web-scheduler-timing.json")
|
|
previous = read(STATE_PATH)
|
|
recovery = sorted(progress.get("modules", {}).items(), key=lambda item: item[1].get("consecutiveNoProgress", 0), reverse=True)[:8]
|
|
return {
|
|
"scheduler": {key: status.get(key) for key in ("implementation", "cycle", "status", "lastError", "lastCommand",
|
|
"lastCommandStatus", "currentStepPhase", "phaseTiming", "lastTurnDurationSeconds", "moduleRecovery")},
|
|
"recentOutcomes": dict(Counter(row.get("outcome", "unknown") for row in history)),
|
|
"recentBusiness": [{key: row.get(key) for key in ("cycle", "module", "outcome", "seconds", "progressedIds")} for row in history[-5:]],
|
|
"provider": progress.get("provider", {}),
|
|
"moduleRecovery": {module: {**{key: state.get(key) for key in ("consecutiveNoProgress", "retryAt", "recoveryLevel")},
|
|
"lastReason": str(state.get("lastReason", ""))[:600]} for module, state in recovery},
|
|
"runtimeIssues": [{**{key: row.get(key) for key in ("id", "capabilityId", "status", "args")},
|
|
"firstError": str(row.get("firstError", ""))[:1000]} for row in pending_issues[:8]],
|
|
"runtimeIssueCount": len(pending_issues),
|
|
"pipeline": {key: pipeline.get(key) for key in ("status", "module", "target", "failedStage", "buildReuse", "totalSeconds")},
|
|
"pipelineSteps": [{key: step.get(key) for key in ("name", "seconds", "exitCode")} for step in pipeline.get("steps", [])],
|
|
"businessAudit": {key: audit.get(key) for key in ("verdict", "businessFunctionalComplete", "businessProgress", "implementationProgress")},
|
|
"timing": {key: timing.get(key) for key in ("window", "totals", "phaseTiming", "redundancy", "activity")},
|
|
"slowestCommands": timing.get("slowestCommands", [])[:5],
|
|
"failureSummary": read(current / "failure-reconciliation.json").get("summary", {}),
|
|
"batch": read(current / "module-batch-manifest.json"),
|
|
"previousOptimization": {key: previous.get(key) for key in ("lastOutcome", "lastDurationSeconds", "lastCommandsFailed", "lastFileChanges", "lastReportPath", "lastResultPath")},
|
|
}
|
|
|
|
|
|
def prepare_work(context: dict, directory: Path, run_id: str) -> dict:
|
|
pending = context.get("runtimeIssues", [])
|
|
batch = context.get("batch", {})
|
|
if pending:
|
|
issue = pending[0]
|
|
capability_id = issue.get("capabilityId")
|
|
rows = read(ROOT / "docs/current/feature-capability-registry.json").get("records", [])
|
|
capability = next((row for row in rows if row.get("id") == capability_id), None)
|
|
if capability:
|
|
module = capability["module"]
|
|
batch = {"status": "ready", "batchId": run_id, "strategy": "feature-first-deferred-validation",
|
|
"selected": {"module": module, "tier": "feature-workflow", "family": "runtime-issue", "recordCount": 1},
|
|
"records": [[capability_id, capability["command"], "implementation_required",
|
|
capability.get("sourcePath") or f"web/src/{capability.get('worker', 'workbench-shell.tsx')}",
|
|
capability.get("sourceLine") or 1]]}
|
|
if batch.get("status") != "ready" or not batch.get("records"):
|
|
return {"status": "no_ready_business", "nextAction": "use the timing diagnosis and existing scheduler code"}
|
|
batch_path = directory / f"{run_id}-batch.json"
|
|
packet_path = directory / f"{run_id}-work.json"
|
|
write(batch_path, batch)
|
|
run = subprocess.run([sys.executable, "tools/prepare_module_implementation_packet.py", "--batch", str(batch_path), "--output", str(packet_path)],
|
|
cwd=ROOT, capture_output=True, text=True, timeout=30)
|
|
if run.returncode:
|
|
return {"status": "packet_failed", "error": (run.stderr or run.stdout)[-2000:]}
|
|
packet = read(packet_path)
|
|
sources = packet.get("supportFiles", [])
|
|
batch["sourcePaths"] = sorted({row["path"] for row in sources if isinstance(row.get("path"), str)})
|
|
write(batch_path, batch)
|
|
ancestors = {ROOT, ROOT.parent}
|
|
for source in sources:
|
|
candidate = (ROOT / str(source.get("path", ""))).resolve()
|
|
ancestors.update(parent for parent in candidate.parents if parent.is_relative_to(ROOT))
|
|
instructions = sorted(str((parent / "AGENTS.md").relative_to(ROOT.parent)) for parent in ancestors if (parent / "AGENTS.md").is_file())
|
|
return {"status": "ready", "sourceBatch": str(batch_path.relative_to(ROOT)), "packetPath": str(packet_path.relative_to(ROOT)),
|
|
"module": packet.get("module"), "build": packet.get("build"), "acceptanceSetup": packet.get("acceptanceSetup"),
|
|
"instructionFilesRelativeToWorkspaceParent": instructions,
|
|
"existingFiles": [{"path": row.get("path"), "sha256": row.get("sha256")} for row in sources],
|
|
"editableExcerpts": packet.get("editableExcerpts", []), "records": packet.get("records", [])}
|
|
|
|
|
|
def prompt_for(context_path: Path, report_path: Path, work: dict | None = None) -> str:
|
|
return f"""Perform the user's scheduled 30-minute unattended optimization now.
|
|
Workspace: {ROOT}
|
|
Further review the unattended workflow; automatically discover functional gaps and idle loops, repair them, identify and optimize expensive bottlenecks, reduce elapsed time and improve execution efficiency. Reuse the supplied FreeCAD source and existing implementations, and find authoritative source material when needed to complete the affected functionality.
|
|
|
|
The iron rule is functional module business and functional routes driving the entire migration. Do not switch to blanket per-API, Python object, workbench or full-parity replay. Never equate a running process, edited source, build receipt, constant fixture or a counter with a working business result.
|
|
User iron rule added 2026-09-08: first search the user's supplied FreeCAD source and related material. Before implementation, acceptance or declaring a gap, inspect the relevant local source, dependency source, documents, existing cases and artifacts. Reuse unchanged inspected excerpts. Consult docs/current/provided-source-index.md for confirmed locations; only seek external authoritative material after identifying the specific remaining local gap. Do not restart full-tree searches or invent replacements for existing source.
|
|
|
|
Read the compact observed context at {context_path.relative_to(ROOT)}. Use python3 tools/show_freecad_web_scheduler.py and tools/analyze_scheduler_timings.py only as needed. Follow docs/UNATTENDED_WORKFLOW.md. Select the highest-impact evidenced bottleneck or business gap. Reuse native source, module artifacts, caches and current acceptance evidence. Make a concrete repair and run the focused tests or business workflow needed to prove it. Do not merely write another diagnostic or scheduling rule when a real implementation failure can be fixed. If no actionable gap is found, report that honestly and stop without gratuitous changes.
|
|
|
|
The prepared work below already identifies existing files, applicable AGENTS.md files and editable source. An empty instruction list means none were found in the inspected ancestor directories. Do not repeat pwd, git status, find AGENTS.md or rediscover these paths. Use the supplied source immediately; never guess missing case filenames. Pure exploration without an edit or verification has a cumulative 180-second budget, even if additional searches produce output.
|
|
PREPARED WORK: {json.dumps(work or {}, ensure_ascii=False, separators=(',', ':'))}
|
|
|
|
This is a dedicated optimization turn, not a restricted implementation-only turn. Scoped builds and meaningful verification are allowed and required for your changes. Native source edits must be built and their artifacts synchronized before business acceptance; do not verify an old artifact against changed source. Use apply_patch for edits. Use tools/fast_source_search.py with explicit paths; keep exploration to eight targeted source reads and the overall turn to twenty commands. Do not rerun unchanged failed commands. Preserve other work and all unresolved failure records; only close failures with current attributable proof. Reuse tools/wasm-probe-runtime.mjs for Node WASM cases and terminate completed one-shot pthread case processes. Declare affected business IDs and use tools/run_business_acceptance.py for real acceptance results.
|
|
|
|
Use the supplied build tree and target. Do not force rebuilds with touch or os.utime; for embedded Python/resource changes, fix the target's declared LINK_DEPENDS and let one normal incremental build decide what changed. Synchronize the complete runtime including pthread workers before checking the public artifacts. Use run_business_acceptance.py --reuse for unchanged verified cases; its source, case, artifact and failure-observation checks decide whether execution can be skipped. Do not treat a successful test of a build directory as proof that the browser's public artifact is complete.
|
|
|
|
The scheduler owns the exclusive workspace execution slot. Do not start, stop, kill or restart scheduler/watchdog processes, edit the periodic schedule or spawn parallel writers. Do not manually overwrite the normal batch/task/acceptance ledgers or erase checkpoints to appear productive. The parent resumes business work and reloads scheduler code after this turn. Runtime logs can be read while this turn runs.
|
|
|
|
Write a concise Markdown report to {report_path.relative_to(ROOT)} with the observed gap, concrete changes, actual commands/results, measured before/after timing when available, remaining work and affected module/route IDs. Distinguish verified outcomes from pending work. Summarize the result in Chinese. Do not send messages to external services or publish anything.
|
|
"""
|
|
|
|
|
|
def execute(request: dict, runner, live_log: Path, *, now=None, context: dict | None = None) -> dict:
|
|
clock = now or time.time
|
|
directory = ROOT / "docs/current/periodic-optimization"
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
context_path = directory / f"{request['runId']}-context.json"
|
|
report_path = directory / f"{request['runId']}.md"
|
|
context = diagnostic_context() if context is None else context
|
|
preparation_started = clock()
|
|
try:
|
|
work = prepare_work(context, directory, request["runId"])
|
|
except (OSError, ValueError, subprocess.TimeoutExpired) as error:
|
|
work = {"status": "packet_failed", "error": str(error)}
|
|
write(context_path, context)
|
|
metadata = {"executionMode": "periodic-optimization"}
|
|
if work.get("sourceBatch"):
|
|
metadata["sourceBatch"] = str(ROOT / work["sourceBatch"])
|
|
started = clock()
|
|
preparation_seconds = max(0, started - preparation_started)
|
|
stream_failed = False
|
|
try:
|
|
code, timed_out, stream_failed, budget_exceeded, idle_stopped = runner(prompt_for(context_path, report_path, work), live_log, metadata)
|
|
completed = (code == 0 and metadata.get("turnCompleted") is True and metadata.get("commandsCompleted", 0) > 0
|
|
and report_path.is_file() and not any((timed_out, stream_failed, budget_exceeded, idle_stopped)))
|
|
outcome = "completed" if completed else "exploration_budget" if metadata.get("explorationBudgetExceeded") else "provider_wait" if stream_failed else "incomplete"
|
|
error = None
|
|
except Exception as exception:
|
|
code, completed, outcome, error = 1, False, "failed", str(exception)
|
|
from analyze_scheduler_timings import detailed_steps
|
|
timing = detailed_steps(live_log)
|
|
agent_report = report_path.is_file()
|
|
if not agent_report:
|
|
lines = [f"# {request['runId']}", "", f"Outcome: {outcome}. This is an execution record, not a successful repair claim.", "",
|
|
f"File changes: {metadata.get('fileChanges', 0)}; completed commands: {metadata.get('commandsCompleted', 0)}.",
|
|
f"Observed tool time: {timing['toolUnionSeconds']:.3f} s; between-event time: {timing['betweenEventSeconds']:.3f} s.", "",
|
|
"| Step | Wait before (s) | Run (s) | Exit | Command |", "|---:|---:|---:|---:|---|"]
|
|
for row in timing["commands"]:
|
|
command = str(row["command"]).replace("|", "\\|").replace("\n", " ")
|
|
lines.append(f"| {row['order']} | {row['waitBeforeSeconds']} | {row['runSeconds']} | {row['exitCode']} | {command} |")
|
|
report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
result = {"runId": request["runId"], "status": outcome, "exitCode": code, "turnCompleted": completed,
|
|
"elapsedSeconds": round(max(0, clock() - started), 3), "finishedAt": timestamp(clock()),
|
|
"preparationSeconds": round(preparation_seconds, 3),
|
|
"commandsCompleted": metadata.get("commandsCompleted", 0), "commandsFailed": metadata.get("commandsFailed", 0),
|
|
"fileChanges": metadata.get("fileChanges", 0), "commandResults": metadata.get("commandResults", []),
|
|
"providerFailed": stream_failed,
|
|
"providerFailureReason": metadata.get("providerFailureReason", ""),
|
|
"modelResponded": bool(metadata.get("turnCompleted") or metadata.get("commandsCompleted")
|
|
or metadata.get("fileChanges") or metadata.get("lastAgentMessage")),
|
|
"summary": metadata.get("lastAgentMessage", ""), "error": error,
|
|
"stepTiming": timing, "reportSource": "agent" if agent_report else "scheduler",
|
|
"logPath": str(live_log.relative_to(ROOT)),
|
|
"reportPath": str(report_path.relative_to(ROOT)) if report_path.is_file() else None}
|
|
result_path = directory / f"{request['runId']}-result.json"
|
|
write(result_path, result)
|
|
with locked_state() as state:
|
|
if state.get("runId") == request["runId"]:
|
|
now_value = clock()
|
|
if float(state["nextRunAt"]) <= now_value:
|
|
next_due, skipped = advance_slot(float(state["nextRunAt"]), int(state["intervalSeconds"]), now_value)
|
|
state.update(nextRunAt=next_due, skippedIntervals=int(state.get("skippedIntervals", 0)) + skipped + 1)
|
|
state.update(status="scheduled" if state.get("enabled") else "disabled", schedulerPid=None,
|
|
lastOutcome=outcome, lastFinishedAt=result["finishedAt"], lastDurationSeconds=result["elapsedSeconds"],
|
|
lastResultPath=str(result_path.relative_to(ROOT)), lastReportPath=result["reportPath"],
|
|
lastLog=result["logPath"], lastCommandsCompleted=result["commandsCompleted"],
|
|
lastCommandsFailed=result["commandsFailed"], lastFileChanges=result["fileChanges"], lastError=error)
|
|
write(STATE_PATH, state)
|
|
return result
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
mode = parser.add_mutually_exclusive_group()
|
|
mode.add_argument("--enable", action="store_true")
|
|
mode.add_argument("--disable", action="store_true")
|
|
parser.add_argument("--interval", type=int, default=DEFAULT_INTERVAL)
|
|
parser.add_argument("--run-now", action="store_true", help="queue one immediate turn when enabling")
|
|
args = parser.parse_args()
|
|
if args.run_now and not args.enable:
|
|
parser.error("--run-now requires --enable")
|
|
state = configure(args.enable, args.interval, run_now=args.run_now) if args.enable or args.disable else read(STATE_PATH)
|
|
print(json.dumps(state, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|