1098 lines
58 KiB
Python
1098 lines
58 KiB
Python
#!/usr/bin/env python3
|
|
"""Show the observable state of the FreeCAD Web continuation scheduler."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
from collections import deque
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from scheduler_progress import progress_summary, cooling_modules
|
|
from failure_reconciliation import summary_current as failure_summary
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PID_PATH = ROOT / "docs/current/freecad-web-scheduler.pid"
|
|
STATUS_PATH = ROOT / "docs/current/freecad-web-scheduler-status.json"
|
|
FAILURE_STATE_PATH = ROOT / "docs/current/freecad-web-scheduler-failures.json"
|
|
FAILURE_PATTERN_PATH = ROOT / "docs/current/failure-patterns.json"
|
|
WATCHDOG_PID_PATH = ROOT / "docs/current/freecad-web-scheduler-watchdog.pid"
|
|
WATCHDOG_STATUS_PATH = ROOT / "docs/current/freecad-web-scheduler-watchdog-status.json"
|
|
CHECKPOINT_PATH = ROOT / "docs/current/continuation-state.json"
|
|
TASK_STATE_PATH = ROOT / "docs/current/task-state.json"
|
|
PYTHON_GATE_PATH = ROOT / "docs/p6/python-api-full-gate.json"
|
|
PYTHON_API_MANIFEST_PATH = ROOT / "docs/p6/python-api-manifest.json"
|
|
SOURCE_API_INVENTORY_PATH = ROOT / "docs/format-api-inventory.json"
|
|
FULL_PARITY_GATE_PATH = ROOT / "docs/p6/full-parity-gate.json"
|
|
FEATURE_CAPABILITY_PATH = ROOT / "docs/current/feature-capability-registry.json"
|
|
IMPLEMENTATION_REGISTRY_PATH = ROOT / "docs/current/implementation-first-registry.json"
|
|
RUNTIME_ISSUE_PATH = ROOT / "docs/current/runtime-issue-ledger.json"
|
|
SCHEDULER_OUTPUT_PATH = ROOT / "docs/current/freecad-web-scheduler.out"
|
|
RUN_LOG_DIR = ROOT / "docs/current/continuation-runs"
|
|
ASSET_SYNC_TIMING_PATH = ROOT / "docs/current/web-assets-sync-timing.json"
|
|
MODULE_BUILD_PATH = ROOT / "docs/current/web-module-build.json"
|
|
PENDING_BUILD_PATH = ROOT / "docs/current/module-build-pending.json"
|
|
MODULE_BATCH_PATH = ROOT / "docs/current/module-batch-manifest.json"
|
|
BUSINESS_AUDIT_PATH = ROOT / "docs/current/web-business-completion-audit.json"
|
|
PROGRESS_PATH = ROOT / "docs/current/scheduler-progress.json"
|
|
|
|
|
|
def process_matches(pid: int | None, expected: str) -> bool:
|
|
"""Reject stale/reused PID files instead of reporting unrelated processes."""
|
|
if not pid or pid <= 0:
|
|
return False
|
|
try:
|
|
command_line = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace")
|
|
except (FileNotFoundError, OSError):
|
|
return False
|
|
return expected in command_line
|
|
|
|
|
|
def read_json(path: Path) -> dict:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except (FileNotFoundError, json.JSONDecodeError):
|
|
return {}
|
|
|
|
|
|
def scalar(value: object) -> str:
|
|
if value is None:
|
|
return "-"
|
|
if isinstance(value, (dict, list)):
|
|
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
return str(value)
|
|
|
|
|
|
def percentage(value: int | float, total: int | float) -> str:
|
|
if not total:
|
|
return "-"
|
|
return f"{100.0 * value / total:.2f}%"
|
|
|
|
|
|
def progress_snapshot(
|
|
task_state: dict,
|
|
python_gate: dict,
|
|
parity_gate: dict,
|
|
python_api_manifest: dict | None = None,
|
|
source_api_inventory: dict | None = None,
|
|
feature_capabilities: dict | None = None,
|
|
implementation_registry: dict | None = None,
|
|
runtime_issues: dict | None = None,
|
|
) -> dict[str, object]:
|
|
counts = task_state.get("counts", {}) if isinstance(task_state.get("counts"), dict) else {}
|
|
tasks = task_state.get("tasks", {}) if isinstance(task_state.get("tasks"), dict) else {}
|
|
task_total = len(tasks)
|
|
task_completed = int(counts.get("completed", sum(value == "completed" for value in tasks.values())))
|
|
modules = task_state.get("module_status", {}) if isinstance(task_state.get("module_status"), dict) else {}
|
|
module_total = len(modules)
|
|
module_completed = sum(value == "completed" for value in modules.values())
|
|
# The full gate is a historical snapshot during implementation. Prefer the
|
|
# canonical API manifest, which is updated by each focused batch, so the
|
|
# live status does not stay frozen at the last full-gate count.
|
|
manifest_counts = python_api_manifest.get("counts", {}) if isinstance(python_api_manifest, dict) else {}
|
|
manifest_records = int(manifest_counts.get("canonicalRecords", 0) or 0) if isinstance(manifest_counts, dict) else 0
|
|
coverage = python_gate.get("coverage", {}) if isinstance(python_gate.get("coverage"), dict) else {}
|
|
if manifest_records:
|
|
api_total = manifest_records
|
|
records = python_api_manifest.get("records", [])
|
|
disposition_counts: dict[str, int] = {}
|
|
if isinstance(records, list):
|
|
for record in records:
|
|
if isinstance(record, dict):
|
|
status = str(record.get("disposition", "planned"))
|
|
disposition_counts[status] = disposition_counts.get(status, 0) + 1
|
|
api_counts = disposition_counts
|
|
else:
|
|
api_counts = coverage.get("counts", {}) if isinstance(coverage.get("counts"), dict) else {}
|
|
api_total = int(coverage.get("inventoryRecords", 0) or 0)
|
|
api_exposed = int(api_counts.get("exposed", 0) or 0)
|
|
api_restricted = int(api_counts.get("restricted", 0) or 0)
|
|
api_not_implemented = int(api_counts.get("not_implemented", 0) or 0)
|
|
api_planned = int(api_counts.get("planned", 0) or 0)
|
|
source_api_total = int(source_api_inventory.get("api_evidence_count", 0) or 0) if isinstance(source_api_inventory, dict) else 0
|
|
supplemental_api_total = max(0, api_total - source_api_total) if source_api_total else 0
|
|
capability_counts = feature_capabilities.get("counts", {}) if isinstance(feature_capabilities, dict) else {}
|
|
implementation_batches = implementation_registry.get("batches", {}) if isinstance(implementation_registry, dict) else {}
|
|
implementation_records = implementation_registry.get("records", {}) if isinstance(implementation_registry, dict) else {}
|
|
capability_rows = feature_capabilities.get("records", []) if isinstance(feature_capabilities, dict) else []
|
|
routed_ids = {
|
|
str(row.get("id"))
|
|
for row in capability_rows
|
|
if isinstance(row, dict) and row.get("implementationStatus") == "routed"
|
|
}
|
|
allowed_receipt_statuses = {"implementation_ready", "implementation_pending_build", "implemented_pending_use"}
|
|
effective_implementation_records = {
|
|
str(record_id)
|
|
for record_id, entry in implementation_records.items()
|
|
if isinstance(implementation_records, dict)
|
|
and isinstance(entry, dict)
|
|
and str(entry.get("status", "")) in allowed_receipt_statuses
|
|
and str(record_id) in routed_ids
|
|
}
|
|
business_implementation_records = {
|
|
str(record_id)
|
|
for record_id, entry in implementation_records.items()
|
|
if isinstance(implementation_records, dict)
|
|
and isinstance(entry, dict)
|
|
and str(entry.get("status", "")) in allowed_receipt_statuses
|
|
and entry.get("scope") == "business-family"
|
|
}
|
|
issue_rows = runtime_issues.get("issues", []) if isinstance(runtime_issues, dict) else []
|
|
unresolved_issues = sum(
|
|
isinstance(row, dict) and str(row.get("status", "open")) not in {"resolved", "closed"}
|
|
for row in issue_rows if isinstance(issue_rows, list)
|
|
)
|
|
summary = parity_gate.get("summary", {}) if isinstance(parity_gate.get("summary"), dict) else {}
|
|
areas = int(summary.get("areas", 0) or 0)
|
|
source_pass = int(summary.get("areasPassingSourceGate", 0) or 0)
|
|
parity_pending = int(summary.get("areasPendingFullParity", areas) or 0)
|
|
return {
|
|
"taskCompleted": task_completed,
|
|
"taskTotal": task_total,
|
|
"moduleCompleted": module_completed,
|
|
"moduleTotal": module_total,
|
|
"apiTotal": api_total,
|
|
"sourceApiTotal": source_api_total,
|
|
"supplementalApiTotal": supplemental_api_total,
|
|
"apiExposed": api_exposed,
|
|
"apiRestricted": api_restricted,
|
|
"apiNotImplemented": api_not_implemented,
|
|
"apiPlanned": api_planned,
|
|
"apiClassified": api_exposed + api_restricted + api_not_implemented,
|
|
"featureModules": int(capability_counts.get("modules", 0) or 0),
|
|
"featureArtifactModules": int(capability_counts.get("artifactModules", 0) or 0),
|
|
"featureCommands": int(capability_counts.get("commands", 0) or 0),
|
|
"featureRouted": int(capability_counts.get("routed", 0) or 0),
|
|
"featureInteractive": int(capability_counts.get("interactive", 0) or 0)
|
|
+ int(capability_counts.get("genericApiEntry", 0) or 0)
|
|
+ int(capability_counts.get("genericCommandEntry", 0) or 0)
|
|
+ int(capability_counts.get("reactNavigation", 0) or 0),
|
|
"featureNeedsUi": int(capability_counts.get("needsParameterUi", 0) or 0),
|
|
"featureNeedsReact": int(capability_counts.get("needsReactWorkflow", 0) or 0),
|
|
"featureImplementationRequired": int(capability_counts.get("implementationRequired", 0) or 0),
|
|
"featureValidationDeferred": int(capability_counts.get("validationDeferred", 0) or 0),
|
|
"implementationReceiptBatches": len(implementation_batches) if isinstance(implementation_batches, dict) else 0,
|
|
"implementationReceiptRecords": len(effective_implementation_records),
|
|
"implementationReceiptHistoricalRecords": len(implementation_records) if isinstance(implementation_records, dict) else 0,
|
|
"businessImplementationRecords": len(business_implementation_records),
|
|
"runtimeIssues": len(issue_rows) if isinstance(issue_rows, list) else 0,
|
|
"runtimeIssuesUnresolved": unresolved_issues,
|
|
"parityAreas": areas,
|
|
"paritySourcePass": source_pass,
|
|
"parityPending": parity_pending,
|
|
"parityComplete": max(0, areas - parity_pending),
|
|
}
|
|
|
|
|
|
def read_log_tail(path: Path, limit: int = 262144) -> str:
|
|
try:
|
|
with path.open("rb") as stream:
|
|
stream.seek(0, os.SEEK_END)
|
|
start = max(0, stream.tell() - limit)
|
|
stream.seek(start)
|
|
data = stream.read(limit)
|
|
if start:
|
|
data = data.partition(b"\n")[2]
|
|
return data.decode("utf-8", errors="replace")
|
|
except OSError:
|
|
return ""
|
|
|
|
|
|
def recent_efficiency(current_cycle: int | None, cycles_run: int | None) -> dict[str, object]:
|
|
"""Summarize effective turns and preparation failures from recent cycles."""
|
|
# Keep status queries bounded. `cycles_run` is the scheduler lifetime and
|
|
# must not turn a lightweight dashboard command into a scan of hundreds of
|
|
# historical JSONL logs. The timing summary uses the same default window.
|
|
try:
|
|
window = max(1, min(50, int(os.environ.get("FREECAD_EFFICIENCY_WINDOW", "20"))))
|
|
except ValueError:
|
|
window = 20
|
|
floor = max(0, int(current_cycle or 0) - window + 1)
|
|
outcomes: dict[int, str] = {}
|
|
ready = 0
|
|
command_started = command_completed = command_failed = resolved = 0
|
|
durations: list[float] = []
|
|
paths = ([RUN_LOG_DIR / f"cycle-{cycle:06d}.log" for cycle in range(floor, current_cycle + 1)]
|
|
if current_cycle is not None else sorted(RUN_LOG_DIR.glob("cycle-*.log"))[-window:])
|
|
for path in paths:
|
|
match = re.search(r"cycle-(\d+)\.log$", path.name)
|
|
if not match or int(match.group(1)) < floor:
|
|
continue
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
continue
|
|
first_line = next((line.strip() for line in text.splitlines() if line.strip()), "")
|
|
cycle_number = int(match.group(1))
|
|
if first_line == "TASK_CYCLE_ERROR" or any("CONTINUATION_STATE_INVALID" in line for line in text.splitlines()[:3]):
|
|
outcomes[cycle_number] = "prepare_error"
|
|
elif "TASK_CYCLE_CONTINUE" in text or "TASK_CYCLE_CLAIMED" in text:
|
|
ready += 1
|
|
started = completed = None
|
|
for line in text.splitlines():
|
|
try:
|
|
event = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
event_type = event.get("type")
|
|
if event_type == "scheduler.cycle.started":
|
|
started = event.get("startedAt")
|
|
elif event_type == "scheduler.cycle.completed":
|
|
completed = event.get("completedAt")
|
|
if event.get("timedOut") is True:
|
|
outcomes[cycle_number] = "timeout"
|
|
elif event.get("exitCode") in (0, None):
|
|
outcomes[cycle_number] = "effective"
|
|
if event_type == "item.started" and isinstance(event.get("item"), dict) and event["item"].get("type") == "command_execution":
|
|
command_started += 1
|
|
if event_type == "item.completed" and isinstance(event.get("item"), dict) and event["item"].get("type") == "command_execution":
|
|
command_completed += 1
|
|
if event["item"].get("exit_code") not in (0, None):
|
|
command_failed += 1
|
|
if started and completed:
|
|
try:
|
|
durations.append((datetime.fromisoformat(completed.replace("Z", "+00:00")) - datetime.fromisoformat(started.replace("Z", "+00:00"))).total_seconds())
|
|
except (TypeError, ValueError):
|
|
pass
|
|
if SCHEDULER_OUTPUT_PATH.exists():
|
|
output = read_log_tail(SCHEDULER_OUTPUT_PATH)
|
|
for line in output.splitlines():
|
|
cycle_match = re.search(r"SCHEDULER_(CYCLE_DONE|CYCLE_PREPARE_ERROR).*?cycle=(\d+)", line)
|
|
if cycle_match and int(cycle_match.group(2)) < floor:
|
|
continue
|
|
if cycle_match:
|
|
cycle_number = int(cycle_match.group(2))
|
|
if outcomes.get(cycle_number) != "timeout":
|
|
outcomes[cycle_number] = "effective" if cycle_match.group(1) == "CYCLE_DONE" else "prepare_error"
|
|
if "SCHEDULER_FAILURE_RESOLVED" in line:
|
|
resolved += 1
|
|
# Deterministic completed-batch fast paths do not create a Codex JSONL
|
|
# cycle log, but they are real successful scheduler cycles. Include them
|
|
# from the bounded timing index so throughput is not reported as zero.
|
|
timing = read_json(ROOT / "docs/current/freecad-web-scheduler-timing.json")
|
|
fast_path_cycles = 0
|
|
no_ready_cycles = 0
|
|
if timing.get("schema") in {"freecad-web-scheduler-timing/v1", "freecad-web-scheduler-timing/v2"}:
|
|
for record in timing.get("cycles", []) if isinstance(timing.get("cycles"), list) else []:
|
|
if not isinstance(record, dict):
|
|
continue
|
|
cycle = int(record.get("cycle", 0) or 0)
|
|
if cycle < floor or (current_cycle is not None and cycle > current_cycle):
|
|
continue
|
|
phases = record.get("phaseTiming", {})
|
|
if isinstance(phases, dict) and "scheduler/fast-path" in phases:
|
|
outcomes[cycle] = "effective"
|
|
fast_path_cycles += 1
|
|
elif isinstance(phases, dict) and "scheduler/no-ready" in phases:
|
|
no_ready_cycles += 1
|
|
effective_turns = sum(value == "effective" for value in outcomes.values())
|
|
prepare_errors = sum(value == "prepare_error" for value in outcomes.values())
|
|
timeouts = sum(value == "timeout" for value in outcomes.values())
|
|
total_cycles = len(outcomes)
|
|
recent_ids = sorted(outcomes)[-10:]
|
|
recent_effective = sum(outcomes[item] == "effective" for item in recent_ids)
|
|
recent_errors = sum(outcomes[item] == "prepare_error" for item in recent_ids)
|
|
recent_timeouts = sum(outcomes[item] == "timeout" for item in recent_ids)
|
|
return {
|
|
"window": window,
|
|
"sessionCycles": total_cycles,
|
|
"ready": ready,
|
|
"effectiveTurns": effective_turns,
|
|
"fastPathCycles": fast_path_cycles,
|
|
"noReadyCycles": no_ready_cycles,
|
|
"prepareErrors": prepare_errors,
|
|
"timeouts": timeouts,
|
|
"cycleEfficiency": percentage(effective_turns, total_cycles),
|
|
"lastWindow": len(recent_ids),
|
|
"lastEffectiveTurns": recent_effective,
|
|
"lastPrepareErrors": recent_errors,
|
|
"lastTimeouts": recent_timeouts,
|
|
"lastCycleEfficiency": percentage(recent_effective, len(recent_ids)),
|
|
"commandsStarted": command_started,
|
|
"commandsCompleted": command_completed,
|
|
"commandsFailed": command_failed,
|
|
"commandSuccess": percentage(command_completed - command_failed, command_completed),
|
|
"resolved": resolved,
|
|
"averageSeconds": (sum(durations) / len(durations)) if durations else None,
|
|
}
|
|
|
|
|
|
def recent_timing(current_cycle: int | None, cycles_run: int | None) -> dict[str, object]:
|
|
"""Aggregate command durations emitted by the streaming scheduler."""
|
|
# Timing is for bottleneck diagnosis, not lifetime accounting. Keep this
|
|
# bounded so every status query does not rescan hundreds of old cycles.
|
|
try:
|
|
window = max(1, min(50, int(os.environ.get("FREECAD_TIMING_WINDOW", "20"))))
|
|
except ValueError:
|
|
window = 20
|
|
cached = read_json(ROOT / "docs/current/freecad-web-scheduler-timing.json")
|
|
if cached.get("schema") in {"freecad-web-scheduler-timing/v1", "freecad-web-scheduler-timing/v2"} and isinstance(cached.get("cycles"), list):
|
|
records = [item for item in cached["cycles"] if isinstance(item, dict)]
|
|
if current_cycle is not None:
|
|
records = [item for item in records if int(item.get("cycle", 0) or 0) <= current_cycle]
|
|
records = records[-window:]
|
|
phases: dict[str, dict[str, float | int]] = {}
|
|
slowest: list[dict[str, object]] = []
|
|
slowest_agent_gaps: list[dict[str, object]] = []
|
|
redundancy: dict[str, float] = {}
|
|
activity: dict[str, int] = {}
|
|
usage: dict[str, int] = {}
|
|
usage_cycles = 0
|
|
effective_cycle_seconds = 0.0
|
|
seen_batches: set[str] = set()
|
|
recovery_cycles = 0
|
|
recovery_seconds = 0.0
|
|
total = 0.0
|
|
timed = 0
|
|
unmeasured = 0
|
|
unaccounted = 0.0
|
|
accounted_total = 0.0
|
|
measured_cycles = 0
|
|
stream_failed_cycles = 0
|
|
budget_exceeded_cycles = 0
|
|
for record in records:
|
|
waiting = record.get("waitingTiming", {})
|
|
waiting_seconds = sum(float(value) for value in waiting.values())
|
|
effective_cycle_seconds += waiting_seconds
|
|
for phase, seconds in waiting.items():
|
|
row = phases.setdefault(phase, {"count": 0, "seconds": 0.0, "maxSeconds": 0.0, "failed": 0, "redundant": 0})
|
|
row["count"] += 1
|
|
row["seconds"] += float(seconds)
|
|
row["maxSeconds"] = max(row["maxSeconds"], float(seconds))
|
|
accounted_total += waiting_seconds
|
|
effective_cycle_seconds += float(
|
|
record.get(
|
|
"effectiveCycleSeconds",
|
|
float(record.get("turnSeconds", 0.0) or 0.0)
|
|
+ sum(float(value) for value in (record.get("preparationTiming", {}) if isinstance(record.get("preparationTiming"), dict) else {}).values())
|
|
+ float(record.get("postPipelineSeconds", 0.0) or 0.0),
|
|
) or 0.0
|
|
)
|
|
batch = record.get("batch", {}) if isinstance(record.get("batch"), dict) else {}
|
|
batch_id = str(batch.get("id") or "")
|
|
if batch_id:
|
|
if batch_id in seen_batches:
|
|
recovery_cycles += 1
|
|
recovery_seconds += float(record.get("turnSeconds", 0.0) or 0.0)
|
|
else:
|
|
seen_batches.add(batch_id)
|
|
command_seconds = float(record.get("commandSeconds", 0.0) or 0.0)
|
|
record_phase_seconds = sum(
|
|
float(seconds)
|
|
for phase, seconds in (record.get("phaseTiming", {}) if isinstance(record.get("phaseTiming"), dict) else {}).items()
|
|
if str(phase) != "event/stream"
|
|
)
|
|
record_preparation_seconds = sum(
|
|
float(seconds)
|
|
for seconds in (record.get("preparationTiming", {}) if isinstance(record.get("preparationTiming"), dict) else {}).values()
|
|
)
|
|
fallback_accounted = record_phase_seconds + record_preparation_seconds
|
|
has_legacy_event_stream = isinstance(record.get("phaseTiming"), dict) and "event/stream" in record.get("phaseTiming", {})
|
|
accounted_seconds = fallback_accounted if has_legacy_event_stream else float(record.get("accountedSeconds", fallback_accounted) or fallback_accounted)
|
|
accounted_total += accounted_seconds + float(record.get("postPipelineSeconds", 0.0) or 0.0)
|
|
if not has_legacy_event_stream:
|
|
accounted_total += record_preparation_seconds
|
|
command_count = int(record.get("commands", 0) or 0)
|
|
timed_count = int(record.get("timedCommands", 0) or 0)
|
|
total += command_seconds
|
|
timed += timed_count
|
|
unmeasured += max(0, command_count - timed_count)
|
|
if timed_count:
|
|
measured_cycles += 1
|
|
unaccounted += max(0.0, float(record.get("turnSeconds", 0.0) or 0.0) - accounted_seconds)
|
|
stream_failed_cycles += int(bool(record.get("streamFailed")))
|
|
budget_exceeded_cycles += int(bool(record.get("commandBudgetExceeded")))
|
|
for phase, seconds in (record.get("phaseTiming", {}) if isinstance(record.get("phaseTiming"), dict) else {}).items():
|
|
if str(phase) == "event/stream":
|
|
continue
|
|
step_stats = record.get("stepStats", {}) if isinstance(record.get("stepStats"), dict) else {}
|
|
current_stats = step_stats.get(str(phase), {}) if isinstance(step_stats.get(str(phase)), dict) else {}
|
|
row = phases.setdefault(str(phase), {"count": 0, "seconds": 0.0, "maxSeconds": 0.0, "failed": 0, "redundant": 0})
|
|
row["count"] = int(row["count"]) + int(current_stats.get("count", 1) or 1)
|
|
row["seconds"] = float(row["seconds"]) + float(seconds)
|
|
row["maxSeconds"] = max(float(row["maxSeconds"]), float(current_stats.get("maxSeconds", seconds) or seconds))
|
|
row["failed"] = int(row["failed"]) + int(current_stats.get("failed", 0) or 0)
|
|
row["redundant"] = int(row["redundant"]) + int(current_stats.get("redundant", 0) or 0)
|
|
for phase, seconds in (record.get("preparationTiming", {}) if isinstance(record.get("preparationTiming"), dict) else {}).items():
|
|
key = f"preparation/{phase}"
|
|
row = phases.setdefault(key, {"count": 0, "seconds": 0.0, "maxSeconds": 0.0, "failed": 0, "redundant": 0})
|
|
row["count"] = int(row["count"]) + 1
|
|
row["seconds"] = float(row["seconds"]) + float(seconds)
|
|
row["maxSeconds"] = max(float(row["maxSeconds"]), float(seconds))
|
|
for phase, seconds in (record.get("postPipelineTiming", {}) if isinstance(record.get("postPipelineTiming"), dict) else {}).items():
|
|
row = phases.setdefault(str(phase), {"count": 0, "seconds": 0.0, "maxSeconds": 0.0, "failed": 0, "redundant": 0})
|
|
row["count"] = int(row["count"]) + 1
|
|
row["seconds"] = float(row["seconds"]) + float(seconds)
|
|
row["maxSeconds"] = max(float(row["maxSeconds"]), float(seconds))
|
|
slowest.extend(record.get("slowestCommands", []) if isinstance(record.get("slowestCommands"), list) else [])
|
|
slowest_agent_gaps.extend(record.get("slowestAgentGaps", []) if isinstance(record.get("slowestAgentGaps"), list) else [])
|
|
for name, value in (record.get("redundancy", {}) if isinstance(record.get("redundancy"), dict) else {}).items():
|
|
if isinstance(value, (int, float)):
|
|
redundancy[str(name)] = redundancy.get(str(name), 0.0) + float(value)
|
|
for name, value in (record.get("activity", {}) if isinstance(record.get("activity"), dict) else {}).items():
|
|
if isinstance(value, int):
|
|
activity[str(name)] = activity.get(str(name), 0) + value
|
|
for name, value in (record.get("usage", {}) if isinstance(record.get("usage"), dict) else {}).items():
|
|
if isinstance(value, int):
|
|
usage[str(name)] = usage.get(str(name), 0) + value
|
|
if isinstance(record.get("usage"), dict) and record.get("usage"):
|
|
usage_cycles += 1
|
|
phase_total = sum(float(row["seconds"]) for row in phases.values())
|
|
for row in phases.values():
|
|
row["share"] = 100.0 * float(row["seconds"]) / phase_total if phase_total else 0.0
|
|
row["averageSeconds"] = float(row["seconds"]) / int(row["count"]) if row["count"] else 0.0
|
|
return {
|
|
"timedCommands": timed,
|
|
"unmeasuredCommands": unmeasured,
|
|
"totalSeconds": total,
|
|
"accountedSeconds": accounted_total,
|
|
"phaseTotalSeconds": phase_total,
|
|
"phases": sorted(phases.items(), key=lambda item: float(item[1]["seconds"]), reverse=True),
|
|
"slowestCycle": max(((int(item.get("cycle", 0)), float(item.get("commandSeconds", 0.0))) for item in records), key=lambda item: item[1], default=None),
|
|
"slowestCommands": sorted(slowest, key=lambda item: float(item.get("seconds", 0.0)), reverse=True)[:10],
|
|
"slowestAgentGaps": sorted(slowest_agent_gaps, key=lambda item: float(item.get("seconds", 0.0)), reverse=True)[:10],
|
|
"redundancy": redundancy,
|
|
"activity": activity,
|
|
"usage": usage,
|
|
"usageCycles": usage_cycles,
|
|
"averageEffectiveCycleSeconds": effective_cycle_seconds / len(records) if records else None,
|
|
"batchFlow": {"uniqueBatches": len(seen_batches), "recoveryCycles": recovery_cycles, "recoverySeconds": recovery_seconds},
|
|
"unaccountedSeconds": unaccounted if measured_cycles else None,
|
|
"measuredCycles": measured_cycles,
|
|
"streamFailedCycles": stream_failed_cycles,
|
|
"budgetExceededCycles": budget_exceeded_cycles,
|
|
"window": window,
|
|
}
|
|
floor = max(0, int(current_cycle or 0) - window + 1)
|
|
phases: dict[str, dict[str, float | int]] = {}
|
|
cycle_seconds: dict[int, float] = {}
|
|
cycle_command_seconds: dict[int, float] = {}
|
|
cycle_wall_started: dict[int, str] = {}
|
|
cycle_wall_completed: dict[int, str] = {}
|
|
stream_failed_cycles = 0
|
|
budget_exceeded_cycles = 0
|
|
timed_commands = 0
|
|
unmeasured_commands = 0
|
|
slowest_commands: list[dict[str, object]] = []
|
|
for path in sorted(RUN_LOG_DIR.glob("cycle-*.log")):
|
|
match = re.search(r"cycle-(\d+)\.log$", path.name)
|
|
if not match or int(match.group(1)) < floor:
|
|
continue
|
|
cycle = int(match.group(1))
|
|
try:
|
|
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
except OSError:
|
|
continue
|
|
for line in lines:
|
|
try:
|
|
event = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if event.get("type") == "scheduler.cycle.started":
|
|
value = event.get("startedAt")
|
|
if isinstance(value, str): cycle_wall_started[cycle] = value
|
|
elif event.get("type") == "scheduler.cycle.completed":
|
|
value = event.get("completedAt")
|
|
if isinstance(value, str): cycle_wall_completed[cycle] = value
|
|
stream_failed_cycles += int(bool(event.get("streamFailed")))
|
|
budget_exceeded_cycles += int(bool(event.get("commandBudgetExceeded")))
|
|
item = event.get("item") if isinstance(event.get("item"), dict) else {}
|
|
duration = item.get("schedulerDurationSeconds")
|
|
if item.get("type") != "command_execution":
|
|
continue
|
|
if not isinstance(duration, (int, float)):
|
|
unmeasured_commands += 1
|
|
continue
|
|
phase = str(item.get("schedulerPhase") or "other")
|
|
row = phases.setdefault(phase, {"count": 0, "seconds": 0.0, "maxSeconds": 0.0})
|
|
row["count"] = int(row["count"]) + 1
|
|
row["seconds"] = float(row["seconds"]) + float(duration)
|
|
row["maxSeconds"] = max(float(row["maxSeconds"]), float(duration))
|
|
cycle_seconds[cycle] = cycle_seconds.get(cycle, 0.0) + float(duration)
|
|
cycle_command_seconds[cycle] = cycle_command_seconds.get(cycle, 0.0) + float(duration)
|
|
timed_commands += 1
|
|
slowest_commands.append({"cycle": cycle, "phase": phase, "seconds": float(duration), "command": str(item.get("command") or "")[:500]})
|
|
total = sum(float(row["seconds"]) for row in phases.values())
|
|
for row in phases.values():
|
|
row["share"] = (100.0 * float(row["seconds"]) / total) if total else 0.0
|
|
row["averageSeconds"] = float(row["seconds"]) / int(row["count"]) if row["count"] else 0.0
|
|
ordered = sorted(phases.items(), key=lambda item: float(item[1]["seconds"]), reverse=True)
|
|
unaccounted = 0.0
|
|
measured_cycle_count = 0
|
|
for cycle, started in cycle_wall_started.items():
|
|
completed = cycle_wall_completed.get(cycle)
|
|
if not completed: continue
|
|
try:
|
|
wall_seconds = (datetime.fromisoformat(completed.replace("Z", "+00:00")) - datetime.fromisoformat(started.replace("Z", "+00:00"))).total_seconds()
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if cycle not in cycle_command_seconds: continue
|
|
measured_cycle_count += 1
|
|
unaccounted += max(0.0, wall_seconds - cycle_command_seconds.get(cycle, 0.0))
|
|
return {
|
|
"timedCommands": timed_commands,
|
|
"window": window,
|
|
"unmeasuredCommands": unmeasured_commands,
|
|
"totalSeconds": total,
|
|
"phases": ordered,
|
|
"slowestCycle": max(cycle_seconds.items(), key=lambda item: item[1], default=None),
|
|
"unaccountedSeconds": unaccounted if measured_cycle_count else None,
|
|
"measuredCycles": measured_cycle_count,
|
|
"streamFailedCycles": stream_failed_cycles,
|
|
"budgetExceededCycles": budget_exceeded_cycles,
|
|
"slowestCommands": sorted(slowest_commands, key=lambda item: float(item["seconds"]), reverse=True)[:10],
|
|
}
|
|
|
|
|
|
def descendants(pid: int) -> list[int]:
|
|
result: list[int] = []
|
|
pending = [pid]
|
|
while pending:
|
|
parent = pending.pop()
|
|
try:
|
|
children = [int(value) for value in Path(f"/proc/{parent}/task/{parent}/children").read_text().split()]
|
|
except (FileNotFoundError, ValueError):
|
|
continue
|
|
result.extend(children)
|
|
pending.extend(children)
|
|
return result
|
|
|
|
|
|
def process_session(pids: list[int]) -> Path | None:
|
|
candidates: list[Path] = []
|
|
for pid in pids:
|
|
fd_root = Path(f"/proc/{pid}/fd")
|
|
try:
|
|
fds = list(fd_root.iterdir())
|
|
except (FileNotFoundError, PermissionError):
|
|
continue
|
|
for fd in fds:
|
|
try:
|
|
target = Path(os.readlink(fd))
|
|
except (FileNotFoundError, OSError):
|
|
continue
|
|
if target.name.startswith("rollout-") and target.suffix == ".jsonl":
|
|
candidates.append(target)
|
|
return max(candidates, key=lambda path: path.stat().st_mtime, default=None)
|
|
|
|
|
|
def show_legacy_activity(pid: int) -> None:
|
|
children = descendants(pid)
|
|
if children:
|
|
print(f" childProcesses: {','.join(map(str, children))}")
|
|
session = process_session(children)
|
|
if not session:
|
|
print(" liveActivity: legacy scheduler; waiting for streaming scheduler handoff")
|
|
return
|
|
print(f" liveSession: {session}")
|
|
recent: deque[dict] = deque(maxlen=6)
|
|
last_timestamp = None
|
|
with session.open(encoding="utf-8", errors="replace") as stream:
|
|
for line in stream:
|
|
try:
|
|
event = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
last_timestamp = event.get("timestamp") or last_timestamp
|
|
payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
|
|
if event.get("type") == "response_item" and payload.get("type") == "custom_tool_call":
|
|
recent.append(payload)
|
|
print(f" lastActivityAt: {last_timestamp or '-'}")
|
|
if recent:
|
|
print(" recent dispatched calls:")
|
|
for payload in recent:
|
|
detail = " ".join(str(payload.get("input", "")).split())[:300]
|
|
print(f" {payload.get('name', 'tool')}: {detail}")
|
|
|
|
|
|
def main() -> int:
|
|
status = read_json(STATUS_PATH)
|
|
failure_state = read_json(FAILURE_STATE_PATH)
|
|
watchdog_status = read_json(WATCHDOG_STATUS_PATH)
|
|
checkpoint = read_json(CHECKPOINT_PATH)
|
|
task_state = read_json(TASK_STATE_PATH)
|
|
python_gate = read_json(PYTHON_GATE_PATH)
|
|
python_api_manifest = read_json(PYTHON_API_MANIFEST_PATH)
|
|
source_api_inventory = read_json(SOURCE_API_INVENTORY_PATH)
|
|
feature_capabilities = read_json(FEATURE_CAPABILITY_PATH)
|
|
implementation_registry = read_json(IMPLEMENTATION_REGISTRY_PATH)
|
|
pending_builds = read_json(PENDING_BUILD_PATH)
|
|
module_batch = read_json(MODULE_BATCH_PATH)
|
|
runtime_issues = read_json(RUNTIME_ISSUE_PATH)
|
|
parity_gate = read_json(FULL_PARITY_GATE_PATH)
|
|
business_audit = read_json(BUSINESS_AUDIT_PATH)
|
|
progress = progress_snapshot(
|
|
task_state,
|
|
python_gate,
|
|
parity_gate,
|
|
python_api_manifest,
|
|
source_api_inventory,
|
|
feature_capabilities,
|
|
implementation_registry,
|
|
runtime_issues,
|
|
)
|
|
completed_batches = checkpoint.get("latestBatch", {}).get("completed", [])
|
|
completed_batch_count = len(completed_batches) if isinstance(completed_batches, list) else 0
|
|
try:
|
|
current_cycle = int(status.get("cycle")) if status.get("cycle") is not None else None
|
|
except (TypeError, ValueError):
|
|
current_cycle = None
|
|
try:
|
|
cycles_run = int(status.get("cyclesRun")) if status.get("cyclesRun") is not None else None
|
|
except (TypeError, ValueError):
|
|
cycles_run = None
|
|
efficiency = recent_efficiency(current_cycle, cycles_run)
|
|
timing = recent_timing(current_cycle, cycles_run)
|
|
try:
|
|
pid = int(PID_PATH.read_text(encoding="ascii").strip())
|
|
except (FileNotFoundError, ValueError):
|
|
pid = None
|
|
|
|
print("FreeCAD Web scheduler")
|
|
running = process_matches(pid, "run_freecad_web_scheduler.py")
|
|
print(f" scheduler: {pid or '-'} ({'running' if running else 'not running'})")
|
|
watchdog_pid = None
|
|
try:
|
|
watchdog_pid = int(WATCHDOG_PID_PATH.read_text(encoding="ascii").strip())
|
|
except (FileNotFoundError, ValueError):
|
|
pass
|
|
watchdog_running = process_matches(watchdog_pid, "watch_freecad_web_scheduler.py")
|
|
print(f" watchdog: {watchdog_pid or '-'} ({'running' if watchdog_running else 'not running'})")
|
|
maintenance = read_json(ROOT / "docs/current/scheduler-maintenance.json")
|
|
if maintenance:
|
|
for key in ("enabled", "intervalSeconds", "status", "runCount", "lastOutcome", "startedAt", "lastFinishedAt",
|
|
"lastDurationSeconds", "lastCommandsCompleted", "lastCommandsFailed", "lastFileChanges",
|
|
"lastReportPath", "lastResultPath", "lastLog", "lastError"):
|
|
if key in maintenance:
|
|
print(f" maintenance.{key}: {scalar(maintenance[key])}")
|
|
if maintenance.get("enabled") and maintenance.get("nextRunAt"):
|
|
next_run = datetime.fromtimestamp(float(maintenance["nextRunAt"]), timezone.utc).isoformat()
|
|
print(f" maintenance.nextRunAt: {next_run}")
|
|
if not running and status.get("status") not in {"terminated", "stop_signal", "max_cycles"}:
|
|
print(f" observedState: stale (last heartbeat {scalar(status.get('heartbeatAt'))})")
|
|
strategy = checkpoint.get("implementationStrategy") if isinstance(checkpoint.get("implementationStrategy"), dict) else {}
|
|
execution_policy = read_json(ROOT / "docs/current/execution-policy.json")
|
|
manual_validation = execution_policy.get("mode") == "implementation-only-user-validation" and execution_policy.get("automaticValidation") is False
|
|
if strategy:
|
|
print(f" implementationStrategy: {scalar(strategy.get('mode'))}")
|
|
print(f" executionUnit: {scalar(strategy.get('executionUnit', 'feature-capability'))}")
|
|
print(f" implementationPlanner: {scalar(strategy.get('planner'))}")
|
|
print(f" validationMode: {'user-during-use (automatic validation disabled)' if manual_validation else scalar(strategy.get('verification'))}")
|
|
if manual_validation:
|
|
print(" executionPolicy: connect-all-functions-first; preserve-artifacts; no-acceptance-queue")
|
|
if watchdog_status:
|
|
for key in ("status", "restartCount", "lastRestartAt", "lastRestartReason", "lastSchedulerPid", "lastSchedulerExitCode", "intervalSeconds", "automaticRepairCount", "lastAutomaticRepair"):
|
|
if key in watchdog_status:
|
|
print(f" watchdog.{key}: {scalar(watchdog_status[key])}")
|
|
if running and not status:
|
|
show_legacy_activity(pid)
|
|
for key in (
|
|
"implementation",
|
|
"executionMode",
|
|
"reasoningEffort",
|
|
"promptBytes",
|
|
"promptLines",
|
|
"sessionMode",
|
|
"sessionThreadId",
|
|
"sessionTurns",
|
|
"sessionReuseLimit",
|
|
"sessionPredictedInputTokens",
|
|
"sessionSaved",
|
|
"idleContextSaved",
|
|
"recoveryMode",
|
|
"status",
|
|
"cycle",
|
|
"activeTask",
|
|
"childPid",
|
|
"turnStartedAt",
|
|
"heartbeatAt",
|
|
"lastActivityAt",
|
|
"lastEvent",
|
|
"currentItemType",
|
|
"currentCommand",
|
|
"currentStepPhase",
|
|
"lastCommand",
|
|
"lastStepPhase",
|
|
"lastStepDurationSeconds",
|
|
"currentTurnElapsedSeconds",
|
|
"schedulerSourceSha256",
|
|
"modelTransport",
|
|
"transportActivity",
|
|
"cancelledModelRequests",
|
|
"contextBudgetExceeded",
|
|
"configuredFirstActivitySeconds",
|
|
"configuredFirstActivityMaxSeconds",
|
|
"firstActivityTimeoutSeconds",
|
|
"firstActivityTimedOut",
|
|
"failureKind",
|
|
"currentAgentGapSeconds",
|
|
"phaseTiming",
|
|
"preparationTiming",
|
|
"implementationPacketStatus",
|
|
"implementationPacketOutput",
|
|
"implementationPacketRefreshStatus",
|
|
"implementationPacketRefreshSeconds",
|
|
"preTurnPipelineStatus",
|
|
"localPipelineDuringBackoff",
|
|
"preTurnPipelineSeconds",
|
|
"preTurnPipelineOutput",
|
|
"postTurnPipelineStatus",
|
|
"postTurnPipelineSeconds",
|
|
"postTurnPipelineOutput",
|
|
"lastCommandExitCode",
|
|
"lastCommandStatus",
|
|
"lastCommandOutput",
|
|
"lastAgentMessage",
|
|
"commandsStarted",
|
|
"commandsCompleted",
|
|
"commandsFailed",
|
|
"sourceReads",
|
|
"sourceReadBudget",
|
|
"sourceReadHardLimit",
|
|
"sourceReadBudgetExceeded",
|
|
"explorationBudgetSeconds",
|
|
"explorationBudgetExceeded",
|
|
"lastRejectedSourceCommand",
|
|
"fileChanges",
|
|
"observedSourceChanges",
|
|
"lastChangedSources",
|
|
"sourceChangeScanSeconds",
|
|
"agentMessages",
|
|
"commandBudget",
|
|
"commandBudgetExceeded",
|
|
"commandBudgetStopRequested",
|
|
"streamErrorCount",
|
|
"streamErrorLimit",
|
|
"streamErrorWindowSeconds",
|
|
"streamRetryCount",
|
|
"streamRetryDelaySeconds",
|
|
"providerBackoffRemainingSeconds",
|
|
"streamFailureReason",
|
|
"repeatedFailureReason",
|
|
"patchConflictCount",
|
|
"patchConflictLimit",
|
|
"redundancy",
|
|
"noReadyBatch",
|
|
"progressOutcome",
|
|
"progressedRecords",
|
|
"deferredRecords",
|
|
"retryAt",
|
|
"rejectedBatchId",
|
|
"cooldownReplanStatus",
|
|
"automaticRepair",
|
|
"moduleRecovery",
|
|
"failureReconciliation",
|
|
"agentIdleSeconds",
|
|
"agentIdleLimitSeconds",
|
|
"postEditAutoFinalize",
|
|
"postEditIdleSeconds",
|
|
"postEditFinalizeSeconds",
|
|
"forbiddenImplementationCommand",
|
|
"eventCount",
|
|
"unresolvedFailures",
|
|
"failureRecovery",
|
|
"lastFailureKey",
|
|
"lastFailureCount",
|
|
"moduleBatchId",
|
|
"moduleBatchModule",
|
|
"moduleBatchTier",
|
|
"moduleBatchFamily",
|
|
"moduleBatchRecords",
|
|
"moduleBatchManifest",
|
|
"batchId",
|
|
"batchSelection",
|
|
"lastError",
|
|
"liveLog",
|
|
"lastLog",
|
|
):
|
|
if key in status:
|
|
print(f" {key}: {scalar(status[key])}")
|
|
context = read_json(ROOT / "docs/current/scheduler-context.json")
|
|
if context:
|
|
usage = context.get("requestUsage", {})
|
|
print(f" context: cycle={context.get('cycle')} batch={context.get('batchId')} status={context.get('status')} mode={context.get('sessionMode')}")
|
|
print(f" promptBytes: {context.get('promptBytes')}/{context.get('promptByteLimit')} compacted={context.get('contextCompacted')}")
|
|
print(f" requestInputTokens: last={usage.get('lastInputTokens')} peak={usage.get('peakInputTokens')} measured={usage.get('measuredRequests', 0)}/{usage.get('requests', 0)} complete={usage.get('complete', False)}")
|
|
print(f" checkpoint: docs/current/scheduler-context.json packet={context.get('packetPath')}")
|
|
unattended = read_json(PROGRESS_PATH)
|
|
business_efficiency = progress_summary(unattended)
|
|
if unattended:
|
|
history = unattended.get("history", [])[-20:]
|
|
cooling = [record_id for record_id, row in unattended.get("records", {}).items() if float(row.get("retryAt", 0)) > time.time()]
|
|
print(f" unattendedProgress: cycles={len(history)} withNewOutcome={sum(row.get('outcome') == 'progress' for row in history)} noOutcome={sum(row.get('outcome') in {'no_progress', 'source_edits_pending_outcome'} for row in history)} providerWait={sum(row.get('outcome') == 'provider_wait' for row in history)} coolingRecords={len(cooling)}")
|
|
print(f" coolingModules: {json.dumps(cooling_modules(state=unattended), ensure_ascii=False)}")
|
|
batch_sizing = module_batch.get("batchSizing") if isinstance(module_batch.get("batchSizing"), dict) else {}
|
|
if batch_sizing:
|
|
print(
|
|
" batchSizing: "
|
|
f"default={scalar(batch_sizing.get('default'))} "
|
|
f"selected={scalar(batch_sizing.get('selected'))} "
|
|
f"reason={scalar(batch_sizing.get('reason'))} "
|
|
f"stallSeconds={scalar(batch_sizing.get('stallSeconds'))}"
|
|
)
|
|
counts = task_state.get("counts", {})
|
|
if counts:
|
|
print(
|
|
" ledger: "
|
|
+ ", ".join(f"{key}={counts.get(key, 0)}" for key in ("completed", "in_progress", "planned", "blocked"))
|
|
)
|
|
print(" progress:")
|
|
business_progress = business_audit.get("businessProgress", {}) if isinstance(business_audit.get("businessProgress"), dict) else {}
|
|
if business_progress:
|
|
print(
|
|
f" {'historicalAcceptanceNotSchedulingGate' if manual_validation else 'functionalModuleBusiness'}: {business_progress.get('completed', 0)}/{business_progress.get('total', 0)} "
|
|
f"({percentage(business_progress.get('completed', 0), business_progress.get('total', 0))}); "
|
|
f"remaining={business_progress.get('remaining', 0)}"
|
|
)
|
|
implementation_progress = business_audit.get("implementationProgress", {})
|
|
if implementation_progress:
|
|
print(f" reusableImplementationReceipts: {implementation_progress.get('completed', 0)}/{implementation_progress.get('total', 0)}; remaining={implementation_progress.get('remaining', 0)}")
|
|
print(
|
|
f" legacyTaskCards: {progress['taskCompleted']}/{progress['taskTotal']} "
|
|
f"({percentage(progress['taskCompleted'], progress['taskTotal'])})"
|
|
)
|
|
print(
|
|
f" historicalModuleGates: {progress['moduleCompleted']}/{progress['moduleTotal']} "
|
|
f"({percentage(progress['moduleCompleted'], progress['moduleTotal'])})"
|
|
)
|
|
print(
|
|
f" featureRoutes: {progress['featureRouted']}/{progress['featureCommands']} "
|
|
f"({percentage(progress['featureRouted'], progress['featureCommands'])})"
|
|
)
|
|
print(
|
|
f" featureInteractive: {progress['featureInteractive']}/{progress['featureCommands']} "
|
|
f"({percentage(progress['featureInteractive'], progress['featureCommands'])}); "
|
|
f"needsParameterUi={progress['featureNeedsUi']} needsReactWorkflow={progress['featureNeedsReact']} "
|
|
f"implementationRequired={progress['featureImplementationRequired']} validationDeferred={progress['featureValidationDeferred']}"
|
|
)
|
|
print(
|
|
f" featureArtifacts: {progress['featureArtifactModules']}/{progress['featureModules']} modules "
|
|
f"({percentage(progress['featureArtifactModules'], progress['featureModules'])})"
|
|
)
|
|
print(
|
|
f" implementationReceipts: batches={progress['implementationReceiptBatches']} "
|
|
f"records={progress['implementationReceiptRecords']} current "
|
|
f"(historical={progress['implementationReceiptHistoricalRecords']}) "
|
|
f"moduleBusinessRecords={progress['businessImplementationRecords']} "
|
|
f"runtimeIssues={progress['runtimeIssuesUnresolved']}/{progress['runtimeIssues']} unresolved"
|
|
)
|
|
pending_targets = pending_builds.get("targets") if isinstance(pending_builds.get("targets"), dict) else {}
|
|
pending_batch_count = sum(len(value) for value in pending_targets.values() if isinstance(value, list))
|
|
print(f" pendingBuilds: targets={len(pending_targets)} batches={pending_batch_count}")
|
|
print(
|
|
f" apiTraceabilityOnly: records={progress['apiTotal']} sourceInventory={progress['sourceApiTotal']} "
|
|
"(not an implementation queue or completion metric)"
|
|
)
|
|
print(
|
|
f" legacyEvidenceBatches: completed={completed_batch_count} "
|
|
f"current={status.get('moduleBatchId', '-') } records={status.get('moduleBatchRecords', 0)} "
|
|
"(does not close the aggregate Y task card)"
|
|
)
|
|
print(
|
|
f" deferredValidationHistory: areas={progress['parityAreas']} pending={progress['parityPending']} "
|
|
"(disabled; nonblocking for functional-module implementation)"
|
|
)
|
|
print(" efficiency:")
|
|
if "preparationSeconds" in status:
|
|
print(f" latestPreparationSeconds: {scalar(status['preparationSeconds'])}")
|
|
if "lastTurnDurationSeconds" in status:
|
|
# This field is finalized when a turn exits; while a turn is running
|
|
# it still contains the previous completed turn's duration.
|
|
print(f" lastCompletedTurnSeconds: {scalar(status['lastTurnDurationSeconds'])}")
|
|
if "currentTurnElapsedSeconds" in status:
|
|
print(f" currentTurnElapsedSeconds: {scalar(status['currentTurnElapsedSeconds'])}")
|
|
live_phase_timing = status.get("phaseTiming")
|
|
if isinstance(live_phase_timing, dict) and live_phase_timing:
|
|
print(" currentTurnPhaseTiming:")
|
|
for phase, seconds in sorted(live_phase_timing.items(), key=lambda item: float(item[1]), reverse=True):
|
|
print(f" {phase}: {float(seconds):.3f}s")
|
|
elapsed = status.get("currentTurnElapsedSeconds")
|
|
if isinstance(elapsed, (int, float)):
|
|
measured = sum(float(value) for value in live_phase_timing.values() if isinstance(value, (int, float)))
|
|
print(f" unaccountedCurrentSeconds: {max(0.0, float(elapsed) - measured):.1f}s")
|
|
print(
|
|
f" schedulerSession: window={efficiency['window']} completed={efficiency['sessionCycles']} exitedNormally={efficiency['effectiveTurns']} fastPath={efficiency.get('fastPathCycles', 0)} noReady={efficiency.get('noReadyCycles', 0)} "
|
|
f"prepareErrors={efficiency['prepareErrors']} timeouts={efficiency['timeouts']} processCompletionRate={efficiency['cycleEfficiency']} "
|
|
f"readyPackets={efficiency['ready']}"
|
|
)
|
|
print(f" activityProgressRate: {percentage(business_efficiency['progressCycles'], business_efficiency['cycles'])}; cyclesWithChangedOutcome={business_efficiency['progressCycles']}/{business_efficiency['cycles']}; noProgressSeconds={business_efficiency['noProgressSeconds']}")
|
|
print(f" businessDelivery: firstPasses={business_efficiency['firstBusinessPasses']} evidenceRefreshes={business_efficiency['businessRefreshes']} classifiedCycles={business_efficiency['deliveryMeasuredCycles']}/{business_efficiency['cycles']} (unclassified history is not new delivery)")
|
|
print(
|
|
f" last10Cycles: exitedNormally={efficiency['lastEffectiveTurns']} "
|
|
f"prepareErrors={efficiency['lastPrepareErrors']} timeouts={efficiency['lastTimeouts']} "
|
|
f"processCompletionRate={efficiency['lastCycleEfficiency']}"
|
|
)
|
|
print(
|
|
f" commands: started={efficiency['commandsStarted']} completed={efficiency['commandsCompleted']} "
|
|
f"failed={efficiency['commandsFailed']} success={efficiency['commandSuccess']} "
|
|
f"resolvedEventsInLogTail={efficiency['resolved']}"
|
|
)
|
|
if efficiency["averageSeconds"] is not None:
|
|
print(f" averageTurnSeconds: {efficiency['averageSeconds']:.1f}")
|
|
if timing.get("averageEffectiveCycleSeconds") is not None:
|
|
print(f" averageEffectiveCycleSeconds: {float(timing['averageEffectiveCycleSeconds']):.1f}")
|
|
batch_flow = timing.get("batchFlow", {}) if isinstance(timing.get("batchFlow"), dict) else {}
|
|
if batch_flow:
|
|
print(
|
|
f" batchFlow: unique={int(batch_flow.get('uniqueBatches', 0))} "
|
|
f"recoveryCycles={int(batch_flow.get('recoveryCycles', 0))} "
|
|
f"recoverySeconds={float(batch_flow.get('recoverySeconds', 0.0)):.1f}"
|
|
)
|
|
activity = timing.get("activity", {}) if isinstance(timing.get("activity"), dict) else {}
|
|
if activity:
|
|
print(
|
|
f" activity: commands={int(activity.get('commands', 0))} "
|
|
f"failures={int(activity.get('commandFailures', 0))} "
|
|
f"sourceReads={int(activity.get('sourceReads', 0))} "
|
|
f"fileChanges={int(activity.get('fileChanges', 0))} "
|
|
f"agentMessages={int(activity.get('agentMessages', 0))} "
|
|
f"patchConflicts={int(activity.get('patchConflicts', 0))} "
|
|
f"sessionsResumed={int(activity.get('sessionResumed', 0))}"
|
|
)
|
|
usage = timing.get("usage", {}) if isinstance(timing.get("usage"), dict) else {}
|
|
if usage:
|
|
print(
|
|
f" tokenUsage: cycles={int(timing.get('usageCycles', 0))} "
|
|
f"input={int(usage.get('input_tokens', 0))} "
|
|
f"cachedInput={int(usage.get('cached_input_tokens', 0))} "
|
|
f"output={int(usage.get('output_tokens', 0))} "
|
|
f"reasoningOutput={int(usage.get('reasoning_output_tokens', 0))}"
|
|
)
|
|
unaccounted = timing.get("unaccountedSeconds")
|
|
unaccounted_text = f"{float(unaccounted):.1f}" if isinstance(unaccounted, (int, float)) else "unknown"
|
|
print(f" timingWindow: {timing.get('window', 20)} cycles")
|
|
print(f" timedCommands: {timing['timedCommands']} totalCommandSeconds: {timing['totalSeconds']:.1f} accountedSeconds={float(timing.get('accountedSeconds', timing['totalSeconds'])):.1f} phaseTotalSeconds={float(timing.get('phaseTotalSeconds', timing.get('accountedSeconds', timing['totalSeconds']))):.1f} unmeasuredCommands: {timing.get('unmeasuredCommands', 0)} unaccountedSeconds: {unaccounted_text} measuredCycles: {timing.get('measuredCycles', 0)} streamFailedCycles={timing.get('streamFailedCycles', 0)} budgetExceededCycles={timing.get('budgetExceededCycles', 0)}")
|
|
asset_timing = read_json(ASSET_SYNC_TIMING_PATH)
|
|
if asset_timing.get("schema") == "freecad-web-assets-sync-timing/v1":
|
|
print(
|
|
" latestArtifactSync: "
|
|
f"total={float(asset_timing.get('totalSeconds', 0.0)):.3f}s "
|
|
f"copy={float(asset_timing.get('copySeconds', 0.0)):.3f}s "
|
|
f"hash={float(asset_timing.get('hashSeconds', 0.0)):.3f}s "
|
|
f"manifest={float(asset_timing.get('manifestSeconds', 0.0)):.3f}s "
|
|
f"copied={int(asset_timing.get('copied', 0))} skipped={int(asset_timing.get('skipped', 0))} "
|
|
f"hashComputed={int(asset_timing.get('hashComputed', 0))} "
|
|
f"hashCacheHits={int(asset_timing.get('hashCacheHits', 0))} "
|
|
f"missing={int(asset_timing.get('missing', 0))} "
|
|
f"buildProfile={asset_timing.get('buildProfile', '-') }"
|
|
)
|
|
module_build = read_json(MODULE_BUILD_PATH)
|
|
if module_build.get("schema") == "freecad-web-module-build/v1":
|
|
targets = module_build.get("targets", [])
|
|
durations = [float(item.get("durationSeconds", 0.0)) for item in targets if isinstance(item, dict)]
|
|
if durations:
|
|
print(
|
|
" latestModuleBuild: "
|
|
f"total={sum(durations):.3f}s targets={len(durations)} "
|
|
f"max={max(durations):.3f}s passed={int(module_build.get('summary', {}).get('passed', 0))} "
|
|
f"failed={int(module_build.get('summary', {}).get('failed', 0))}"
|
|
)
|
|
if timing["phases"]:
|
|
print(" phaseTiming:")
|
|
for phase, row in timing["phases"]:
|
|
failure_text = f" failed={int(row.get('failed', 0))}" if int(row.get("failed", 0)) else ""
|
|
redundant_text = f" redundant={int(row.get('redundant', 0))}" if int(row.get("redundant", 0)) else ""
|
|
print(
|
|
f" {phase}: total={float(row['seconds']):.3f}s share={float(row['share']):.1f}% "
|
|
f"count={int(row['count'])} avg={float(row['averageSeconds']):.3f}s max={float(row['maxSeconds']):.3f}s"
|
|
f"{failure_text}{redundant_text}"
|
|
)
|
|
redundancy = timing.get("redundancy", {}) if isinstance(timing.get("redundancy"), dict) else {}
|
|
if redundancy and any(float(value) for value in redundancy.values() if isinstance(value, (int, float))):
|
|
print(
|
|
" redundantWork: "
|
|
f"commands={int(redundancy.get('duplicateCommands', 0))} "
|
|
f"seconds={float(redundancy.get('duplicateCommandSeconds', 0.0)):.3f} "
|
|
f"reads={int(redundancy.get('duplicateReads', 0))} "
|
|
f"gates={int(redundancy.get('duplicateGates', 0))} "
|
|
f"builds={int(redundancy.get('duplicateBuilds', 0))} "
|
|
f"patchConflicts={int(redundancy.get('patchConflicts', 0))}"
|
|
)
|
|
if timing.get("slowestAgentGaps"):
|
|
print(" slowestAgentGaps:")
|
|
for item in timing["slowestAgentGaps"][:5]:
|
|
print(
|
|
f" cycle={item.get('cycle', '-')} phase={item.get('phase', 'agent/decision')} "
|
|
f"seconds={float(item.get('seconds', 0.0)):.2f} after={item.get('after', '-')} before={item.get('before', '-')}"
|
|
)
|
|
if timing["slowestCycle"]:
|
|
cycle, seconds = timing["slowestCycle"]
|
|
print(f" slowestMeasuredCycle: {cycle} commandSeconds={seconds:.1f}")
|
|
if timing.get("slowestCommands"):
|
|
print(" slowestCommands:")
|
|
for item in timing["slowestCommands"][:5]:
|
|
print(f" cycle={item.get('cycle', '-')} phase={item.get('phase', 'other')} seconds={float(item.get('seconds', 0.0)):.2f} command={item.get('command', '')}")
|
|
agent_seconds = sum(float(row.get("seconds", 0.0)) for phase, row in timing.get("phases", []) if str(phase).startswith("agent/"))
|
|
phase_seconds = float(timing.get("phaseTotalSeconds", 0.0) or 0.0)
|
|
agent_share = 100.0 * agent_seconds / phase_seconds if phase_seconds else 0.0
|
|
if business_efficiency["cycles"]:
|
|
verdict = business_efficiency["verdict"]
|
|
elif efficiency["sessionCycles"] == 0:
|
|
verdict = "RUNNING: current time slice has not completed yet"
|
|
elif efficiency["lastTimeouts"] > 0:
|
|
verdict = "LOW: one or more latest turns hit the timeout budget"
|
|
elif efficiency["effectiveTurns"] == 0 and efficiency["prepareErrors"] > 0:
|
|
verdict = "LOW: recent cycles are blocked in preparation; no effective agent turns"
|
|
elif efficiency["cycleEfficiency"] != "-" and float(str(efficiency["cycleEfficiency"]).rstrip("%")) < 50:
|
|
verdict = "LOW: preparation/failure overhead exceeds effective turns"
|
|
elif agent_share >= 70.0 and efficiency["averageSeconds"] is not None and efficiency["averageSeconds"] >= 180.0:
|
|
verdict = f"MIXED: scheduler is healthy, but agent decision/edit time is {agent_share:.1f}% of measured work"
|
|
elif efficiency["commandsFailed"] > 0:
|
|
verdict = "MIXED: turns are running, but command failures remain"
|
|
elif efficiency["lastEffectiveTurns"] > 0 and efficiency["lastPrepareErrors"] == 0:
|
|
verdict = "GOOD: latest cycles are entering implementation without preparation errors"
|
|
else:
|
|
verdict = "GOOD: effective turns are progressing without recorded command failures"
|
|
print(f" verdict: {verdict}")
|
|
failures = failure_state.get("failures", {})
|
|
if isinstance(failures, dict):
|
|
records = [item for item in failures.values() if isinstance(item, dict)]
|
|
elif isinstance(failures, list):
|
|
records = [item for item in failures if isinstance(item, dict)]
|
|
else:
|
|
records = []
|
|
pending = [item for item in records if item.get("resolved") is not True]
|
|
reconciliation = failure_summary()
|
|
print(f" failureLedger: originalRecords={len(records)} originalOpen={len(pending)}")
|
|
print(f" failureReconciliation: pending={reconciliation['pending']} closed={reconciliation['closed']} unclassified={reconciliation['unclassified']} finalReady={reconciliation['complete']}")
|
|
patterns = read_json(FAILURE_PATTERN_PATH).get("patterns", [])
|
|
if isinstance(patterns, list) and patterns:
|
|
print(" failurePatterns:")
|
|
for pattern in patterns[:5]:
|
|
if isinstance(pattern, dict):
|
|
print(
|
|
f" {pattern.get('category', 'other')}: occurrences={pattern.get('occurrences', 0)} "
|
|
f"unresolved={pattern.get('unresolved', 0)}"
|
|
)
|
|
for item in sorted(pending, key=lambda value: (-int(value.get("count", 0)), str(value.get("lastSeenAt", ""))))[:5]:
|
|
print(
|
|
f" {item.get('key')}: count={item.get('count')} exit={item.get('lastExitCode')} "
|
|
f"task={item.get('lastTask')} command={str(item.get('command', ''))[:220]}"
|
|
)
|
|
print(f" checkpoint: phase={checkpoint.get('phase', '-')} mustContinue={checkpoint.get('mustContinue', '-')}")
|
|
|
|
live_log = status.get("liveLog") or status.get("lastLog")
|
|
if isinstance(live_log, str):
|
|
log_path = ROOT / live_log
|
|
if log_path.exists():
|
|
print(" recent events:")
|
|
with log_path.open(encoding="utf-8", errors="replace") as stream:
|
|
lines = deque(stream, maxlen=8)
|
|
for line in lines:
|
|
try:
|
|
event = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
print(f" {line[:240]}")
|
|
continue
|
|
event_type = event.get("type", "unknown")
|
|
item = event.get("item") if isinstance(event.get("item"), dict) else {}
|
|
detail = item.get("command") or item.get("text") or event.get("text") or event.get("exitCode")
|
|
print(f" {event_type}: {scalar(detail)[:240]}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|