57 lines
3.0 KiB
Python
57 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare recorded scheduling timings with real read-only context checks."""
|
|
import contextlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
import tempfile
|
|
|
|
from wbv2_autopilot import Autopilot, CONTROL, POLICY, ROOT, read_json, write_json
|
|
from wbv2_timing import timing_report
|
|
|
|
|
|
def main():
|
|
policy = read_json(ROOT / POLICY)
|
|
before = timing_report(ROOT / CONTROL)
|
|
samples = []
|
|
with tempfile.TemporaryDirectory(prefix="wbv2-timing-") as temporary:
|
|
root = Path(temporary)
|
|
runner = Autopilot(root, policy)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
for script in ["check-task-context.mjs", "check-context-governance.mjs"]:
|
|
for iteration in range(3):
|
|
result = runner.run_process(["node", str(ROOT / "tools/web" / script)], root, root / f"{script}-{iteration}.log", 30, phase="context_check")
|
|
if result["exitCode"] != 0:
|
|
raise RuntimeError(f"context benchmark failed: {script}")
|
|
samples.append({"script": script, "seconds": result["durationSeconds"]})
|
|
old = [row["seconds"] for row in before["steps"] if row["phase"] == "context_check"]
|
|
median_old = statistics.median(old)
|
|
median_new = statistics.median(sample["seconds"] for sample in samples)
|
|
report = {
|
|
"schemaVersion": 1,
|
|
"operation": "AUTOPILOT_PERFORMANCE_OPTIMIZATION",
|
|
"observedAt": timing_report(ROOT / CONTROL)["observedAt"],
|
|
"timezone": "Asia/Shanghai",
|
|
"labelPrecision": "minute",
|
|
"baseline": before,
|
|
"benchmark": {
|
|
"scope": "Same read-only context entrypoints: historical observed baseline versus three current samples per entrypoint; not whole-project throughput.",
|
|
"samples": samples,
|
|
"baselineMedianMinutes": round(median_old / 60, 6),
|
|
"optimizedMedianMinutes": round(median_new / 60, 6),
|
|
"observedSpeedup": round(median_old / median_new, 2),
|
|
"observedReductionPercent": round((1 - median_new / median_old) * 100, 2),
|
|
},
|
|
"providerPolicy": {key: policy[key] for key in ["modelStartTimeoutSeconds", "modelIdleTimeoutSeconds", "providerBackoffSeconds", "providerMaxBackoffSeconds", "maxProviderFailures"]},
|
|
"sourcePolicy": {"driver": policy["strategy"], "authority": policy["sourceAuthority"], "sourceFirst": policy["sourceFirst"]},
|
|
"limitations": ["Provider availability/account concurrency is external; these changes do not remove upstream limits.", "No source capability completion or inventory increment is claimed.", "Legacy JSONL lacks per-item timestamps; new runs record observed per-item intervals separately."],
|
|
}
|
|
destination = ROOT / "tests/golden/WBV2-P1424-001/performance-optimization-report.json"
|
|
write_json(destination, report)
|
|
print(json.dumps({"report": str(destination.relative_to(ROOT)), "observedAt": report["observedAt"], **report["benchmark"]}, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|