92 lines
6.2 KiB
Python
92 lines
6.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare fresh business execution with reuse of the same verified evidence."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--module", required=True)
|
|
parser.add_argument("--ids", nargs="+", required=True)
|
|
parser.add_argument("--output", type=Path, default=ROOT / "docs/current/unattended-performance-comparison")
|
|
parser.add_argument("--measure-registry", action="store_true")
|
|
args = parser.parse_args()
|
|
command = [sys.executable, "tools/run_business_acceptance.py", "--module", args.module, "--ids", *args.ids, "--require-issue-closure"]
|
|
runs = []
|
|
for name, options in (("fresh", []), ("reused", ["--reuse"])):
|
|
started = time.perf_counter()
|
|
process = subprocess.run([*command, *options], cwd=ROOT, capture_output=True, text=True, timeout=210)
|
|
elapsed = time.perf_counter() - started
|
|
if process.returncode:
|
|
print(process.stdout, end="")
|
|
print(process.stderr, end="", file=sys.stderr)
|
|
return process.returncode
|
|
report = json.loads(process.stdout)
|
|
runs.append({"mode": name, "wallSeconds": round(elapsed, 6), "result": report})
|
|
if not runs[1]["result"].get("reused") or runs[0]["result"]["evidencePath"] != runs[1]["result"]["evidencePath"]:
|
|
raise RuntimeError("the second execution did not reuse the first verified evidence")
|
|
evidence = json.loads((ROOT / runs[0]["result"]["evidencePath"]).read_text(encoding="utf-8"))
|
|
fresh, reused = (row["wallSeconds"] for row in runs)
|
|
registry_runs = []
|
|
if args.measure_registry:
|
|
for name, command in (("full-refresh", [sys.executable, "tools/generate_feature_capability_registry.py", "--force"]),
|
|
("unchanged-sync", ["node", "web/tools/sync-assets.mjs"]),
|
|
("cached-refresh", [sys.executable, "tools/generate_feature_capability_registry.py"])):
|
|
began = time.perf_counter()
|
|
process = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, timeout=60, check=True)
|
|
registry_runs.append({"step": name, "seconds": round(time.perf_counter() - began, 6), "result": json.loads(process.stdout)})
|
|
observations = []
|
|
for label, file in (("Earlier window", "unattended-performance-before.json"), ("v11 window", "unattended-performance-v11.json")):
|
|
path = ROOT / "docs/current" / file
|
|
if path.is_file():
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
observations.append({"label": label, "cycles": len(data["cycles"]), "firstCycle": data["cycles"][0]["cycle"],
|
|
"lastCycle": data["cycles"][-1]["cycle"], "averageCycleSeconds": data.get("averageEffectiveCycleSeconds"),
|
|
"phases": data["phases"]})
|
|
result = {"module": args.module, "businessIds": args.ids, "runs": runs,
|
|
"savedSeconds": round(fresh - reused, 6), "reductionPercent": round(100 * (fresh - reused) / fresh, 2),
|
|
"caseTimings": evidence.get("timings", {}), "observedWindows": observations, "registryRuns": registry_runs,
|
|
"scope": "Controlled comparison uses identical verified business evidence. Historical windows contain different workloads and are observational."}
|
|
prefix = args.output if args.output.is_absolute() else ROOT / args.output
|
|
prefix.parent.mkdir(parents=True, exist_ok=True)
|
|
prefix.with_suffix(".json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
lines = ["# Execution Efficiency Comparison", "", result["scope"], "", "## Controlled Business Check", "",
|
|
f"Module: {args.module}; business: {', '.join(args.ids)}.", "", "| Mode | Wall time (s) | New case execution |", "|---|---:|---|",
|
|
f"| Fresh | {fresh:.3f} | Yes |", f"| Verified evidence reused | {reused:.3f} | No |", "",
|
|
f"Saved: {result['savedSeconds']:.3f} s ({result['reductionPercent']:.2f}%). Evidence: {runs[0]['result']['evidencePath']}", "",
|
|
"## Native Case Steps", "", "| Step | Seconds | Result |", "|---|---:|---|"]
|
|
timings = result["caseTimings"]
|
|
for key in ("transpileSeconds", "nativeLoadSeconds"):
|
|
if key in timings:
|
|
lines.append(f"| {key} | {timings[key]:.6f} | completed |")
|
|
for index, operation in enumerate(timings.get("operations", []), 1):
|
|
lines.append(f"| {index}. {operation['format']} {operation['direction']} | {operation['seconds']:.6f} | {operation['result']} |")
|
|
if registry_runs:
|
|
lines.extend(["", "## Registry Cache", "", "| Step | Seconds | Result |", "|---|---:|---|"])
|
|
for row in registry_runs:
|
|
lines.append(f"| {row['step']} | {row['seconds']:.3f} | {row['result'].get('status', row['result'].get('timing', {}).get('manifestChanged'))} |")
|
|
if len(observations) == 2:
|
|
before, after = observations
|
|
lines.extend(["", "## Historical Windows", "", "Different workloads: this table does not establish a causal speedup.", "",
|
|
"| Window | Cycles | Average full cycle (s) |", "|---|---|---:|"])
|
|
for observed in observations:
|
|
lines.append(f"| {observed['label']} | {observed['firstCycle']} - {observed['lastCycle']} | {observed['averageCycleSeconds']:.3f} |")
|
|
lines.extend(["", "| Phase | Earlier total (s) | v11 total (s) | Earlier average (s) | v11 average (s) |", "|---|---:|---:|---:|---:|"])
|
|
for phase in dict.fromkeys([*before["phases"], *after["phases"]]):
|
|
left, right = before["phases"].get(phase, {}), after["phases"].get(phase, {})
|
|
lines.append(f"| {phase} | {left.get('seconds', 0):.3f} | {right.get('seconds', 0):.3f} | {left.get('averageSeconds', 0):.3f} | {right.get('averageSeconds', 0):.3f} |")
|
|
prefix.with_suffix(".md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
print(json.dumps({"status": "passed", "freshSeconds": fresh, "reusedSeconds": reused,
|
|
"reductionPercent": result["reductionPercent"], "report": str(prefix.with_suffix('.md'))}))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|