364 lines
15 KiB
Python
364 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""Shared parsing and bounded-context helpers for the Web task system.
|
||
|
||
The task cards are intentionally human-readable. This module turns the
|
||
references in a card into a small, auditable context closure without ever
|
||
loading project-wide source trees, raw logs, or full inventories.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
from pathlib import Path
|
||
from typing import Any, Iterable
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
DOCS = ROOT / "docs"
|
||
CURRENT = DOCS / "current"
|
||
TASKS = DOCS / "tasks"
|
||
HANDOFF = CURRENT / "FREECAD_WEB_HANDOFF.md"
|
||
MIN_CONTEXT = CURRENT / "FREECAD_WEB_CONTEXT_MIN.md"
|
||
STATE = CURRENT / "task-state.json"
|
||
BUDGET = CURRENT / "FREECAD_WEB_CONTEXT_BUDGET.json"
|
||
|
||
CARD_ID = re.compile(r"^#\s+(FW-[^：: ]+)", re.MULTILINE)
|
||
CARD_STATUS = re.compile(r"^[-*]\s*状态：`([^`]+)`", re.MULTILINE)
|
||
PREDECESSORS = re.compile(r"^[-*]\s*前置：(.*)$", re.MULTILINE)
|
||
FIELD = re.compile(r"^[-*]\s*([^：:]+)[：:](.*)$", re.MULTILINE)
|
||
ID_TOKEN = re.compile(r"FW-[A-Z0-9-]+")
|
||
|
||
# These files are useful audit inputs but are too broad to be task context.
|
||
BROAD_NAMES = {
|
||
"format-api-inventory.json",
|
||
"object-inventory.json",
|
||
"workbench-inventory.json",
|
||
"dependency-closure.json",
|
||
"web-parity-matrix.json",
|
||
"web-parity-matrix.md",
|
||
}
|
||
RAW_SUFFIXES = (".log", ".trace", ".out", ".err")
|
||
SOURCE_PREFIXES = ("freecad-source/", "src/", "FreeCAD/freecad-source/")
|
||
|
||
|
||
def load_budget() -> dict[str, Any]:
|
||
return json.loads(BUDGET.read_text(encoding="utf-8"))
|
||
|
||
|
||
def load_cards(task_ids: Iterable[str] | None = None) -> dict[str, tuple[Path, str, str]]:
|
||
"""Load all cards or only the requested IDs.
|
||
|
||
Continuation turns need only the active/planned cards and direct
|
||
prerequisites. Listing paths is cheap; reading every card body on each
|
||
context refresh was a measurable gate-side cost. Full callers retain the
|
||
original behavior by omitting ``task_ids``.
|
||
"""
|
||
cards: dict[str, tuple[Path, str, str]] = {}
|
||
paths = sorted(TASKS.rglob("*.md"))
|
||
if task_ids is not None:
|
||
wanted = {str(item).lower() for item in task_ids if item}
|
||
selected = [path for path in paths if path.stem.lower() in wanted]
|
||
# Module package cards use human-readable lowercase filenames (for
|
||
# example ``FW-P6-cam.md``) while their canonical IDs are uppercase.
|
||
# Resolve those few IDs from the compact P6 index without reading all
|
||
# task bodies.
|
||
index_path = TASKS / "p6" / "INDEX.md"
|
||
try:
|
||
index_text = index_path.read_text(encoding="utf-8", errors="replace")
|
||
except OSError:
|
||
index_text = ""
|
||
for match in re.finditer(r"\[([^\]]+)\]\(([^)]+)\)", index_text):
|
||
if match.group(1).lower() not in wanted:
|
||
continue
|
||
candidate = (index_path.parent / match.group(2)).resolve()
|
||
if candidate.is_file() and candidate not in selected:
|
||
selected.append(candidate)
|
||
paths = sorted(selected)
|
||
for path in paths:
|
||
if path.name in {"INDEX.md", "TASK_CARD_TEMPLATE.md"}:
|
||
continue
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
card = CARD_ID.search(text)
|
||
status = CARD_STATUS.search(text)
|
||
if card and status:
|
||
cards[card.group(1)] = (path, status.group(1), text)
|
||
return cards
|
||
|
||
|
||
def predecessor_ids(text: str) -> list[str]:
|
||
match = PREDECESSORS.search(text)
|
||
return ID_TOKEN.findall(match.group(1)) if match else []
|
||
|
||
|
||
def field_values(text: str, field_name: str) -> list[str]:
|
||
return [match.group(2).strip() for match in FIELD.finditer(text) if match.group(1).strip() == field_name]
|
||
|
||
|
||
def _path_candidates(value: str) -> Iterable[str]:
|
||
# Markdown links, backticks and plain relative paths are all accepted.
|
||
for target in re.findall(r"\]\(([^)]+)\)", value):
|
||
target = target.split("#", 1)[0].strip()
|
||
if target:
|
||
yield target
|
||
for candidate in re.findall(r"`([^`]+)`", value):
|
||
candidate = candidate.split("#", 1)[0].strip("[]")
|
||
if candidate:
|
||
yield candidate
|
||
for candidate in re.findall(r"<([^>]+)>|([^\s,;`()]+)", value):
|
||
candidate = next((part for part in candidate if part), "")
|
||
candidate = candidate.split("#", 1)[0].strip("[]")
|
||
if candidate:
|
||
yield candidate
|
||
|
||
|
||
def resolve_repo_path(candidate: str, origin: Path | None = None) -> Path | None:
|
||
candidate = candidate.replace("\\", "/")
|
||
if candidate.startswith("https://") or candidate.startswith("http://"):
|
||
return None
|
||
# Links in task cards are relative to docs/tasks/<module>; normalize the
|
||
# common docs-relative and repository-relative spellings.
|
||
options = []
|
||
if origin is not None:
|
||
options.append(origin.parent / candidate)
|
||
options.extend([ROOT / candidate, DOCS / candidate, TASKS / candidate])
|
||
if candidate.startswith("../") and origin is None:
|
||
# Most task-card links are relative to docs/tasks/<module>. Try each
|
||
# module directory as the origin, then the docs root.
|
||
options = [
|
||
*(TASKS / module / candidate for module in ("p6", "cam", "bim", "assembly")),
|
||
DOCS / candidate,
|
||
*options,
|
||
]
|
||
for path in options:
|
||
try:
|
||
resolved = path.resolve()
|
||
resolved.relative_to(ROOT.resolve())
|
||
except (OSError, ValueError):
|
||
continue
|
||
if resolved.exists() and resolved.is_file():
|
||
return resolved
|
||
return None
|
||
|
||
|
||
def classify_path(path: Path) -> tuple[str, str | None]:
|
||
rel = path.relative_to(ROOT).as_posix()
|
||
name = path.name
|
||
if rel == "docs/current/module-batch-manifest.json":
|
||
return "read", None
|
||
if name in BROAD_NAMES:
|
||
return "excluded", "broad inventory or matrix"
|
||
if name.endswith(RAW_SUFFIXES):
|
||
return "excluded", "raw log/trace"
|
||
if any(rel.startswith(prefix) for prefix in SOURCE_PREFIXES):
|
||
return "excluded", "source tree is inspected by the task command, not loaded as context"
|
||
if "/archive/" in rel or rel.startswith("docs/archive/"):
|
||
return "excluded", "historical archive"
|
||
# A task may cite a large JSON artifact for audit, but loading the whole
|
||
# artifact defeats bounded context. Keep only compact summaries in L2.
|
||
if path.stat().st_size > 4 * 1024:
|
||
return "metadata_only", "evidence exceeds per-file context cap"
|
||
return "read", None
|
||
|
||
|
||
def sha256(path: Path) -> str:
|
||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||
|
||
|
||
def extract_referenced_files(texts: Iterable[tuple[str, Path | None]]) -> list[Path]:
|
||
found: dict[str, Path] = {}
|
||
for text, origin in texts:
|
||
for field_name in ("输入", "产物", "证据", "复现命令", "命令/方法"):
|
||
for value in field_values(text, field_name):
|
||
for candidate in _path_candidates(value):
|
||
path = resolve_repo_path(candidate, origin)
|
||
if path:
|
||
found[str(path)] = path
|
||
return sorted(found.values(), key=lambda path: str(path))
|
||
|
||
|
||
def context_manifest(
|
||
task_id: str,
|
||
*,
|
||
include_evidence: bool = True,
|
||
cards: dict[str, tuple[Path, str, str]] | None = None,
|
||
) -> dict[str, Any]:
|
||
cards = cards or load_cards([task_id])
|
||
if task_id not in cards:
|
||
raise KeyError(f"unknown task card: {task_id}")
|
||
state = json.loads(STATE.read_text(encoding="utf-8"))
|
||
budget = load_budget()
|
||
card_path, status, card_text = cards[task_id]
|
||
predecessor_list = predecessor_ids(card_text)
|
||
missing_predecessors = [item for item in predecessor_list if item not in cards]
|
||
if missing_predecessors:
|
||
cards.update(load_cards(missing_predecessors))
|
||
entries: list[dict[str, Any]] = []
|
||
excluded: list[dict[str, str]] = []
|
||
|
||
def add(path: Path, role: str, task: str | None = None, mode: str = "read") -> None:
|
||
kind, reason = classify_path(path)
|
||
if mode == "read" and kind != "read":
|
||
excluded.append({"path": str(path.relative_to(ROOT)), "reason": reason or kind})
|
||
return
|
||
file_stat = path.stat()
|
||
entries.append(
|
||
{
|
||
"role": role,
|
||
"task_id": task,
|
||
"path": str(path.relative_to(ROOT)),
|
||
"bytes": file_stat.st_size,
|
||
"mtime_ns": file_stat.st_mtime_ns,
|
||
"sha256": sha256(path),
|
||
"mode": mode if kind == "read" else kind,
|
||
}
|
||
)
|
||
|
||
add(HANDOFF, "handoff")
|
||
add(card_path, "task_card", task_id)
|
||
batch_path = CURRENT / "module-batch-manifest.json"
|
||
if batch_path.exists() and state.get("active_task") == task_id:
|
||
add(batch_path, "module_batch")
|
||
for predecessor in predecessor_list:
|
||
if predecessor in cards:
|
||
add(cards[predecessor][0], "direct_prerequisite_card", predecessor)
|
||
else:
|
||
excluded.append({"path": predecessor, "reason": "missing prerequisite card"})
|
||
|
||
if include_evidence:
|
||
evidence_texts = [(card_text, card_path)] + [
|
||
(cards[item][2], cards[item][0]) for item in predecessor_list if item in cards
|
||
]
|
||
for path in extract_referenced_files(evidence_texts):
|
||
if path in {HANDOFF, card_path} or any(path == cards[item][0] for item in predecessor_list if item in cards):
|
||
continue
|
||
kind, reason = classify_path(path)
|
||
if kind == "read":
|
||
add(path, "direct_evidence")
|
||
else:
|
||
excluded.append({"path": str(path.relative_to(ROOT)), "reason": reason or kind})
|
||
|
||
# A context packet must remain small even if a card accidentally names many
|
||
# artifacts. Keep the mandatory L0/L1 files first, then greedily select the
|
||
# smallest evidence files that fit *both* evidence and total-byte budgets.
|
||
# The old implementation applied only the file-count cap, which allowed a
|
||
# handful of large JSON files to overflow the packet before validation.
|
||
limits = budget["limits"]
|
||
max_files = limits["direct_evidence_file_count"]
|
||
evidence = [item for item in entries if item["role"] == "direct_evidence"]
|
||
base_bytes = sum(item["bytes"] for item in entries if item["role"] != "direct_evidence")
|
||
available_evidence_bytes = max(0, limits["task_context_bytes"] - base_bytes)
|
||
available_evidence_bytes = min(available_evidence_bytes, limits["direct_evidence_bytes"])
|
||
keep_evidence: list[dict[str, Any]] = []
|
||
keep_paths: set[str] = set()
|
||
selected_bytes = 0
|
||
for item in sorted(evidence, key=lambda item: (item["bytes"], item["path"])):
|
||
if len(keep_evidence) >= max_files:
|
||
excluded.append({"path": item["path"], "reason": "evidence file-count cap"})
|
||
continue
|
||
if selected_bytes + item["bytes"] > available_evidence_bytes:
|
||
excluded.append({"path": item["path"], "reason": "evidence byte budget"})
|
||
continue
|
||
keep_evidence.append(item)
|
||
keep_paths.add(item["path"])
|
||
selected_bytes += item["bytes"]
|
||
entries = [
|
||
item
|
||
for item in entries
|
||
if item["role"] != "direct_evidence" or item["path"] in keep_paths
|
||
]
|
||
|
||
# Keep the active card and L0 handoff authoritative, then fit completed
|
||
# prerequisite cards into the bounded packet. The dependency graph remains
|
||
# unchanged; omitted cards are explicitly recorded in excluded so a fresh
|
||
# executor never mistakes a compact packet for missing prerequisites.
|
||
total_bytes = sum(item["bytes"] for item in entries)
|
||
if total_bytes > limits["task_context_bytes"]:
|
||
prerequisite_entries = [item for item in entries if item["role"] == "direct_prerequisite_card"]
|
||
for item in sorted(prerequisite_entries, key=lambda value: (-value["bytes"], value["path"])):
|
||
if total_bytes <= limits["task_context_bytes"]:
|
||
break
|
||
entries.remove(item)
|
||
total_bytes -= item["bytes"]
|
||
excluded.append({"path": item["path"], "reason": "prerequisite byte budget; dependency retained in task card"})
|
||
|
||
total_bytes = sum(item["bytes"] for item in entries)
|
||
evidence_bytes = sum(item["bytes"] for item in entries if item["role"] == "direct_evidence")
|
||
return {
|
||
"schema": "freecad-web-task-context/v2",
|
||
"task_id": task_id,
|
||
"status": status,
|
||
"active_task": state.get("active_task"),
|
||
"source_matrix_sha256": state.get("source_matrix_sha256"),
|
||
"budget": {
|
||
"total_bytes_limit": limits["task_context_bytes"],
|
||
"evidence_bytes_limit": limits["direct_evidence_bytes"],
|
||
"evidence_file_limit": limits["direct_evidence_file_count"],
|
||
"estimated_tokens": total_bytes // budget["bytes_per_token_estimate"],
|
||
},
|
||
"files": entries,
|
||
"excluded": excluded,
|
||
"totals": {
|
||
"bytes": total_bytes,
|
||
"estimated_tokens": total_bytes // budget["bytes_per_token_estimate"],
|
||
"evidence_bytes": evidence_bytes,
|
||
"evidence_files": len(keep_evidence),
|
||
"direct_prerequisites": len(predecessor_list),
|
||
},
|
||
"read_order": [
|
||
"handoff",
|
||
"task_card",
|
||
"direct_prerequisite_card",
|
||
"direct_evidence",
|
||
],
|
||
"instructions": [
|
||
"只读取 files 中 mode=read 的路径；excluded 不得加载。",
|
||
"原始日志、源码树和全量 inventory 只能按任务命令定点检查，不得复制进对话上下文。",
|
||
"若 totals 超限，先拆卡或生成更小的 JSON 摘要，不得放宽预算。",
|
||
],
|
||
}
|
||
|
||
|
||
def write_context_manifest(
|
||
task_id: str,
|
||
output: Path | None = None,
|
||
manifest: dict[str, Any] | None = None,
|
||
) -> Path:
|
||
"""Write one context snapshot without recomputing it when provided."""
|
||
if manifest is None:
|
||
manifest = context_manifest(task_id)
|
||
output = output or (CURRENT / "context" / f"{task_id}.json")
|
||
output.parent.mkdir(parents=True, exist_ok=True)
|
||
output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
return output
|
||
|
||
|
||
def write_context_summary(task_id: str, manifest: dict[str, Any] | None = None) -> Path:
|
||
"""Write the small human-facing continuation file for the active task."""
|
||
if manifest is None:
|
||
manifest = context_manifest(task_id)
|
||
manifest_path = CURRENT / "context" / f"{task_id}.json"
|
||
manifest_hash = sha256(manifest_path) if manifest_path.exists() else ""
|
||
read_files = [item["path"] for item in manifest["files"] if item["mode"] == "read"]
|
||
excluded = [item["path"] for item in manifest["excluded"]]
|
||
lines = [
|
||
"# FreeCAD Web 最小续接上下文",
|
||
"",
|
||
f"- 当前任务：`{task_id}`（`{manifest['status']}`）",
|
||
f"- 上下文包：`{manifest['totals']['bytes']}B`，约 `{manifest['totals']['estimated_tokens']}` token",
|
||
"- 只读取以下文件；不要打开历史归档、完整 inventory、源码树或原始日志：",
|
||
]
|
||
lines.extend(f" - `{path}`" for path in read_files)
|
||
if excluded:
|
||
lines.append("- 已排除（禁止装入上下文）：")
|
||
lines.extend(f" - `{path}`" for path in excluded)
|
||
lines.extend(
|
||
[
|
||
"- 完成或阻断当前卡后，先更新任务卡和状态账本，再重新生成本摘要。",
|
||
f"- 清单 hash：`{manifest_hash}`",
|
||
]
|
||
)
|
||
MIN_CONTEXT.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
return MIN_CONTEXT
|