wangdequan/linuxcncWebWASM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement WebAssembly runtime and kinematics coverage

Browse Source
This commit is contained in:
wangdequan
2026-08-21 06:04:48 +08:00
parent 5dd614c335
commit 554b72dae8
112 changed files with 47148 additions and 807 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
scripts/context_packet.py
View File

@@ -10,6 +10,7 @@ from __future__ import annotations
import argparse
import hashlib
import json
import math
import sys
from dataclasses import dataclass
from pathlib import Path
@@ -34,8 +35,13 @@ EXCLUSION_REASONS = {
"E3": "evidence_file_count_exceed_budget",
"E4": "evidence_bytes_exceed_budget",
"E5": "context_remaining_bytes_insufficient",
"E6": "token_budget_exceeded",
}
# Without a provider tokenizer, one UTF-8 byte per token is the only portable
# hard upper bound. The request validator can replace it with an exact count.
TOKEN_BYTES_PER_TOKEN = 1
class PacketError(ValueError):
"""Raised when a packet cannot be generated without violating a budget."""
@@ -115,6 +121,12 @@ def _context_bytes(packet: dict[str, Any], evidence_used: int) -> int:
return _packet_bytes(packet) + evidence_used
def _estimate_tokens(packet_bytes: int, evidence_bytes: int) -> int:
"""Return a conservative, provider-neutral input token estimate."""
return math.ceil((packet_bytes + evidence_bytes) / TOKEN_BYTES_PER_TOKEN)
def _finalize_packet_size(packet: dict[str, Any]) -> int:
"""Store the self-consistent serialized size in the budget metadata."""
@@ -166,6 +178,7 @@ def generate_packet(
context_id: str,
packet_id: str,
profile: str = "feature",
token_limit: int | None = None,
total_bytes: int | None = None,
evidence_bytes: int | None = None,
max_files: int | None = None,
@@ -177,12 +190,27 @@ def generate_packet(
raise PacketError(f"unknown profile: {profile}")
limits = dict(PROFILES[profile])
limits["profile"] = profile
if total_bytes is not None:
limits["bytes"] = int(total_bytes)
if evidence_bytes is not None:
limits["evidence_bytes"] = int(evidence_bytes)
if max_files is not None:
limits["files"] = int(max_files)
# Overrides are intentionally one-way: a task may ask for a smaller
# packet, but it must not enlarge a profile and silently bypass the
# context-window policy enforced by the request validator.
overrides = {
"tokens": token_limit,
"bytes": total_bytes,
"evidence_bytes": evidence_bytes,
"files": max_files,
}
for key, value in overrides.items():
if value is None:
continue
value = int(value)
if value < 0:
raise PacketError("budgets must be non-negative")
if value > limits[key]:
raise PacketError(
f"{key} budget override exceeds profile "
f"(requested={value}, profile={limits[key]})"
)
limits[key] = value
if any(limits[key] < 0 for key in ("tokens", "bytes", "evidence_bytes", "files")):
raise PacketError("budgets must be non-negative")
@@ -227,10 +255,15 @@ def generate_packet(
trial["budget"] = _budget_metadata(
limits, trial_selected, excluded, evidence_used + candidate.size
)
# This check is intentionally performed before selection is committed.
trial_context_bytes = _context_bytes(trial, evidence_used + candidate.size)
# Finalize the trial before checking budgets. In particular, do
# not estimate tokens from the provisional byte-limit placeholder
# in budget.packet_bytes.
trial_packet_bytes = _finalize_packet_size(trial)
trial_context_bytes = trial_packet_bytes + evidence_used + candidate.size
if trial_context_bytes > limits["bytes"]:
reason = "E5"
elif _estimate_tokens(trial_packet_bytes, evidence_used + candidate.size) > limits["tokens"]:
reason = "E6"
if reason:
excluded.append({"path": candidate.path, "reason_code": reason})
if candidate.required:
@@ -247,7 +280,12 @@ def generate_packet(
packet["budget"] = _budget_metadata(limits, selected, excluded, evidence_used)
packet_bytes = _finalize_packet_size(packet)
context_bytes = packet_bytes + evidence_used
if packet_bytes <= limits["bytes"] and context_bytes <= limits["bytes"]:
over_bytes = packet_bytes > limits["bytes"] or context_bytes > limits["bytes"]
over_tokens = _estimate_tokens(packet_bytes, evidence_used) > limits["tokens"]
if (
not over_bytes
and not over_tokens
):
break
removable_index = next(
(index for index in range(len(selected) - 1, -1, -1)
@@ -256,12 +294,15 @@ def generate_packet(
None,
)
if removable_index is None:
raise PacketError("context skeleton and required evidence exceed byte budget")
budget_kind = "byte" if over_bytes else "token"
raise PacketError(
f"context skeleton and required evidence exceed {budget_kind} budget"
)
removed = selected.pop(removable_index)
evidence_used -= int(removed["bytes"])
excluded.append({
"path": removed["path"],
"reason_code": "E5",
"reason_code": "E5" if over_bytes else "E6",
})
return packet
@@ -282,6 +323,7 @@ def main(argv: Iterable[str] | None = None) -> int:
parser.add_argument("--packet-id", required=True)
parser.add_argument("--profile", choices=sorted(PROFILES), default="feature")
parser.add_argument("--base-dir", type=Path, default=Path("."))
parser.add_argument("--token-limit", type=int)
parser.add_argument("--total-bytes", type=int)
parser.add_argument("--evidence-bytes", type=int)
parser.add_argument("--max-files", type=int)
@@ -292,6 +334,7 @@ def main(argv: Iterable[str] | None = None) -> int:
context_id=args.context_id,
packet_id=args.packet_id,
profile=args.profile,
token_limit=args.token_limit,
total_bytes=args.total_bytes,
evidence_bytes=args.evidence_bytes,
max_files=args.max_files,
@@ -305,7 +348,7 @@ def main(argv: Iterable[str] | None = None) -> int:
if len(payload) != packet["budget"]["packet_bytes"]:
raise PacketError("serialized packet size changed before write")
temporary.replace(args.output)
except (PacketError, OSError, json.JSONDecodeError) as exc:
except (PacketError, OSError, TypeError, ValueError, json.JSONDecodeError) as exc:
print(f"context packet generation failed: {exc}", file=sys.stderr)
return 2
return 0