Complete M16 action select circle parity

web/package.json

@@ -61,6 +61,7 @@
     "test:browser": "playwright test --config playwright.release.config.ts",
     "test:task-context": "node --test tests/unit/task-context.test.mjs && node ../tools/web/check-task-context.mjs",
     "test:context-governance": "node --test tests/unit/task-context.test.mjs tests/unit/context-governance.test.mjs && node ../tools/web/check-context-governance.mjs",
+    "test:streaming-resilience": "node --test tests/unit/streaming-resilience.test.mjs",
     "task:context": "node ../tools/web/print-task-context.mjs",
     "test:cross-browser": "npm run test:browser",
     "test:golden": "node ../tools/web/run-blender-golden.mjs",

web/protocol/atomic-checkpoint.ts

@@ -0,0 +1,88 @@
+/**
+ * The durable part of a streamed model response.  Deliberately no text or
+ * provider payload is accepted here: a checkpoint is a resume cursor, not a
+ * second copy of the response.
+ */
+export type CheckpointState = "STREAMING" | "COMPLETED" | "FAILED";
+
+export interface AtomicCheckpoint {
+  state: CheckpointState;
+  byteCount: number;
+  chunkCount: number;
+  sha256: string;
+}
+
+export interface AtomicCheckpointStore {
+  writeTemporary(path: string, checkpoint: AtomicCheckpoint): Promise<void> | void;
+  renameTemporary(temporaryPath: string, targetPath: string): Promise<void> | void;
+}
+
+const CHECKPOINT_KEYS = ["state", "byteCount", "chunkCount", "sha256"] as const;
+const HASH_PATTERN = /^[a-f0-9]{64}$/u;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** Validate and clone so callers cannot mutate a value after the atomic write. */
+export function createAtomicCheckpoint(value: AtomicCheckpoint): AtomicCheckpoint {
+  if (!isRecord(value)) throw new TypeError("CHECKPOINT_INVALID: checkpoint must be an object");
+  const keys = Object.keys(value).sort();
+  const expected = [...CHECKPOINT_KEYS].sort();
+  if (keys.length !== expected.length || keys.some((key, index) => key !== expected[index])) {
+    throw new TypeError("CHECKPOINT_INVALID: checkpoint may contain only state, byteCount, chunkCount and sha256");
+  }
+  if (!["STREAMING", "COMPLETED", "FAILED"].includes(value.state as string)) {
+    throw new TypeError("CHECKPOINT_INVALID: state is invalid");
+  }
+  if (!Number.isSafeInteger(value.byteCount) || value.byteCount < 0 || !Number.isSafeInteger(value.chunkCount) || value.chunkCount < 0) {
+    throw new TypeError("CHECKPOINT_INVALID: counts must be non-negative safe integers");
+  }
+  if (typeof value.sha256 !== "string" || !HASH_PATTERN.test(value.sha256)) {
+    throw new TypeError("CHECKPOINT_INVALID: sha256 must be a lowercase SHA-256 digest");
+  }
+  return Object.freeze({
+    state: value.state as CheckpointState,
+    byteCount: value.byteCount,
+    chunkCount: value.chunkCount,
+    sha256: value.sha256,
+  });
+}
+
+/**
+ * Write a checkpoint to a temporary key and publish it with one rename.  A
+ * failed write never replaces the previously committed checkpoint.
+ */
+export async function commitAtomicCheckpoint(store: AtomicCheckpointStore, targetPath: string, value: AtomicCheckpoint): Promise<AtomicCheckpoint> {
+  if (!store || typeof store.writeTemporary !== "function" || typeof store.renameTemporary !== "function") {
+    throw new TypeError("CHECKPOINT_STORE_INVALID: atomic write and rename are required");
+  }
+  if (typeof targetPath !== "string" || targetPath.length === 0) throw new TypeError("CHECKPOINT_TARGET_INVALID: target path is required");
+  const checkpoint = createAtomicCheckpoint(value);
+  const temporaryPath = `${targetPath}.tmp`;
+  await store.writeTemporary(temporaryPath, checkpoint);
+  await store.renameTemporary(temporaryPath, targetPath);
+  return checkpoint;
+}
+
+/** In-memory store useful for recovery tests and callers without OPFS. */
+export function createMemoryCheckpointStore(): AtomicCheckpointStore & { get(path: string): AtomicCheckpoint | undefined } {
+  const files = new Map<string, AtomicCheckpoint>();
+  return {
+    writeTemporary(path, checkpoint) {
+      files.set(path, createAtomicCheckpoint(checkpoint));
+    },
+    renameTemporary(temporaryPath, targetPath) {
+      const checkpoint = files.get(temporaryPath);
+      if (!checkpoint) throw new Error("CHECKPOINT_RENAME_FAILED: temporary checkpoint is missing");
+      files.set(targetPath, checkpoint);
+      files.delete(temporaryPath);
+    },
+    get(path) {
+      return files.get(path);
+    },
+  };
+}
+
+export const createCheckpoint = createAtomicCheckpoint;
+export const commitCheckpoint = commitAtomicCheckpoint;

web/protocol/provider-usage.ts

@@ -0,0 +1,93 @@
+export type ProviderUsageStatus = "AVAILABLE" | "PARTIAL" | "MISSING" | "INCONSISTENT";
+export type ProviderUsageField = "inputTokens" | "outputTokens" | "totalTokens";
+
+export interface ProviderUsage {
+  status: ProviderUsageStatus;
+  inputTokens: number | null;
+  outputTokens: number | null;
+  totalTokens: number | null;
+  missing: ProviderUsageField[];
+  inconsistent: ProviderUsageField[];
+  derivedTotal: boolean;
+  sources: Partial<Record<ProviderUsageField, string[]>>;
+}
+
+const ALIASES: Record<ProviderUsageField, Set<string>> = {
+  inputTokens: new Set(["input_tokens", "inputtokens", "prompt_tokens", "prompttokens", "prompt_token_count", "prompttokencount", "input_token_count", "inputtokencount"]),
+  outputTokens: new Set(["output_tokens", "outputtokens", "completion_tokens", "completiontokens", "candidates_token_count", "candidatestokencount", "output_token_count", "outputtokencount"]),
+  totalTokens: new Set(["total_tokens", "totaltokens", "total_token_count", "totaltokencount"]),
+};
+
+function keyName(key: string): string {
+  return key.replace(/[A-Z]/gu, (letter) => `_${letter.toLowerCase()}`).replace(/-/gu, "_").toLowerCase();
+}
+
+function tokenCount(value: unknown): number | undefined {
+  if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) return value;
+  if (typeof value === "string" && /^\d+$/u.test(value)) {
+    const parsed = Number(value);
+    if (Number.isSafeInteger(parsed)) return parsed;
+  }
+  return undefined;
+}
+
+interface Observation { value: number; path: string; }
+
+/** Normalize OpenAI, Responses, Anthropic, Gemini and nested gateway usage shapes. */
+export function parseProviderUsage(input: unknown): ProviderUsage {
+  const observations: Record<ProviderUsageField, Observation[]> = { inputTokens: [], outputTokens: [], totalTokens: [] };
+  const seen = new Set<object>();
+  const visit = (value: unknown, path: string): void => {
+    if (typeof value !== "object" || value === null) return;
+    if (seen.has(value)) return;
+    seen.add(value);
+    if (Array.isArray(value)) {
+      value.forEach((item, index) => visit(item, `${path}[${index}]`));
+      return;
+    }
+    for (const [key, child] of Object.entries(value)) {
+      const normalized = keyName(key);
+      for (const field of Object.keys(ALIASES) as ProviderUsageField[]) {
+        if (ALIASES[field].has(normalized)) {
+          const count = tokenCount(child);
+          if (count !== undefined) observations[field].push({ value: count, path: `${path}.${key}` });
+        }
+      }
+      visit(child, `${path}.${key}`);
+    }
+  };
+  visit(input, "$");
+
+  const values = {} as Record<ProviderUsageField, number | null>;
+  const inconsistent: ProviderUsageField[] = [];
+  const sources: Partial<Record<ProviderUsageField, string[]>> = {};
+  for (const field of Object.keys(observations) as ProviderUsageField[]) {
+    const items = observations[field];
+    const unique = [...new Set(items.map((item) => item.value))];
+    values[field] = unique.length === 0 ? null : unique[0];
+    sources[field] = items.map((item) => item.path);
+    if (unique.length > 1) inconsistent.push(field);
+  }
+
+  let derivedTotal = false;
+  if (values.totalTokens === null && values.inputTokens !== null && values.outputTokens !== null && inconsistent.length === 0) {
+    values.totalTokens = values.inputTokens + values.outputTokens;
+    derivedTotal = true;
+  }
+  if (values.totalTokens !== null && values.inputTokens !== null && values.outputTokens !== null
+    && values.totalTokens !== values.inputTokens + values.outputTokens && !inconsistent.includes("totalTokens")) {
+    inconsistent.push("totalTokens");
+  }
+  const missing = (Object.keys(values) as ProviderUsageField[]).filter((field) => values[field] === null);
+  const status: ProviderUsageStatus = inconsistent.length > 0
+    ? "INCONSISTENT"
+    : missing.length === 3
+      ? "MISSING"
+      : missing.length > 0
+        ? "PARTIAL"
+        : "AVAILABLE";
+  return { status, ...values, missing, inconsistent, derivedTotal, sources };
+}
+
+export const normalizeProviderUsage = parseProviderUsage;
+export const parseUsage = parseProviderUsage;

web/protocol/streaming-consumer.ts

@@ -0,0 +1,154 @@
+import { createAtomicCheckpoint, type AtomicCheckpoint } from "./atomic-checkpoint";
+
+export type StreamErrorCode = "STREAM_DISCONNECTED" | "STREAM_UTF8_INVALID" | "STREAM_CANCELLED" | "STREAM_CHECKPOINT_FAILED";
+
+function boundedError(error: unknown): Error | unknown {
+  const message = error instanceof Error ? error.message : String(error);
+  if (message.length <= 512) return error;
+  const result = new Error(`${message.slice(0, 509)}...`);
+  if (error instanceof Error && error.name) result.name = error.name;
+  return result;
+}
+
+export class StreamConsumeError extends Error {
+  readonly code: StreamErrorCode;
+  readonly byteCount: number;
+  readonly chunkCount: number;
+
+  constructor(code: StreamErrorCode, message: string, byteCount = 0, chunkCount = 0, options?: { cause?: unknown }) {
+    const cause = options && Object.hasOwn(options, "cause") ? boundedError(options.cause) : undefined;
+    super(`${code}: ${message}`, cause === undefined ? undefined : { cause });
+    this.name = "StreamConsumeError";
+    this.code = code;
+    this.byteCount = byteCount;
+    this.chunkCount = chunkCount;
+  }
+}
+
+export interface StreamProgress {
+  byteCount: number;
+  chunkCount: number;
+  sha256: string;
+}
+
+export interface StreamConsumerOptions {
+  signal?: AbortSignal;
+  requireCompletion?: boolean;
+  isComplete?: (progress: StreamProgress) => boolean;
+  onCheckpoint?: (checkpoint: AtomicCheckpoint) => Promise<void> | void;
+}
+
+export interface StreamConsumeResult extends StreamProgress {
+  text: string;
+}
+
+async function sha256Hex(bytes: Uint8Array): Promise<string> {
+  const subtle = globalThis.crypto?.subtle;
+  if (!subtle) throw new Error("SHA-256 is unavailable in this runtime");
+  const digest = await subtle.digest("SHA-256", bytes.slice().buffer);
+  return [...new Uint8Array(digest)].map((value) => value.toString(16).padStart(2, "0")).join("");
+}
+
+function asStream(source: ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>): ReadableStream<Uint8Array> {
+  if (typeof (source as ReadableStream<Uint8Array>).getReader === "function") return source as ReadableStream<Uint8Array>;
+  const iterable = source as AsyncIterable<Uint8Array>;
+  const iterator = iterable[Symbol.asyncIterator]();
+  let iteratorClosed = false;
+  const closeIterator = async (reason?: unknown): Promise<void> => {
+    if (iteratorClosed) return;
+    iteratorClosed = true;
+    await iterator.return?.(reason);
+  };
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        const next = await iterator.next();
+        if (next.done) controller.close(); else controller.enqueue(next.value);
+      } catch (error) {
+        await closeIterator(error).catch(() => undefined);
+        controller.error(error);
+      }
+    },
+    async cancel(reason) {
+      // A fetch body is cancelled by the reader, but an async iterable needs
+      // its return hook called explicitly so generators can release sockets or
+      // other resources after a transport failure.
+      await closeIterator(reason);
+    },
+  });
+}
+
+function errorMessage(error: unknown): string {
+  const message = error instanceof Error ? error.message : String(error);
+  // Provider/network errors occasionally contain a full response body. Keep
+  // the diagnostic bounded so a failure cannot recreate the context problem.
+  return message.length > 512 ? `${message.slice(0, 509)}...` : message;
+}
+
+async function checkpoint(options: StreamConsumerOptions, state: "STREAMING" | "COMPLETED" | "FAILED", bytes: Uint8Array, byteCount: number, chunkCount: number): Promise<void> {
+  if (!options.onCheckpoint) return;
+  const sha256 = await sha256Hex(bytes);
+  try { await options.onCheckpoint(createAtomicCheckpoint({ state, byteCount, chunkCount, sha256 })); }
+  catch (error) { throw new StreamConsumeError("STREAM_CHECKPOINT_FAILED", "checkpoint publication failed", byteCount, chunkCount, { cause: error }); }
+}
+
+/** Consume a UTF-8 byte stream without replacement characters or partial durable output. */
+export async function consumeUtf8Stream(source: ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>, options: StreamConsumerOptions = {}): Promise<StreamConsumeResult> {
+  const reader = asStream(source).getReader();
+  const decoder = new TextDecoder("utf-8", { fatal: true });
+  const parts: Uint8Array[] = [];
+  let text = "";
+  let byteCount = 0;
+  let chunkCount = 0;
+  let completed = false;
+  const aborted = (): boolean => options.signal?.aborted === true;
+  const cancel = (): void => { void reader.cancel("STREAM_CANCELLED").catch(() => undefined); };
+  options.signal?.addEventListener("abort", cancel, { once: true });
+  try {
+    while (true) {
+      if (aborted()) throw new StreamConsumeError("STREAM_CANCELLED", "stream consumption was cancelled", byteCount, chunkCount);
+      let next: ReadableStreamReadResult<Uint8Array>;
+      try { next = await reader.read(); }
+      catch (error) {
+        if (aborted()) throw new StreamConsumeError("STREAM_CANCELLED", "stream consumption was cancelled", byteCount, chunkCount, { cause: error });
+        throw new StreamConsumeError("STREAM_DISCONNECTED", `stream disconnected before completion: ${errorMessage(error)}`, byteCount, chunkCount, { cause: error });
+      }
+      if (aborted()) throw new StreamConsumeError("STREAM_CANCELLED", "stream consumption was cancelled", byteCount, chunkCount);
+      if (next.done) break;
+      if (!(next.value instanceof Uint8Array)) throw new StreamConsumeError("STREAM_DISCONNECTED", "stream yielded a non-byte chunk", byteCount, chunkCount);
+      const bytes = new Uint8Array(next.value);
+      parts.push(bytes);
+      byteCount += bytes.byteLength;
+      chunkCount += 1;
+      try { text += decoder.decode(bytes, { stream: true }); }
+      catch (error) { throw new StreamConsumeError("STREAM_UTF8_INVALID", "stream ended or yielded an invalid UTF-8 sequence", byteCount, chunkCount, { cause: error }); }
+      await checkpoint(options, "STREAMING", Uint8Array.from(parts.flatMap((part) => [...part])), byteCount, chunkCount);
+    }
+    try { text += decoder.decode(); }
+    catch (error) { throw new StreamConsumeError("STREAM_UTF8_INVALID", "stream ended with an incomplete UTF-8 sequence", byteCount, chunkCount, { cause: error }); }
+    const progress = { byteCount, chunkCount, sha256: await sha256Hex(Uint8Array.from(parts.flatMap((part) => [...part]))) };
+    if ((options.requireCompletion && !options.isComplete) || (options.isComplete && !options.isComplete(progress))) {
+      throw new StreamConsumeError("STREAM_DISCONNECTED", "stream disconnected before completion", byteCount, chunkCount);
+    }
+    await checkpoint(options, "COMPLETED", Uint8Array.from(parts.flatMap((part) => [...part])), byteCount, chunkCount);
+    completed = true;
+    return { text, ...progress };
+  } catch (error) {
+    if (!completed && error instanceof StreamConsumeError && options.onCheckpoint) {
+      try {
+        const bytes = Uint8Array.from(parts.flatMap((part) => [...part]));
+        await options.onCheckpoint(createAtomicCheckpoint({ state: "FAILED", byteCount, chunkCount, sha256: await sha256Hex(bytes) }));
+      } catch {
+        // Preserve the original transport/UTF-8 error; failed checkpoints are advisory.
+      }
+    }
+    throw error;
+  } finally {
+    options.signal?.removeEventListener("abort", cancel);
+    if (!completed) await reader.cancel("STREAM_CONSUMER_STOPPED").catch(() => undefined);
+    reader.releaseLock();
+  }
+}
+
+export const consumeModelStream = consumeUtf8Stream;
+export const consumeStream = consumeUtf8Stream;

web/protocol/token-limit.ts

@@ -0,0 +1,71 @@
+import { parseProviderUsage } from "./provider-usage";
+
+export type TokenLimitCategory = "NONE" | "CONTEXT_LIMIT" | "OUTPUT_LIMIT" | "MAX_OUTPUT_TOKENS" | "UNKNOWN";
+
+export interface TokenLimitClassification {
+  category: TokenLimitCategory;
+  detected: boolean;
+  retryable: boolean;
+  reason: string | null;
+  actual: number | null;
+  limit: number | null;
+}
+
+export interface TokenLimitOptions {
+  contextTokens?: number;
+  outputTokens?: number;
+  maxOutputTokens?: number;
+}
+
+function textOf(input: unknown): string {
+  try {
+    const serialized = JSON.stringify(input);
+    return (serialized ?? String(input)).toLowerCase();
+  } catch { return String(input).toLowerCase(); }
+}
+
+export const classifyLimit = classifyTokenLimit;
+
+function numberAt(input: unknown, names: Set<string>, seen = new Set<object>()): number | null {
+  if (typeof input !== "object" || input === null) return null;
+  if (seen.has(input)) return null;
+  seen.add(input);
+  if (Array.isArray(input)) {
+    for (const value of input) { const result = numberAt(value, names, seen); if (result !== null) return result; }
+    return null;
+  }
+  for (const [key, value] of Object.entries(input)) {
+    const normalized = key.replace(/[A-Z]/gu, (letter) => `_${letter.toLowerCase()}`).toLowerCase();
+    if (names.has(normalized) && typeof value === "number" && Number.isFinite(value)) return value;
+    const result = numberAt(value, names, seen);
+    if (result !== null) return result;
+  }
+  return null;
+}
+
+function result(category: TokenLimitCategory, reason: string | null, actual: number | null, limit: number | null): TokenLimitClassification {
+  return { category, detected: category !== "NONE", retryable: category === "CONTEXT_LIMIT" || category === "OUTPUT_LIMIT", reason, actual, limit };
+}
+
+/** Recognize provider errors, finish reasons and measured usage consistently. */
+export function classifyTokenLimit(input: unknown, options: TokenLimitOptions = {}): TokenLimitClassification {
+  const text = textOf(input);
+  const usage = parseProviderUsage(input);
+  const contextActual = numberAt(input, new Set(["context_tokens", "contexttokens", "prompt_tokens", "prompttokens", "input_tokens", "inputtokens"]));
+  const outputActual = usage.outputTokens ?? numberAt(input, new Set(["output_tokens", "outputtokens", "completion_tokens", "completiontokens"]));
+  const contextMarker = /(context[_ -]?length|context[_ -]?window|maximum[_ -]?context|prompt[_ -]?too[_ -]?long|input[_ -]?too[_ -]?long|context[_ -]?limit|context[_ -]?exceed)/u.test(text);
+  const outputMarker = /(output[_ -]?limit|completion[_ -]?limit|output[_ -]?too[_ -]?long|output[_ -]?exceed|completion[_ -]?tokens?.{0,20}(limit|exceed|too))/u.test(text);
+  const maxMarker = /(max(?:imum)?[_ -]?output[_ -]?tokens|max[_ -]?completion[_ -]?tokens|max[_ -]?tokens|finish[_ -]?reason.{0,12}(length|max_tokens)|stop[_ -]?reason.{0,12}max_tokens|incomplete.{0,30}max_output_tokens)/u.test(text);
+
+  if (contextMarker || (options.contextTokens !== undefined && contextActual !== null && contextActual > options.contextTokens)) {
+    return result("CONTEXT_LIMIT", "context token limit exceeded", contextActual, options.contextTokens ?? null);
+  }
+  if (outputMarker || (options.outputTokens !== undefined && outputActual !== null && outputActual > options.outputTokens)) {
+    return result("OUTPUT_LIMIT", "output token limit exceeded", outputActual, options.outputTokens ?? null);
+  }
+  if (maxMarker || (options.maxOutputTokens !== undefined && outputActual !== null && outputActual >= options.maxOutputTokens && /(?:length|max[_ -]?tokens)/u.test(text))) {
+    return result("MAX_OUTPUT_TOKENS", "generation stopped at max_output_tokens", outputActual, options.maxOutputTokens ?? null);
+  }
+  if (text.includes("token") && /(limit|exceed|truncat|too long)/u.test(text)) return result("UNKNOWN", "provider reported an unclassified token limit", null, null);
+  return result("NONE", null, null, null);
+}

web/tests/unit/context-governance.test.mjs

@@ -1,13 +1,14 @@
 import test from "node:test";
 import assert from "node:assert/strict";
-import { buildTaskContext } from "../../../tools/web/task-context-lib.mjs";
+import { buildTaskContext, CONTEXT_LIMITS } from "../../../tools/web/task-context-lib.mjs";
 import { validateContextBundle } from "../../../tools/web/context-governance.mjs";
 
 test("current context satisfies document, task-card, and pointer budgets", () => {
   const bundle = buildTaskContext();
   const result = validateContextBundle(bundle);
   assert.deepEqual(result.violations, []);
-  assert.ok(result.report.totalTokens <= 3500);
+  assert.ok(result.report.estimatedContextTokens <= CONTEXT_LIMITS.contextTokens);
+  assert.ok(result.report.serializedContextTokens <= CONTEXT_LIMITS.contextEnvelopeTokens);
 });
 
 test("task cards cannot smuggle full plans or history into the default context", () => {
@@ -45,3 +46,13 @@ test("governance rejects forged selection totals and unaudited exclusions", () =
   assert.ok(result.violations.some(({ code }) => code === "INPUT_EXCLUSION_REASON_UNKNOWN"));
   assert.ok(result.violations.some(({ code }) => code === "INPUT_SELECTION_ENTRY_INVALID"));
 });
+
+test("governance reports a serialized context envelope that exceeds its reserved budget", () => {
+  const bundle = buildTaskContext();
+  const oversized = {
+    ...bundle,
+    context: { ...bundle.context, commands: ["node "+"x".repeat(6000)] },
+  };
+  const result = validateContextBundle(oversized);
+  assert.ok(result.violations.some(({ code }) => code === "CONTEXT_ENVELOPE_OVER_BUDGET"));
+});

web/tests/unit/streaming-resilience.test.mjs

@@ -0,0 +1,137 @@
+import assert from "node:assert/strict";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import test from "node:test";
+import ts from "typescript";
+
+const root = path.resolve(import.meta.dirname, "../../..");
+function transpile(file, replacements = {}) {
+  let source = fs.readFileSync(path.join(root, file), "utf8");
+  for (const [from, to] of Object.entries(replacements)) source = source.replaceAll(`from "${from}"`, `from "${to}"`);
+  const result = ts.transpileModule(source, {
+    compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022 },
+    fileName: file,
+    reportDiagnostics: true,
+  });
+  assert.deepEqual(result.diagnostics, []);
+  return `data:text/javascript;base64,${Buffer.from(result.outputText).toString("base64")}`;
+}
+
+const atomicUrl = transpile("web/protocol/atomic-checkpoint.ts");
+const usage = await import(transpile("web/protocol/provider-usage.ts"));
+const limits = await import(transpile("web/protocol/token-limit.ts", { "./provider-usage": transpile("web/protocol/provider-usage.ts") }));
+const stream = await import(transpile("web/protocol/streaming-consumer.ts", { "./atomic-checkpoint": atomicUrl }));
+const checkpoint = await import(atomicUrl);
+
+function bytes(...values) { return Uint8Array.from(values); }
+function streamOf(chunks, error) {
+  let index = 0;
+  return new ReadableStream({
+    pull(controller) {
+      if (index < chunks.length) controller.enqueue(chunks[index++]);
+      else if (error) controller.error(error);
+      else controller.close();
+    },
+  });
+}
+function digest(value) { return crypto.createHash("sha256").update(value).digest("hex"); }
+
+test("stream consumer preserves UTF-8 code points split across chunks", async () => {
+  const checkpoints = [];
+  const result = await stream.consumeUtf8Stream(streamOf([bytes(0xe2), bytes(0x82), bytes(0xac, 0x20, 0x41)]), {
+    onCheckpoint: (value) => checkpoints.push(value),
+  });
+  assert.equal(result.text, "€ A");
+  assert.equal(result.byteCount, 5);
+  assert.equal(result.chunkCount, 3);
+  assert.equal(result.sha256, digest(Buffer.from([0xe2, 0x82, 0xac, 0x20, 0x41])));
+  assert.ok(checkpoints.every((value) => !Object.hasOwn(value, "text")));
+  assert.equal(checkpoints.at(-1).state, "COMPLETED");
+});
+
+test("stream consumer rejects a truncated UTF-8 sequence and records only failed metadata", async () => {
+  const checkpoints = [];
+  await assert.rejects(
+    stream.consumeUtf8Stream(streamOf([bytes(0xf0, 0x9f, 0x92)]), { onCheckpoint: (value) => checkpoints.push(value) }),
+    (error) => error.code === "STREAM_UTF8_INVALID" && error.byteCount === 3,
+  );
+  assert.equal(checkpoints.at(-1).state, "FAILED");
+  assert.deepEqual(Object.keys(checkpoints.at(-1)).sort(), ["byteCount", "chunkCount", "sha256", "state"]);
+});
+
+test("stream consumer classifies a transport/body decoding disconnect as incomplete", async () => {
+  await assert.rejects(
+    stream.consumeUtf8Stream(streamOf([bytes(0x61)], new Error("Transport error: network error: error decoding response body"))),
+    (error) => error.code === "STREAM_DISCONNECTED" && /stream disconnected before completion.*error decoding response body/u.test(error.message),
+  );
+});
+
+test("stream consumer bounds a provider error body before surfacing it", async () => {
+  const detail = "x".repeat(10_000);
+  await assert.rejects(
+    stream.consumeUtf8Stream(streamOf([], new Error(detail))),
+    (error) => error.code === "STREAM_DISCONNECTED" && error.message.length < 600 && error.cause?.message.length < 600,
+  );
+});
+
+test("stream consumer can require an explicit completion marker", async () => {
+  await assert.rejects(
+    stream.consumeUtf8Stream(streamOf([bytes(0x61)]), { requireCompletion: true }),
+    (error) => error.code === "STREAM_DISCONNECTED" && /before completion/u.test(error.message),
+  );
+  const result = await stream.consumeUtf8Stream(streamOf([bytes(0x61)]), { requireCompletion: true, isComplete: ({ byteCount }) => byteCount === 1 });
+  assert.equal(result.text, "a");
+});
+
+test("stream consumer cancels a pending read without publishing completion", async () => {
+  const controller = new AbortController();
+  controller.abort();
+  await assert.rejects(stream.consumeUtf8Stream(streamOf([bytes(0x61)]), { signal: controller.signal }), (error) => error.code === "STREAM_CANCELLED");
+});
+
+test("stream consumer closes async iterables after a transport failure", async () => {
+  let returned = false;
+  const source = {
+    [Symbol.asyncIterator]() {
+      return {
+        async next() {
+          throw new Error("connection reset");
+        },
+        async return() {
+          returned = true;
+          return { done: true, value: undefined };
+        },
+      };
+    },
+  };
+  await assert.rejects(stream.consumeUtf8Stream(source), (error) => error.code === "STREAM_DISCONNECTED");
+  assert.equal(returned, true);
+});
+
+test("atomic checkpoint publishes through temp then rename and rejects model text", async () => {
+  const store = checkpoint.createMemoryCheckpointStore();
+  const value = await checkpoint.commitAtomicCheckpoint(store, "response.checkpoint", {
+    state: "STREAMING", byteCount: 4, chunkCount: 2, sha256: "a".repeat(64),
+  });
+  assert.deepEqual(store.get("response.checkpoint"), value);
+  assert.throws(() => checkpoint.createAtomicCheckpoint({ ...value, text: "partial model output" }), /CHECKPOINT_INVALID/u);
+});
+
+test("provider usage normalizes nested aliases, derives totals, and detects arithmetic conflicts", () => {
+  assert.deepEqual(usage.parseProviderUsage({ response: { usage: { prompt_tokens: 7, completion_tokens: 5 } } }), {
+    status: "AVAILABLE", inputTokens: 7, outputTokens: 5, totalTokens: 12,
+    missing: [], inconsistent: [], derivedTotal: true,
+    sources: { inputTokens: ["$.response.usage.prompt_tokens"], outputTokens: ["$.response.usage.completion_tokens"], totalTokens: [] },
+  });
+  assert.equal(usage.parseProviderUsage({ usage: { input_tokens: 2, output_tokens: 3, total_tokens: 99 } }).status, "INCONSISTENT");
+  assert.equal(usage.parseProviderUsage({ result: { usage: { input_tokens: 2 } } }).status, "PARTIAL");
+  assert.equal(usage.parseProviderUsage({}).status, "MISSING");
+});
+
+test("token limit classifier distinguishes context, output and max-output truncation", () => {
+  assert.equal(limits.classifyTokenLimit({ error: { code: "context_length_exceeded" } }).category, "CONTEXT_LIMIT");
+  assert.equal(limits.classifyTokenLimit({ error: { code: "output_limit_exceeded" } }).category, "OUTPUT_LIMIT");
+  assert.equal(limits.classifyTokenLimit({ choices: [{ finish_reason: "length" }], usage: { completion_tokens: 16 } }, { maxOutputTokens: 16 }).category, "MAX_OUTPUT_TOKENS");
+  assert.equal(limits.classifyTokenLimit({ message: "ordinary provider failure" }).category, "NONE");
+});

web/tests/unit/task-context.test.mjs

@@ -1,6 +1,6 @@
 import test from "node:test";
 import assert from "node:assert/strict";
-import { buildTaskContext, contextSizeReport, CONTEXT_LIMITS, readIndexedTask, selectInputPaths } from "../../../tools/web/task-context-lib.mjs";
+import { buildTaskContext, compactTaskContext, contextSizeReport, CONTEXT_LIMITS, readIndexedTask, selectInputPaths } from "../../../tools/web/task-context-lib.mjs";
 
 test("current task context is bounded and follows the parent pointer", () => {
   const bundle = buildTaskContext();
@@ -17,6 +17,20 @@ test("current task context is bounded and follows the parent pointer", () => {
   assert.equal(report.totalTokens, report.sourceTokens + report.evidenceTokens);
   assert.equal(report.evidenceTokens, bundle.context.inputSelection.totals.tokens);
   assert.ok(report.totalTokens < CONTEXT_LIMITS.contextTokens);
+  assert.ok(report.serializedContextTokens <= CONTEXT_LIMITS.contextEnvelopeTokens);
+  assert.ok(report.estimatedContextTokens <= CONTEXT_LIMITS.contextTokens);
+});
+
+test("printed context reserves envelope space and omits verbose exclusion audit", () => {
+  const bundle = buildTaskContext();
+  const compact = compactTaskContext(bundle.context);
+  assert.equal(compact.inputSelection.excludedCount, bundle.context.inputSelection.excluded.length);
+  assert.deepEqual(compact.inputSelection.excludedReasons, {
+    EVIDENCE_BYTE_BUDGET: 3,
+    GENERATED_EVIDENCE_OUTPUT: 3,
+  });
+  assert.ok(!Object.hasOwn(compact.inputSelection, "excluded"));
+  assert.ok(contextSizeReport(bundle).serializedContextTokens <= CONTEXT_LIMITS.contextEnvelopeTokens);
 });
 
 test("the indexed task record remains available beside the compact card", () => {