wangdequan/workinf_Blender_Wasm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Complete M16 action select circle parity
Some checks failed
M6 deployable RC / quick (push) Has been cancelled
Details
M6 deployable RC / chromium (push) Has been cancelled
Details
M6 deployable RC / release (push) Has been cancelled
Details

Browse Source
This commit is contained in:
mes123456
2026-08-20 22:57:59 -04:00
parent fb5abd8e37
commit 6a0b980d75
368 changed files with 17417 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
web/protocol/atomic-checkpoint.ts Normal file
View File

@@ -0,0 +1,88 @@
/**
* The durable part of a streamed model response. Deliberately no text or
* provider payload is accepted here: a checkpoint is a resume cursor, not a
* second copy of the response.
*/
export type CheckpointState = "STREAMING" | "COMPLETED" | "FAILED";
export interface AtomicCheckpoint {
state: CheckpointState;
byteCount: number;
chunkCount: number;
sha256: string;
}
export interface AtomicCheckpointStore {
writeTemporary(path: string, checkpoint: AtomicCheckpoint): Promise<void> | void;
renameTemporary(temporaryPath: string, targetPath: string): Promise<void> | void;
}
const CHECKPOINT_KEYS = ["state", "byteCount", "chunkCount", "sha256"] as const;
const HASH_PATTERN = /^[a-f0-9]{64}$/u;
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
/** Validate and clone so callers cannot mutate a value after the atomic write. */
export function createAtomicCheckpoint(value: AtomicCheckpoint): AtomicCheckpoint {
if (!isRecord(value)) throw new TypeError("CHECKPOINT_INVALID: checkpoint must be an object");
const keys = Object.keys(value).sort();
const expected = [...CHECKPOINT_KEYS].sort();
if (keys.length !== expected.length || keys.some((key, index) => key !== expected[index])) {
throw new TypeError("CHECKPOINT_INVALID: checkpoint may contain only state, byteCount, chunkCount and sha256");
}
if (!["STREAMING", "COMPLETED", "FAILED"].includes(value.state as string)) {
throw new TypeError("CHECKPOINT_INVALID: state is invalid");
}
if (!Number.isSafeInteger(value.byteCount) || value.byteCount < 0 || !Number.isSafeInteger(value.chunkCount) || value.chunkCount < 0) {
throw new TypeError("CHECKPOINT_INVALID: counts must be non-negative safe integers");
}
if (typeof value.sha256 !== "string" || !HASH_PATTERN.test(value.sha256)) {
throw new TypeError("CHECKPOINT_INVALID: sha256 must be a lowercase SHA-256 digest");
}
return Object.freeze({
state: value.state as CheckpointState,
byteCount: value.byteCount,
chunkCount: value.chunkCount,
sha256: value.sha256,
});
}
/**
* Write a checkpoint to a temporary key and publish it with one rename. A
* failed write never replaces the previously committed checkpoint.
*/
export async function commitAtomicCheckpoint(store: AtomicCheckpointStore, targetPath: string, value: AtomicCheckpoint): Promise<AtomicCheckpoint> {
if (!store || typeof store.writeTemporary !== "function" || typeof store.renameTemporary !== "function") {
throw new TypeError("CHECKPOINT_STORE_INVALID: atomic write and rename are required");
}
if (typeof targetPath !== "string" || targetPath.length === 0) throw new TypeError("CHECKPOINT_TARGET_INVALID: target path is required");
const checkpoint = createAtomicCheckpoint(value);
const temporaryPath = `${targetPath}.tmp`;
await store.writeTemporary(temporaryPath, checkpoint);
await store.renameTemporary(temporaryPath, targetPath);
return checkpoint;
}
/** In-memory store useful for recovery tests and callers without OPFS. */
export function createMemoryCheckpointStore(): AtomicCheckpointStore & { get(path: string): AtomicCheckpoint | undefined } {
const files = new Map<string, AtomicCheckpoint>();
return {
writeTemporary(path, checkpoint) {
files.set(path, createAtomicCheckpoint(checkpoint));
},
renameTemporary(temporaryPath, targetPath) {
const checkpoint = files.get(temporaryPath);
if (!checkpoint) throw new Error("CHECKPOINT_RENAME_FAILED: temporary checkpoint is missing");
files.set(targetPath, checkpoint);
files.delete(temporaryPath);
},
get(path) {
return files.get(path);
},
};
}
export const createCheckpoint = createAtomicCheckpoint;
export const commitCheckpoint = commitAtomicCheckpoint;

93
web/protocol/provider-usage.ts Normal file
View File

@@ -0,0 +1,93 @@
export type ProviderUsageStatus = "AVAILABLE" | "PARTIAL" | "MISSING" | "INCONSISTENT";
export type ProviderUsageField = "inputTokens" | "outputTokens" | "totalTokens";
export interface ProviderUsage {
status: ProviderUsageStatus;
inputTokens: number | null;
outputTokens: number | null;
totalTokens: number | null;
missing: ProviderUsageField[];
inconsistent: ProviderUsageField[];
derivedTotal: boolean;
sources: Partial<Record<ProviderUsageField, string[]>>;
}
const ALIASES: Record<ProviderUsageField, Set<string>> = {
inputTokens: new Set(["input_tokens", "inputtokens", "prompt_tokens", "prompttokens", "prompt_token_count", "prompttokencount", "input_token_count", "inputtokencount"]),
outputTokens: new Set(["output_tokens", "outputtokens", "completion_tokens", "completiontokens", "candidates_token_count", "candidatestokencount", "output_token_count", "outputtokencount"]),
totalTokens: new Set(["total_tokens", "totaltokens", "total_token_count", "totaltokencount"]),
};
function keyName(key: string): string {
return key.replace(/[A-Z]/gu, (letter) => `_${letter.toLowerCase()}`).replace(/-/gu, "_").toLowerCase();
}
function tokenCount(value: unknown): number | undefined {
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) return value;
if (typeof value === "string" && /^\d+$/u.test(value)) {
const parsed = Number(value);
if (Number.isSafeInteger(parsed)) return parsed;
}
return undefined;
}
interface Observation { value: number; path: string; }
/** Normalize OpenAI, Responses, Anthropic, Gemini and nested gateway usage shapes. */
export function parseProviderUsage(input: unknown): ProviderUsage {
const observations: Record<ProviderUsageField, Observation[]> = { inputTokens: [], outputTokens: [], totalTokens: [] };
const seen = new Set<object>();
const visit = (value: unknown, path: string): void => {
if (typeof value !== "object" || value === null) return;
if (seen.has(value)) return;
seen.add(value);
if (Array.isArray(value)) {
value.forEach((item, index) => visit(item, `${path}[${index}]`));
return;
}
for (const [key, child] of Object.entries(value)) {
const normalized = keyName(key);
for (const field of Object.keys(ALIASES) as ProviderUsageField[]) {
if (ALIASES[field].has(normalized)) {
const count = tokenCount(child);
if (count !== undefined) observations[field].push({ value: count, path: `${path}.${key}` });
}
}
visit(child, `${path}.${key}`);
}
};
visit(input, "$");
const values = {} as Record<ProviderUsageField, number | null>;
const inconsistent: ProviderUsageField[] = [];
const sources: Partial<Record<ProviderUsageField, string[]>> = {};
for (const field of Object.keys(observations) as ProviderUsageField[]) {
const items = observations[field];
const unique = [...new Set(items.map((item) => item.value))];
values[field] = unique.length === 0 ? null : unique[0];
sources[field] = items.map((item) => item.path);
if (unique.length > 1) inconsistent.push(field);
}
let derivedTotal = false;
if (values.totalTokens === null && values.inputTokens !== null && values.outputTokens !== null && inconsistent.length === 0) {
values.totalTokens = values.inputTokens + values.outputTokens;
derivedTotal = true;
}
if (values.totalTokens !== null && values.inputTokens !== null && values.outputTokens !== null
&& values.totalTokens !== values.inputTokens + values.outputTokens && !inconsistent.includes("totalTokens")) {
inconsistent.push("totalTokens");
}
const missing = (Object.keys(values) as ProviderUsageField[]).filter((field) => values[field] === null);
const status: ProviderUsageStatus = inconsistent.length > 0
? "INCONSISTENT"
: missing.length === 3
? "MISSING"
: missing.length > 0
? "PARTIAL"
: "AVAILABLE";
return { status, ...values, missing, inconsistent, derivedTotal, sources };
}
export const normalizeProviderUsage = parseProviderUsage;
export const parseUsage = parseProviderUsage;

154
web/protocol/streaming-consumer.ts Normal file
View File

@@ -0,0 +1,154 @@
import { createAtomicCheckpoint, type AtomicCheckpoint } from "./atomic-checkpoint";
export type StreamErrorCode = "STREAM_DISCONNECTED" | "STREAM_UTF8_INVALID" | "STREAM_CANCELLED" | "STREAM_CHECKPOINT_FAILED";
function boundedError(error: unknown): Error | unknown {
const message = error instanceof Error ? error.message : String(error);
if (message.length <= 512) return error;
const result = new Error(`${message.slice(0, 509)}...`);
if (error instanceof Error && error.name) result.name = error.name;
return result;
}
export class StreamConsumeError extends Error {
readonly code: StreamErrorCode;
readonly byteCount: number;
readonly chunkCount: number;
constructor(code: StreamErrorCode, message: string, byteCount = 0, chunkCount = 0, options?: { cause?: unknown }) {
const cause = options && Object.hasOwn(options, "cause") ? boundedError(options.cause) : undefined;
super(`${code}: ${message}`, cause === undefined ? undefined : { cause });
this.name = "StreamConsumeError";
this.code = code;
this.byteCount = byteCount;
this.chunkCount = chunkCount;
}
}
export interface StreamProgress {
byteCount: number;
chunkCount: number;
sha256: string;
}
export interface StreamConsumerOptions {
signal?: AbortSignal;
requireCompletion?: boolean;
isComplete?: (progress: StreamProgress) => boolean;
onCheckpoint?: (checkpoint: AtomicCheckpoint) => Promise<void> | void;
}
export interface StreamConsumeResult extends StreamProgress {
text: string;
}
async function sha256Hex(bytes: Uint8Array): Promise<string> {
const subtle = globalThis.crypto?.subtle;
if (!subtle) throw new Error("SHA-256 is unavailable in this runtime");
const digest = await subtle.digest("SHA-256", bytes.slice().buffer);
return [...new Uint8Array(digest)].map((value) => value.toString(16).padStart(2, "0")).join("");
}
function asStream(source: ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>): ReadableStream<Uint8Array> {
if (typeof (source as ReadableStream<Uint8Array>).getReader === "function") return source as ReadableStream<Uint8Array>;
const iterable = source as AsyncIterable<Uint8Array>;
const iterator = iterable[Symbol.asyncIterator]();
let iteratorClosed = false;
const closeIterator = async (reason?: unknown): Promise<void> => {
if (iteratorClosed) return;
iteratorClosed = true;
await iterator.return?.(reason);
};
return new ReadableStream<Uint8Array>({
async pull(controller) {
try {
const next = await iterator.next();
if (next.done) controller.close(); else controller.enqueue(next.value);
} catch (error) {
await closeIterator(error).catch(() => undefined);
controller.error(error);
}
},
async cancel(reason) {
// A fetch body is cancelled by the reader, but an async iterable needs
// its return hook called explicitly so generators can release sockets or
// other resources after a transport failure.
await closeIterator(reason);
},
});
}
function errorMessage(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
// Provider/network errors occasionally contain a full response body. Keep
// the diagnostic bounded so a failure cannot recreate the context problem.
return message.length > 512 ? `${message.slice(0, 509)}...` : message;
}
async function checkpoint(options: StreamConsumerOptions, state: "STREAMING" | "COMPLETED" | "FAILED", bytes: Uint8Array, byteCount: number, chunkCount: number): Promise<void> {
if (!options.onCheckpoint) return;
const sha256 = await sha256Hex(bytes);
try { await options.onCheckpoint(createAtomicCheckpoint({ state, byteCount, chunkCount, sha256 })); }
catch (error) { throw new StreamConsumeError("STREAM_CHECKPOINT_FAILED", "checkpoint publication failed", byteCount, chunkCount, { cause: error }); }
}
/** Consume a UTF-8 byte stream without replacement characters or partial durable output. */
export async function consumeUtf8Stream(source: ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>, options: StreamConsumerOptions = {}): Promise<StreamConsumeResult> {
const reader = asStream(source).getReader();
const decoder = new TextDecoder("utf-8", { fatal: true });
const parts: Uint8Array[] = [];
let text = "";
let byteCount = 0;
let chunkCount = 0;
let completed = false;
const aborted = (): boolean => options.signal?.aborted === true;
const cancel = (): void => { void reader.cancel("STREAM_CANCELLED").catch(() => undefined); };
options.signal?.addEventListener("abort", cancel, { once: true });
try {
while (true) {
if (aborted()) throw new StreamConsumeError("STREAM_CANCELLED", "stream consumption was cancelled", byteCount, chunkCount);
let next: ReadableStreamReadResult<Uint8Array>;
try { next = await reader.read(); }
catch (error) {
if (aborted()) throw new StreamConsumeError("STREAM_CANCELLED", "stream consumption was cancelled", byteCount, chunkCount, { cause: error });
throw new StreamConsumeError("STREAM_DISCONNECTED", `stream disconnected before completion: ${errorMessage(error)}`, byteCount, chunkCount, { cause: error });
}
if (aborted()) throw new StreamConsumeError("STREAM_CANCELLED", "stream consumption was cancelled", byteCount, chunkCount);
if (next.done) break;
if (!(next.value instanceof Uint8Array)) throw new StreamConsumeError("STREAM_DISCONNECTED", "stream yielded a non-byte chunk", byteCount, chunkCount);
const bytes = new Uint8Array(next.value);
parts.push(bytes);
byteCount += bytes.byteLength;
chunkCount += 1;
try { text += decoder.decode(bytes, { stream: true }); }
catch (error) { throw new StreamConsumeError("STREAM_UTF8_INVALID", "stream ended or yielded an invalid UTF-8 sequence", byteCount, chunkCount, { cause: error }); }
await checkpoint(options, "STREAMING", Uint8Array.from(parts.flatMap((part) => [...part])), byteCount, chunkCount);
}
try { text += decoder.decode(); }
catch (error) { throw new StreamConsumeError("STREAM_UTF8_INVALID", "stream ended with an incomplete UTF-8 sequence", byteCount, chunkCount, { cause: error }); }
const progress = { byteCount, chunkCount, sha256: await sha256Hex(Uint8Array.from(parts.flatMap((part) => [...part]))) };
if ((options.requireCompletion && !options.isComplete) || (options.isComplete && !options.isComplete(progress))) {
throw new StreamConsumeError("STREAM_DISCONNECTED", "stream disconnected before completion", byteCount, chunkCount);
}
await checkpoint(options, "COMPLETED", Uint8Array.from(parts.flatMap((part) => [...part])), byteCount, chunkCount);
completed = true;
return { text, ...progress };
} catch (error) {
if (!completed && error instanceof StreamConsumeError && options.onCheckpoint) {
try {
const bytes = Uint8Array.from(parts.flatMap((part) => [...part]));
await options.onCheckpoint(createAtomicCheckpoint({ state: "FAILED", byteCount, chunkCount, sha256: await sha256Hex(bytes) }));
} catch {
// Preserve the original transport/UTF-8 error; failed checkpoints are advisory.
}
}
throw error;
} finally {
options.signal?.removeEventListener("abort", cancel);
if (!completed) await reader.cancel("STREAM_CONSUMER_STOPPED").catch(() => undefined);
reader.releaseLock();
}
}
export const consumeModelStream = consumeUtf8Stream;
export const consumeStream = consumeUtf8Stream;

71
web/protocol/token-limit.ts Normal file
View File

@@ -0,0 +1,71 @@
import { parseProviderUsage } from "./provider-usage";
export type TokenLimitCategory = "NONE" | "CONTEXT_LIMIT" | "OUTPUT_LIMIT" | "MAX_OUTPUT_TOKENS" | "UNKNOWN";
export interface TokenLimitClassification {
category: TokenLimitCategory;
detected: boolean;
retryable: boolean;
reason: string | null;
actual: number | null;
limit: number | null;
}
export interface TokenLimitOptions {
contextTokens?: number;
outputTokens?: number;
maxOutputTokens?: number;
}
function textOf(input: unknown): string {
try {
const serialized = JSON.stringify(input);
return (serialized ?? String(input)).toLowerCase();
} catch { return String(input).toLowerCase(); }
}
export const classifyLimit = classifyTokenLimit;
function numberAt(input: unknown, names: Set<string>, seen = new Set<object>()): number | null {
if (typeof input !== "object" || input === null) return null;
if (seen.has(input)) return null;
seen.add(input);
if (Array.isArray(input)) {
for (const value of input) { const result = numberAt(value, names, seen); if (result !== null) return result; }
return null;
}
for (const [key, value] of Object.entries(input)) {
const normalized = key.replace(/[A-Z]/gu, (letter) => `_${letter.toLowerCase()}`).toLowerCase();
if (names.has(normalized) && typeof value === "number" && Number.isFinite(value)) return value;
const result = numberAt(value, names, seen);
if (result !== null) return result;
}
return null;
}
function result(category: TokenLimitCategory, reason: string | null, actual: number | null, limit: number | null): TokenLimitClassification {
return { category, detected: category !== "NONE", retryable: category === "CONTEXT_LIMIT" || category === "OUTPUT_LIMIT", reason, actual, limit };
}
/** Recognize provider errors, finish reasons and measured usage consistently. */
export function classifyTokenLimit(input: unknown, options: TokenLimitOptions = {}): TokenLimitClassification {
const text = textOf(input);
const usage = parseProviderUsage(input);
const contextActual = numberAt(input, new Set(["context_tokens", "contexttokens", "prompt_tokens", "prompttokens", "input_tokens", "inputtokens"]));
const outputActual = usage.outputTokens ?? numberAt(input, new Set(["output_tokens", "outputtokens", "completion_tokens", "completiontokens"]));
const contextMarker = /(context[_ -]?length|context[_ -]?window|maximum[_ -]?context|prompt[_ -]?too[_ -]?long|input[_ -]?too[_ -]?long|context[_ -]?limit|context[_ -]?exceed)/u.test(text);
const outputMarker = /(output[_ -]?limit|completion[_ -]?limit|output[_ -]?too[_ -]?long|output[_ -]?exceed|completion[_ -]?tokens?.{0,20}(limit|exceed|too))/u.test(text);
const maxMarker = /(max(?:imum)?[_ -]?output[_ -]?tokens|max[_ -]?completion[_ -]?tokens|max[_ -]?tokens|finish[_ -]?reason.{0,12}(length|max_tokens)|stop[_ -]?reason.{0,12}max_tokens|incomplete.{0,30}max_output_tokens)/u.test(text);
if (contextMarker || (options.contextTokens !== undefined && contextActual !== null && contextActual > options.contextTokens)) {
return result("CONTEXT_LIMIT", "context token limit exceeded", contextActual, options.contextTokens ?? null);
}
if (outputMarker || (options.outputTokens !== undefined && outputActual !== null && outputActual > options.outputTokens)) {
return result("OUTPUT_LIMIT", "output token limit exceeded", outputActual, options.outputTokens ?? null);
}
if (maxMarker || (options.maxOutputTokens !== undefined && outputActual !== null && outputActual >= options.maxOutputTokens && /(?:length|max[_ -]?tokens)/u.test(text))) {
return result("MAX_OUTPUT_TOKENS", "generation stopped at max_output_tokens", outputActual, options.maxOutputTokens ?? null);
}
if (text.includes("token") && /(limit|exceed|truncat|too long)/u.test(text)) return result("UNKNOWN", "provider reported an unclassified token limit", null, null);
return result("NONE", null, null, null);
}