Accumulated uncommitted work: GHOST Web backend, WebGL/GPU web layer, web workflow engine and protocol, tooling, docs and task records. Large generated evidence, user-testing archives, build snapshots and imported test assets remain untracked. Co-Authored-By: Claude Code <noreply@anthropic.com>
213 lines
9.5 KiB
TypeScript
213 lines
9.5 KiB
TypeScript
import { NATIVE_AUDIO_CHANNELS, NATIVE_AUDIO_FRAMES, NATIVE_AUDIO_RATE, NATIVE_AUDIO_BUFFER_BLOCKS, NATIVE_AUDIO_RECOVERY_BLOCKS, NATIVE_AUDIO_BUSY_BLOCKS, type NativeAudioOutput as OutputMessage, type NativeAudioPull, type NativeAudioClock, type NativeAudioSegment, type NativeAudioPrepared } from "../../../protocol/native-audio";
|
|
|
|
/** Browser device only: no scene, playback, decoding or mixing emulation. */
|
|
export class NativeAudioOutput {
|
|
private context: AudioContext | null = null;
|
|
private generation = 0;
|
|
private sequence = 0;
|
|
private pending: number | null = null;
|
|
private preparation: { sequence: number; expiresAtMs: number } | null = null;
|
|
private bufferBlocks = NATIVE_AUDIO_BUFFER_BLOCKS;
|
|
private nextTime = 0;
|
|
private disposed = false;
|
|
private failed = false;
|
|
private segments: NativeAudioSegment[] = [];
|
|
private scheduledFrames = 0;
|
|
private completedFrames = 0;
|
|
private outputTime = 0;
|
|
private clockSequence = 0;
|
|
private pendingClock: number | null = null;
|
|
private nodes = new Set<AudioBufferSourceNode>();
|
|
private timer: ReturnType<typeof setInterval>;
|
|
|
|
constructor(private port: MessagePort, private onError: (message: string) => void) {
|
|
port.onmessage = (event: MessageEvent<OutputMessage>) => this.receive(event.data);
|
|
// resume() must execute within the same trusted gesture as Blender's input.
|
|
window.addEventListener("pointerdown", this.unlock, true);
|
|
window.addEventListener("keydown", this.unlock, true);
|
|
this.timer = setInterval(() => { this.publishClock(); this.pull(); }, 10);
|
|
}
|
|
|
|
private unlock = (): void => {
|
|
if (this.disposed) return;
|
|
try {
|
|
if (!this.context) {
|
|
this.context = new AudioContext({ latencyHint: "interactive" });
|
|
this.context.onstatechange = () => { this.publishClock(); this.pull(); };
|
|
}
|
|
if (this.context.state !== "running") {
|
|
void this.context.resume().then(() => this.pull()).catch(error => this.onError(String(error)));
|
|
}
|
|
}
|
|
catch (error) { this.onError(String(error)); }
|
|
};
|
|
|
|
private flush(resetClock = true): void {
|
|
// Freeze the last presented position before removing queued nodes on error
|
|
// or shutdown. Only a native generation reset may rewind the PCM counters.
|
|
if (!resetClock) this.publishClock(false);
|
|
for (const node of this.nodes) {
|
|
node.onended = null;
|
|
node.stop();
|
|
node.disconnect();
|
|
}
|
|
this.nodes.clear();
|
|
this.nextTime = 0;
|
|
if (resetClock) {
|
|
this.preparation = null;
|
|
this.bufferBlocks = NATIVE_AUDIO_BUFFER_BLOCKS;
|
|
this.segments = [];
|
|
this.scheduledFrames = 0;
|
|
this.completedFrames = 0;
|
|
this.outputTime = 0;
|
|
this.pendingClock = null;
|
|
}
|
|
}
|
|
|
|
private publishClock(allowAdvance = true, force = false): void {
|
|
if (this.disposed) return;
|
|
const context = this.context;
|
|
const now = performance.now();
|
|
const running = !!context && context.state === "running" && !this.failed && allowAdvance;
|
|
if (context && !this.failed) {
|
|
const stamp = context.getOutputTimestamp?.();
|
|
// getOutputTimestamp already accounts for physical output delay; do not
|
|
// subtract baseLatency again. Older devices use the explicit latency sum.
|
|
const time = stamp && typeof stamp.performanceTime === 'number' && typeof stamp.contextTime === 'number' && stamp.performanceTime > 0 && Number.isFinite(stamp.contextTime)
|
|
? stamp.contextTime + (running ? Math.max(0, now - stamp.performanceTime) / 1000 : 0)
|
|
: context.currentTime - (context.baseLatency || 0) - (context.outputLatency || 0);
|
|
this.outputTime = Math.max(this.outputTime, Math.min(context.currentTime, time));
|
|
}
|
|
// Retire by device presentation, not AudioBufferSourceNode.onended: that
|
|
// event describes graph rendering and can precede the actual speaker output.
|
|
while (this.segments.length && this.outputTime >=
|
|
this.segments[0].start + this.segments[0].frames / NATIVE_AUDIO_RATE) {
|
|
const segment = this.segments.shift()!;
|
|
this.completedFrames = segment.firstFrame + segment.frames;
|
|
}
|
|
// Coalesce normal feedback while native code owns the Worker synchronously.
|
|
// A final freeze bypasses this slot so shutdown cannot leave a running clock.
|
|
if (allowAdvance && !force && this.pendingClock !== null) return;
|
|
this.pendingClock = ++this.clockSequence;
|
|
this.port.postMessage({
|
|
type: "clock", generation: this.generation,
|
|
sequence: this.pendingClock,
|
|
sampledAtMs: performance.timeOrigin + now, outputTime: this.outputTime,
|
|
running, completedFrames: this.completedFrames, segments: this.segments,
|
|
} satisfies NativeAudioClock);
|
|
}
|
|
|
|
private finishPreparation(allowAdvance = true): void {
|
|
if (!this.preparation) return;
|
|
const sequence = this.preparation.sequence;
|
|
this.preparation = null;
|
|
// Same port ordering makes this scheduled device clock visible to the
|
|
// Worker before it resumes the native command after the acknowledgement.
|
|
this.publishClock(allowAdvance, true);
|
|
this.port.postMessage({ type: "prepared", generation: this.generation, sequence } satisfies NativeAudioPrepared);
|
|
}
|
|
|
|
private pull(): void {
|
|
const context = this.context;
|
|
if (this.preparation && (this.failed || !context || context.state !== "running" ||
|
|
performance.timeOrigin + performance.now() >= this.preparation.expiresAtMs)) {
|
|
this.finishPreparation();
|
|
}
|
|
if (this.disposed || this.failed || !context || context.state !== "running" || this.pending !== null) return;
|
|
const blockTime = NATIVE_AUDIO_FRAMES / NATIVE_AUDIO_RATE;
|
|
const queued = Math.max(0, this.nextTime - context.currentTime);
|
|
const target = (this.preparation ? NATIVE_AUDIO_BUSY_BLOCKS : this.bufferBlocks) * blockTime;
|
|
if (this.preparation && queued >= target - blockTime) {
|
|
this.finishPreparation();
|
|
return;
|
|
}
|
|
// Refill below the low-water mark in one bounded native batch. During a
|
|
// command preparation fill the entire reserve, then acknowledge scheduling.
|
|
if (!this.preparation && queued > target / 2) return;
|
|
const blocks = Math.min(NATIVE_AUDIO_BUSY_BLOCKS, Math.max(1, Math.ceil((target - queued) / blockTime)));
|
|
this.pending = ++this.sequence;
|
|
this.port.postMessage({ type: "pull", generation: this.generation, sequence: this.pending, blocks } satisfies NativeAudioPull);
|
|
}
|
|
|
|
private receive(message: OutputMessage): void {
|
|
if (this.disposed) return;
|
|
if (message.type === "reset") {
|
|
// An old pull can receive the same reset after the native notification.
|
|
// Do not discard a newer in-flight block on that duplicate acknowledgement.
|
|
if (message.generation <= this.generation) return;
|
|
this.generation = message.generation;
|
|
this.pending = null;
|
|
this.failed = false;
|
|
this.flush();
|
|
this.publishClock();
|
|
return;
|
|
}
|
|
if (message.generation !== this.generation) return;
|
|
if (message.type === "prepare") {
|
|
this.preparation = { sequence: message.sequence, expiresAtMs: message.expiresAtMs };
|
|
this.pull();
|
|
return;
|
|
}
|
|
if (message.type === "clockAck") {
|
|
if (message.sequence === this.pendingClock) this.pendingClock = null;
|
|
return;
|
|
}
|
|
if (message.type === "error") {
|
|
this.pending = null;
|
|
this.flush(false);
|
|
this.failed = true;
|
|
this.finishPreparation();
|
|
this.onError(message.message);
|
|
return;
|
|
}
|
|
if (message.sequence !== this.pending) return;
|
|
this.pending = null;
|
|
const context = this.context;
|
|
if (!message.samples || !context) { this.finishPreparation(); return; }
|
|
try {
|
|
const samples = new Float32Array(message.samples);
|
|
const frames = samples.length / NATIVE_AUDIO_CHANNELS;
|
|
if (frames < NATIVE_AUDIO_FRAMES || frames > NATIVE_AUDIO_FRAMES * NATIVE_AUDIO_BUSY_BLOCKS ||
|
|
frames % NATIVE_AUDIO_FRAMES !== 0) throw new Error("Invalid native audio block");
|
|
const buffer = context.createBuffer(NATIVE_AUDIO_CHANNELS, frames, NATIVE_AUDIO_RATE);
|
|
for (let channel = 0; channel < NATIVE_AUDIO_CHANNELS; channel++) {
|
|
const output = buffer.getChannelData(channel);
|
|
for (let frame = 0; frame < frames; frame++) output[frame] = samples[frame * NATIVE_AUDIO_CHANNELS + channel];
|
|
}
|
|
const node = context.createBufferSource();
|
|
node.buffer = buffer;
|
|
node.connect(context.destination);
|
|
node.onended = () => { this.nodes.delete(node); node.disconnect(); };
|
|
this.nodes.add(node);
|
|
if (this.nextTime > 0 && this.nextTime <= context.currentTime) {
|
|
// Never replay missed PCM or pretend the native timeline advanced during
|
|
// silence. Start a new device-clock segment and increase bounded reserve.
|
|
this.bufferBlocks = Math.min(NATIVE_AUDIO_RECOVERY_BLOCKS, this.bufferBlocks + 2);
|
|
}
|
|
this.nextTime = Math.max(this.nextTime, context.currentTime + 0.005);
|
|
node.start(this.nextTime);
|
|
this.segments.push({ start: this.nextTime, frames, firstFrame: this.scheduledFrames });
|
|
this.scheduledFrames += frames;
|
|
this.nextTime += buffer.duration;
|
|
this.publishClock();
|
|
this.pull();
|
|
}
|
|
catch (error) { this.flush(false); this.failed = true; this.finishPreparation(); this.onError(String(error)); }
|
|
}
|
|
|
|
dispose(): void {
|
|
this.flush(false);
|
|
this.finishPreparation(false);
|
|
this.disposed = true;
|
|
clearInterval(this.timer);
|
|
window.removeEventListener("pointerdown", this.unlock, true);
|
|
window.removeEventListener("keydown", this.unlock, true);
|
|
this.port.close();
|
|
if (this.context) {
|
|
this.context.onstatechange = null;
|
|
void this.context.close().catch(() => {});
|
|
}
|
|
this.context = null;
|
|
}
|
|
}
|