From 752343634176fa3843341a8b337251d7e4a804ab Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 16:27:33 -0400 Subject: [PATCH 1/9] fix(grok-voice): cancel stale read-aloud playback Signed-off-by: Rudy Celekli (cherry picked from commit f4d4d37d9c94936c4e3143466c74027a7b93f730) --- grok-voice/skills/add-read-aloud/SKILL.md | 31 ++++++-- grok-voice/tests/read-aloud.test.cjs | 94 +++++++++++++++++++++++ 2 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 grok-voice/tests/read-aloud.test.cjs diff --git a/grok-voice/skills/add-read-aloud/SKILL.md b/grok-voice/skills/add-read-aloud/SKILL.md index 1fdb96d9e..d0cb50da8 100644 --- a/grok-voice/skills/add-read-aloud/SKILL.md +++ b/grok-voice/skills/add-read-aloud/SKILL.md @@ -77,18 +77,35 @@ export async function speak(text: string, voice_id = "eve", language = "auto") { ```ts // client +// Keep this player in the shared store, not in each message button. let current: HTMLAudioElement | null = null; +let currentUrl: string | null = null; +let generation = 0; +function stop() { + generation++; // invalidate pending requests as well as active playback + current?.pause(); current = null; + if (currentUrl) URL.revokeObjectURL(currentUrl); + currentUrl = null; +} async function readAloud(text: string, voiceId = "eve") { - current?.pause(); current = null; // one utterance at a time + stop(); + const request = generation; const res = await fetch("/api/tts", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text, voice_id: voiceId }) }); + if (request !== generation) return; if (!res.ok) throw new Error("TTS request failed"); - const url = URL.createObjectURL(await res.blob()); + const blob = await res.blob(); + if (request !== generation) return; // stop/new click may occur while reading the body + const url = URL.createObjectURL(blob); const audio = new Audio(url); - audio.addEventListener("ended", () => URL.revokeObjectURL(url)); // avoid blob leaks - current = audio; - await audio.play(); // call from the click handler’s promise chain + current = audio; currentUrl = url; + audio.addEventListener("ended", () => { if (current === audio) stop(); }); + try { + await audio.play(); // call from the click handler's promise chain + } catch (error) { + if (current === audio) stop(); + throw error; + } } -function stop() { current?.pause(); current = null; } ``` - Safari: `audio.duration` is `Infinity` on blob URLs. If you need a progress bar, decode with `AudioContext.decodeAudioData(buf.slice(0))` or request `with_timestamps: true` and read `duration` from the JSON envelope (audio is then base64 in `audio`). @@ -170,7 +187,7 @@ r.raise_for_status(); audio_bytes = r.content # audio/mpeg 7. **Smoke** - Batch: `curl -X POST https://api.x.ai/v1/tts -H "Authorization: Bearer $XAI_API_KEY" -H "Content-Type: application/json" -d '{"text":"Hello from read aloud.","voice_id":"eve","language":"en"}' --output /tmp/hello.mp3` → 200 `audio/mpeg` (MP3, 24 kHz, 128 kbps, mono), plays. Omit `language` → 422 (observed; the docs’ table only lists 400). `voice_id: "nope"` → 404. - - In the app: no speaker while a reply streams; it appears when the reply ends. Click → spinner → stop square → back to speaker when audio ends; click again mid-play → stops at once; click a second reply while the first plays → first stops, second plays. Safari: first play works from the click, `URL.revokeObjectURL` fires on `ended`. + - In the app: no speaker while a reply streams; it appears when the reply ends. Click → spinner → stop square → back to speaker when audio ends; click again mid-play → stops at once; click a second reply while the first plays → first stops, second plays. Click two replies before either fetch resolves, resolving the older one last → only the latest plays. Stop during fetch or body download → nothing starts afterward; stopped/replaced audio releases its blob URL. Safari: first play works from the click, `URL.revokeObjectURL` fires on `ended`. - Text prep: a reply with a fenced block, a table, a `[1]` citation and a stray `[laugh]` → spoken as “Code block omitted”, one sentence per row, the citation intact, the tag gone. Unit-test this; it is pure. - Streaming: send a two-sentence reply token by token; audio should start before `text.done`. Send `text.clear` mid-utterance → `audio.clear`, playback stops with nothing stale. Second utterance on the same socket → fresh `audio.delta`s, no bleed from the first. - Search the client bundle for `XAI_API_KEY`; it must not be there. Against a running dev server, fetch `/`, collect the `/_next/static/chunks/*.js` (or equivalent) URLs it references, and grep each; do not rely on a production build you have not made. diff --git a/grok-voice/tests/read-aloud.test.cjs b/grok-voice/tests/read-aloud.test.cjs new file mode 100644 index 000000000..bdf8b9ae8 --- /dev/null +++ b/grok-voice/tests/read-aloud.test.cjs @@ -0,0 +1,94 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { stripTypeScriptTypes } = require('node:module'); +const test = require('node:test'); +const vm = require('node:vm'); + +function deferred() { + let resolve; + const promise = new Promise((r) => { resolve = r; }); + return { promise, resolve }; +} + +function setup() { + const skill = fs.readFileSync(path.join(__dirname, '../skills/add-read-aloud/SKILL.md'), 'utf8'); + const snippet = skill.match(/```ts\n\/\/ client\n([\s\S]*?)\n```/)[1]; + const requests = [], playing = new Set(), urls = new Set(), audios = []; + let nextUrl = 0; + class Audio { + constructor(url) { this.url = url; this.listeners = {}; audios.push(this); } + addEventListener(name, fn) { this.listeners[name] = fn; } + pause() { playing.delete(this); } + async play() { playing.add(this); } + end() { playing.delete(this); this.listeners.ended(); } + } + const context = vm.createContext({ + Audio, + URL: { + createObjectURL() { const url = `blob:${++nextUrl}`; urls.add(url); return url; }, + revokeObjectURL(url) { urls.delete(url); }, + }, + fetch() { const request = deferred(); requests.push(request); return request.promise; }, + }); + vm.runInContext(stripTypeScriptTypes(snippet), context); + const finish = (index) => requests[index].resolve({ ok: true, async blob() { return new Blob(['audio']); } }); + return { context, requests, playing, urls, audios, finish }; +} + +test('latest request wins when responses arrive out of order', async () => { + const s = setup(); + const old = s.context.readAloud('old'); + const latest = s.context.readAloud('latest'); + s.finish(1); await latest; + const latestAudio = s.audios[0]; + s.finish(0); await old; + assert.equal(s.playing.size, 1); + assert.ok(s.playing.has(latestAudio)); +}); + +test('stop while a request is pending prevents delayed playback', async () => { + const s = setup(); + const pending = s.context.readAloud('pending'); + s.context.stop(); + s.finish(0); await pending; + assert.equal(s.playing.size, 0); + assert.equal(s.urls.size, 0); +}); + +test('stopping active playback releases its blob URL', async () => { + const s = setup(); + const pending = s.context.readAloud('active'); + s.finish(0); await pending; + s.context.stop(); + assert.equal(s.playing.size, 0); + assert.equal(s.urls.size, 0); +}); + +test('replacing active playback releases its blob URL', async () => { + const s = setup(); + const first = s.context.readAloud('first'); s.finish(0); await first; + const second = s.context.readAloud('second'); s.finish(1); await second; + assert.equal(s.playing.size, 1); + assert.equal(s.urls.size, 1); +}); + +test('natural completion releases its blob URL', async () => { + const s = setup(); + const pending = s.context.readAloud('normal'); s.finish(0); await pending; + s.audios[0].end(); + assert.equal(s.urls.size, 0); +}); + +test('stop during body download prevents delayed playback', async () => { + const s = setup(); + const body = deferred(); + const pending = s.context.readAloud('downloading'); + s.requests[0].resolve({ ok: true, blob: () => body.promise }); + await new Promise(setImmediate); + s.context.stop(); + body.resolve(new Blob(['audio'])); + await pending; + assert.equal(s.playing.size, 0); + assert.equal(s.urls.size, 0); +}); From 9ec4f35db1d244da35f963bdb614f7b7b44bc109 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 17:45:54 -0400 Subject: [PATCH 2/9] fix(grok-voice): ignore superseded playback rejections Signed-off-by: Rudy Celekli (cherry picked from commit 7b09b09deda878d218a0a5e30238529989f2becf) --- grok-voice/skills/add-read-aloud/SKILL.md | 1 + grok-voice/tests/read-aloud.test.cjs | 45 ++++++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/grok-voice/skills/add-read-aloud/SKILL.md b/grok-voice/skills/add-read-aloud/SKILL.md index d0cb50da8..435e68d72 100644 --- a/grok-voice/skills/add-read-aloud/SKILL.md +++ b/grok-voice/skills/add-read-aloud/SKILL.md @@ -102,6 +102,7 @@ async function readAloud(text: string, voiceId = "eve") { try { await audio.play(); // call from the click handler's promise chain } catch (error) { + if (request !== generation) return; // stop/replacement can reject a pending play() if (current === audio) stop(); throw error; } diff --git a/grok-voice/tests/read-aloud.test.cjs b/grok-voice/tests/read-aloud.test.cjs index bdf8b9ae8..6109c7325 100644 --- a/grok-voice/tests/read-aloud.test.cjs +++ b/grok-voice/tests/read-aloud.test.cjs @@ -6,12 +6,12 @@ const test = require('node:test'); const vm = require('node:vm'); function deferred() { - let resolve; - const promise = new Promise((r) => { resolve = r; }); - return { promise, resolve }; + let resolve, reject; + const promise = new Promise((r, j) => { resolve = r; reject = j; }); + return { promise, resolve, reject }; } -function setup() { +function setup({ deferPlay = false, playError } = {}) { const skill = fs.readFileSync(path.join(__dirname, '../skills/add-read-aloud/SKILL.md'), 'utf8'); const snippet = skill.match(/```ts\n\/\/ client\n([\s\S]*?)\n```/)[1]; const requests = [], playing = new Set(), urls = new Set(), audios = []; @@ -19,8 +19,12 @@ function setup() { class Audio { constructor(url) { this.url = url; this.listeners = {}; audios.push(this); } addEventListener(name, fn) { this.listeners[name] = fn; } - pause() { playing.delete(this); } - async play() { playing.add(this); } + pause() { playing.delete(this); this.pendingPlay?.reject(Object.assign(new Error("Playback cancelled"), { name: "AbortError" })); } + async play() { + if (playError) throw playError; + playing.add(this); + if (deferPlay) { this.pendingPlay = deferred(); await this.pendingPlay.promise; } + } end() { playing.delete(this); this.listeners.ended(); } } const context = vm.createContext({ @@ -92,3 +96,32 @@ test('stop during body download prevents delayed playback', async () => { assert.equal(s.playing.size, 0); assert.equal(s.urls.size, 0); }); + + +test('stop while play is pending treats pause rejection as cancellation', async () => { + const s = setup({ deferPlay: true }); + const pending = s.context.readAloud('pending playback'); s.finish(0); + await new Promise(setImmediate); + s.context.stop(); + await assert.doesNotReject(pending); + assert.equal(s.urls.size, 0); +}); + +test('replacing pending playback does not reject the cancelled request', async () => { + const s = setup({ deferPlay: true }); + const first = s.context.readAloud('first'); s.finish(0); + await new Promise(setImmediate); + const second = s.context.readAloud('second'); s.finish(1); + await assert.doesNotReject(first); + await new Promise(setImmediate); + s.audios[1].pendingPlay.resolve(); + await second; + assert.equal(s.playing.size, 1); +}); + +test('an active playback failure still rejects and releases its URL', async () => { + const s = setup({ playError: new Error('Audio decode failed') }); + const pending = s.context.readAloud('broken'); s.finish(0); + await assert.rejects(pending, /Audio decode failed/); + assert.equal(s.urls.size, 0); +}); From aaffd4e0fda7579f76bee1b12efad89380ed53c0 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 16:28:56 -0400 Subject: [PATCH 3/9] fix(grok-voice): retain TTS text during upstream handshake Signed-off-by: Rudy Celekli (cherry picked from commit 8576b063b6fabeae045b36df16d907a26d3fd05d) --- grok-voice/skills/add-read-aloud/SKILL.md | 23 +++++++++- grok-voice/tests/tts-relay.test.cjs | 53 +++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 grok-voice/tests/tts-relay.test.cjs diff --git a/grok-voice/skills/add-read-aloud/SKILL.md b/grok-voice/skills/add-read-aloud/SKILL.md index 435e68d72..7ebc00493 100644 --- a/grok-voice/skills/add-read-aloud/SKILL.md +++ b/grok-voice/skills/add-read-aloud/SKILL.md @@ -122,8 +122,27 @@ new WebSocketServer({ port: 8789 }).on("connection", (client) => { const q = new URLSearchParams({ language: "en", voice: "eve", codec: "pcm", sample_rate: "24000" /* optimize_streaming_latency: "1" */ }); const up = new WebSocket(`wss://api.x.ai/v1/tts?${q}`, { headers: { Authorization: `Bearer ${process.env.XAI_API_KEY}` } }); up.on("message", (d) => client.send(d.toString())); // audio.delta, audio.done, audio.clear, session.updated, error - client.on("message", (d) => up.readyState === WebSocket.OPEN && up.send(d.toString())); // text.delta, text.done, text.clear, session.update - const end = () => { client.close(); up.close(); }; + // The browser connection opens before the upstream handshake completes. + // Preserve early text.delta/text.done order, with a bounded pre-open queue. + let pending: string[] = [], pendingBytes = 0; + const end = () => { pending = []; pendingBytes = 0; client.close(); up.close(); }; + up.on("open", () => { + if (client.readyState !== WebSocket.OPEN) return; + pending.forEach((message) => up.send(message)); + pending = []; pendingBytes = 0; + }); + client.on("message", (d) => { + const message = d.toString(); + if (up.readyState === WebSocket.OPEN) up.send(message); + else if (up.readyState === WebSocket.CONNECTING) { + pendingBytes += Buffer.byteLength(message); + if (pendingBytes > 64 * 1024) { + pending = []; pendingBytes = 0; + client.close(1009, "Upstream not ready; pending text limit exceeded"); + up.close(); + } else pending.push(message); + } + }); up.on("close", end); up.on("error", end); client.on("close", end); }); ``` diff --git a/grok-voice/tests/tts-relay.test.cjs b/grok-voice/tests/tts-relay.test.cjs new file mode 100644 index 000000000..4cadfbb6a --- /dev/null +++ b/grok-voice/tests/tts-relay.test.cjs @@ -0,0 +1,53 @@ +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const { stripTypeScriptTypes } = require('node:module'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const vm = require('node:vm'); + +function setup() { + const skill = fs.readFileSync(path.join(__dirname, '../skills/add-read-aloud/SKILL.md'), 'utf8'); + const snippet = skill.match(/```ts\n(import \{ WebSocketServer, WebSocket \} from "ws";[\s\S]*?)\n```/)[1]; + let server, upstream; + class Socket extends EventEmitter { + static CONNECTING = 0; static OPEN = 1; static CLOSED = 3; + constructor() { super(); this.readyState = Socket.CONNECTING; this.sent = []; upstream = this; } + send(data) { assert.equal(this.readyState, Socket.OPEN); this.sent.push(data); } + close(code) { this.readyState = Socket.CLOSED; this.closeCode = code; } + } + class Server extends EventEmitter { constructor() { super(); server = this; } } + const context = vm.createContext({ WebSocketServer: Server, WebSocket: Socket, URLSearchParams, Buffer, process: { env: { XAI_API_KEY: 'offline-test' } } }); + vm.runInContext(stripTypeScriptTypes(snippet.replace('import { WebSocketServer, WebSocket } from "ws";', '')), context); + const client = new Socket(); client.readyState = Socket.OPEN; + server.emit('connection', client); + return { client, upstream, open() { upstream.readyState = Socket.OPEN; upstream.emit('open'); } }; +} + +test('messages received during upstream handshake are forwarded in order', () => { + const s = setup(); + const messages = ['{"type":"text.delta","delta":"hello"}', '{"type":"text.done"}']; + messages.forEach((message) => s.client.emit('message', Buffer.from(message))); + s.open(); + assert.deepEqual(s.upstream.sent, messages); +}); + +test('messages after upstream open are forwarded immediately', () => { + const s = setup(); s.open(); + s.client.emit('message', Buffer.from('{"type":"text.done"}')); + assert.deepEqual(s.upstream.sent, ['{"type":"text.done"}']); +}); + +test('closing browser before upstream opens discards queued work', () => { + const s = setup(); + s.client.emit('message', Buffer.from('pending')); + s.client.close(); s.client.emit('close'); + s.open(); + assert.deepEqual(s.upstream.sent, []); +}); + +test('pre-open buffer rejects excessive data rather than growing forever', () => { + const s = setup(); + s.client.emit('message', Buffer.alloc(65 * 1024, 'x')); + assert.equal(s.client.closeCode, 1009); +}); From f4758f4e664f7d5dcd9a35c78b59023e94fa3d5c Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 17:42:01 -0400 Subject: [PATCH 4/9] fix(grok-voice): retain dictation frames during relay handshake Signed-off-by: Rudy Celekli (cherry picked from commit 687b9652fa6f114bfebe35d6ed0ab11119540e99) --- grok-voice/skills/add-dictation/SKILL.md | 22 +++++++- grok-voice/tests/stt-relay.test.cjs | 69 ++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 grok-voice/tests/stt-relay.test.cjs diff --git a/grok-voice/skills/add-dictation/SKILL.md b/grok-voice/skills/add-dictation/SKILL.md index b651704d1..b2a547266 100644 --- a/grok-voice/skills/add-dictation/SKILL.md +++ b/grok-voice/skills/add-dictation/SKILL.md @@ -95,8 +95,26 @@ new WebSocketServer({ port: 8788 }).on("connection", (client) => { const q = new URLSearchParams({ model: "grok-voice-transcribe-2.0", sample_rate: "16000", encoding: "pcm", interim_results: "true", language: "en" }); const up = new WebSocket(`wss://api.x.ai/v1/stt?${q}`, { headers: { Authorization: `Bearer ${process.env.XAI_API_KEY}` } }); up.on("message", (d) => client.send(d.toString())); // transcript.* and error events - client.on("message", (d, isBinary) => up.readyState === WebSocket.OPEN && up.send(d, { binary: isBinary })); // audio + finalize/audio.done - const end = () => { client.close(); up.close(); }; + let pending: { data: Buffer; isBinary: boolean }[] = [], pendingBytes = 0; + const end = () => { pending = []; pendingBytes = 0; client.close(); up.close(); }; + up.on("open", () => { + if (client.readyState !== WebSocket.OPEN) return; + pending.forEach(({ data, isBinary }) => up.send(data, { binary: isBinary })); + pending = []; pendingBytes = 0; + }); + client.on("message", (d, isBinary) => { + // ws RawData is Buffer, ArrayBuffer, or Buffer[]; preserve bytes and frame kind. + const data = Array.isArray(d) ? Buffer.concat(d) : Buffer.from(d); + if (up.readyState === WebSocket.OPEN) up.send(data, { binary: isBinary }); + else if (up.readyState === WebSocket.CONNECTING) { + pendingBytes += data.byteLength; + if (pendingBytes > 64 * 1024) { + pending = []; pendingBytes = 0; + client.close(1009, "Upstream not ready; pending audio limit exceeded"); + up.close(); + } else pending.push({ data, isBinary }); + } + }); // audio + finalize/audio.done up.on("close", end); up.on("error", end); client.on("close", end); }); ``` diff --git a/grok-voice/tests/stt-relay.test.cjs b/grok-voice/tests/stt-relay.test.cjs new file mode 100644 index 000000000..970274df0 --- /dev/null +++ b/grok-voice/tests/stt-relay.test.cjs @@ -0,0 +1,69 @@ +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const { stripTypeScriptTypes } = require('node:module'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const vm = require('node:vm'); + +function setup() { + const skill = fs.readFileSync(path.join(__dirname, '../skills/add-dictation/SKILL.md'), 'utf8'); + const snippet = skill.match(/```ts\n(import \{ WebSocketServer, WebSocket \} from "ws";[\s\S]*?)\n```/)[1]; + let server, upstream; + class Socket extends EventEmitter { + static CONNECTING = 0; static OPEN = 1; static CLOSED = 3; + constructor() { super(); this.readyState = Socket.CONNECTING; this.sent = []; upstream = this; } + send(data, options) { assert.equal(this.readyState, Socket.OPEN); this.sent.push({ data: Buffer.from(data), binary: options.binary }); } + close(code) { this.readyState = Socket.CLOSED; this.closeCode = code; } + } + class Server extends EventEmitter { constructor() { super(); server = this; } } + const context = vm.createContext({ WebSocketServer: Server, WebSocket: Socket, URLSearchParams, Buffer, process: { env: { XAI_API_KEY: 'offline-test' } } }); + vm.runInContext(stripTypeScriptTypes(snippet.replace('import { WebSocketServer, WebSocket } from "ws";', '')), context); + const client = new Socket(); client.readyState = Socket.OPEN; + server.emit('connection', client); + return { client, upstream, open() { upstream.readyState = Socket.OPEN; upstream.emit('open'); } }; +} + +test('audio and finalize received during handshake flush in order with original frame kinds', () => { + const s = setup(); + const audio = Buffer.from([1, 0, 255, 127]); + const control = Buffer.from('{"type":"Finalize"}'); + s.client.emit('message', audio, true); + s.client.emit('message', control, false); + s.open(); + assert.deepEqual(s.upstream.sent, [{ data: audio, binary: true }, { data: control, binary: false }]); +}); + +test('all ws RawData forms retain exact binary bytes while queued', () => { + const s = setup(); + s.client.emit('message', Uint8Array.from([0, 255, 1, 2]).buffer, true); + s.client.emit('message', [Buffer.from([3, 4]), Buffer.from([5, 6])], true); + s.open(); + assert.deepEqual(s.upstream.sent, [ + { data: Buffer.from([0, 255, 1, 2]), binary: true }, + { data: Buffer.from([3, 4, 5, 6]), binary: true }, + ]); +}); + +test('frames after upstream opens forward immediately', () => { + const s = setup(); s.open(); + const audio = Buffer.from([1, 2]); s.client.emit('message', audio, true); + assert.deepEqual(s.upstream.sent, [{ data: audio, binary: true }]); +}); + +test('closing browser during handshake discards queued audio', () => { + const s = setup(); s.client.emit('message', Buffer.from([1, 2]), true); + s.client.close(); s.client.emit('close'); s.open(); + assert.deepEqual(s.upstream.sent, []); +}); + +test('audio buffering is bounded and exceeding the cap closes both sockets', () => { + const s = setup(); + s.client.emit('message', Buffer.alloc(64 * 1024), true); + assert.equal(s.client.readyState, 1); + s.client.emit('message', Buffer.from([1]), true); + assert.equal(s.client.closeCode, 1009); + assert.equal(s.upstream.readyState, 3); + s.open(); + assert.deepEqual(s.upstream.sent, []); +}); From 3560da7e6384a74b115c060ccc10f7be45cbced5 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 16:29:23 -0400 Subject: [PATCH 5/9] fix(grok-voice): release completed streaming audio sources Signed-off-by: Rudy Celekli (cherry picked from commit a9fa8d7a7f249d9f285e7cafa16ec1e601fb0acc) --- grok-voice/skills/add-read-aloud/SKILL.md | 6 ++- grok-voice/tests/tts-player.test.cjs | 61 +++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 grok-voice/tests/tts-player.test.cjs diff --git a/grok-voice/skills/add-read-aloud/SKILL.md b/grok-voice/skills/add-read-aloud/SKILL.md index 7ebc00493..ae587d7a5 100644 --- a/grok-voice/skills/add-read-aloud/SKILL.md +++ b/grok-voice/skills/add-read-aloud/SKILL.md @@ -163,6 +163,10 @@ ws.addEventListener("message", (e) => { const ch = buf.getChannelData(0); for (let i = 0; i < pcm.length; i++) ch[i] = pcm[i] / 32768; const src = ctx.createBufferSource(); src.buffer = buf; src.connect(ctx.destination); + src.onended = () => { + src.disconnect(); + sources = sources.filter((source) => source !== src); // retain queued/playing audio only + }; playhead = Math.max(playhead, ctx.currentTime + 0.15); // ~150 ms lead so chunks butt together src.start(playhead); playhead += buf.duration; sources.push(src); } else if (ev.type === "audio.done") { /* utterance finished; socket stays open */ } @@ -209,7 +213,7 @@ r.raise_for_status(); audio_bytes = r.content # audio/mpeg - Batch: `curl -X POST https://api.x.ai/v1/tts -H "Authorization: Bearer $XAI_API_KEY" -H "Content-Type: application/json" -d '{"text":"Hello from read aloud.","voice_id":"eve","language":"en"}' --output /tmp/hello.mp3` → 200 `audio/mpeg` (MP3, 24 kHz, 128 kbps, mono), plays. Omit `language` → 422 (observed; the docs’ table only lists 400). `voice_id: "nope"` → 404. - In the app: no speaker while a reply streams; it appears when the reply ends. Click → spinner → stop square → back to speaker when audio ends; click again mid-play → stops at once; click a second reply while the first plays → first stops, second plays. Click two replies before either fetch resolves, resolving the older one last → only the latest plays. Stop during fetch or body download → nothing starts afterward; stopped/replaced audio releases its blob URL. Safari: first play works from the click, `URL.revokeObjectURL` fires on `ended`. - Text prep: a reply with a fenced block, a table, a `[1]` citation and a stray `[laugh]` → spoken as “Code block omitted”, one sentence per row, the citation intact, the tag gone. Unit-test this; it is pure. - - Streaming: send a two-sentence reply token by token; audio should start before `text.done`. Send `text.clear` mid-utterance → `audio.clear`, playback stops with nothing stale. Second utterance on the same socket → fresh `audio.delta`s, no bleed from the first. + - Streaming: send a two-sentence reply token by token; audio should start before `text.done`. Send `text.clear` mid-utterance → `audio.clear`, playback stops with nothing stale. Second utterance on the same socket → fresh `audio.delta`s, no bleed from the first. After playback drains, the active-source collection is empty; finished chunks and their PCM buffers are released between utterances. - Search the client bundle for `XAI_API_KEY`; it must not be there. Against a running dev server, fetch `/`, collect the `/_next/static/chunks/*.js` (or equivalent) URLs it references, and grep each; do not rely on a production build you have not made. - Debug from logs with `/debug-voice`; swap its hook points to `audio.delta` (byte counts), `audio.done`, `audio.clear`, `error`. diff --git a/grok-voice/tests/tts-player.test.cjs b/grok-voice/tests/tts-player.test.cjs new file mode 100644 index 000000000..d4ee2006f --- /dev/null +++ b/grok-voice/tests/tts-player.test.cjs @@ -0,0 +1,61 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const { stripTypeScriptTypes } = require('node:module'); +const path = require('node:path'); +const test = require('node:test'); +const vm = require('node:vm'); + +function setup() { + const skill = fs.readFileSync(path.join(__dirname, '../skills/add-read-aloud/SKILL.md'), 'utf8'); + const snippet = skill.match(/```ts\n(const ws = new WebSocket\(relayUrl\);[\s\S]*?)\n```/)[1]; + let socket; + const nodes = []; + class Socket { + constructor() { socket = this; this.handlers = {}; } + addEventListener(type, fn) { this.handlers[type] = fn; } + } + class AudioContext { + constructor() { this.currentTime = 1; this.destination = {}; } + createBuffer(channels, length, rate) { + return { duration: length / rate, getChannelData() { return new Float32Array(length); } }; + } + createBufferSource() { + const node = { + connect() {}, disconnect() { this.disconnected = true; }, start() {}, + stop() { this.stopped = true; }, + end() { if (this.onended) this.onended(); }, + }; + nodes.push(node); return node; + } + } + const context = vm.createContext({ WebSocket: Socket, AudioContext, relayUrl: 'offline', atob, showError() {} }); + vm.runInContext(stripTypeScriptTypes(snippet), context); + const event = (data) => socket.handlers.message({ data: JSON.stringify(data) }); + return { + nodes, context, + chunk() { event({ type: 'audio.delta', delta: Buffer.from([0, 0, 255, 127]).toString('base64') }); }, + clear() { event({ type: 'audio.clear' }); }, + retained() { return vm.runInContext('sources.length', context); }, + }; +} + +test('completed chunks are released across utterances on one session', () => { + const s = setup(); + for (let i = 0; i < 100; i++) { s.chunk(); s.nodes[i].end(); } + assert.equal(s.retained(), 0); + assert.ok(s.nodes.every((node) => node.disconnected)); +}); + +test('completion removes only the finished node', () => { + const s = setup(); s.chunk(); s.chunk(); + s.nodes[0].end(); + assert.equal(s.retained(), 1); + assert.equal(s.nodes[1].stopped, undefined); +}); + +test('barge-in still stops queued sources and resets scheduling', () => { + const s = setup(); s.chunk(); s.chunk(); s.clear(); + assert.ok(s.nodes.every((node) => node.stopped)); + assert.equal(s.retained(), 0); + assert.equal(vm.runInContext('playhead', s.context), 0); +}); From 48cd55a4a8f37de080f30dc44e13b2dec4789941 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 17:34:25 -0400 Subject: [PATCH 6/9] fix(grok-voice): release microphone capture after dictation Signed-off-by: Rudy Celekli (cherry picked from commit 4fdc0366bae430d87653d3d5e95abee34a17f334) --- grok-voice/skills/add-dictation/SKILL.md | 3 +- grok-voice/tests/dictation-capture.test.cjs | 49 +++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 grok-voice/tests/dictation-capture.test.cjs diff --git a/grok-voice/skills/add-dictation/SKILL.md b/grok-voice/skills/add-dictation/SKILL.md index b2a547266..4d420e08a 100644 --- a/grok-voice/skills/add-dictation/SKILL.md +++ b/grok-voice/skills/add-dictation/SKILL.md @@ -71,12 +71,13 @@ export async function transcribe(blob: Blob, filename: string) { ``` ```ts -// client +// client (stream is a dedicated microphone stream owned by this recording) const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : "audio/mp4"; const rec = new MediaRecorder(stream, { mimeType: mime }); const parts: BlobPart[] = []; rec.ondataavailable = (e) => parts.push(e.data); rec.onstop = async () => { + stream.getTracks().forEach((track) => track.stop()); // MediaRecorder.stop() does not release the microphone const fd = new FormData(); fd.append("file", new Blob(parts, { type: mime }), "dictation"); const { text } = await (await fetch("/api/dictation", { method: "POST", body: fd })).json(); diff --git a/grok-voice/tests/dictation-capture.test.cjs b/grok-voice/tests/dictation-capture.test.cjs new file mode 100644 index 000000000..4282cad8f --- /dev/null +++ b/grok-voice/tests/dictation-capture.test.cjs @@ -0,0 +1,49 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { stripTypeScriptTypes } = require('node:module'); +const test = require('node:test'); +const vm = require('node:vm'); + +function setup() { + const skill = fs.readFileSync(path.join(__dirname, '../skills/add-dictation/SKILL.md'), 'utf8'); + const snippet = skill.match(/```ts\n\/\/ client[^\n]*\n([\s\S]*?)\n```/)[1]; + const tracks = [{ live: true, stop() { this.live = false; } }, { live: true, stop() { this.live = false; } }]; + let recorder, resolveRequest, rejectRequest, body; + const inserted = []; + class Recorder { + static isTypeSupported() { return true; } + constructor(stream) { this.stream = stream; recorder = this; } + start() {} + } + const context = vm.createContext({ + MediaRecorder: Recorder, stream: { getTracks: () => tracks }, Blob, FormData, + fetch(url, options) { body = options.body; return new Promise((resolve, reject) => { resolveRequest = resolve; rejectRequest = reject; }); }, + insertAtCursor(text) { inserted.push(text); }, + }); + vm.runInContext(stripTypeScriptTypes(snippet), context); + recorder.ondataavailable({ data: new Blob(['recorded audio']) }); + return { tracks, recorder, inserted, body: () => body, + resolve() { resolveRequest({ async json() { return { text: 'hello' }; } }); }, + reject() { rejectRequest(new Error('offline')); }, + }; +} + +test('recording stop releases every microphone track before upload completes', async () => { + const s = setup(); const pending = s.recorder.onstop(); + assert.equal(s.tracks.filter((track) => track.live).length, 0); + s.resolve(); await pending; +}); + +test('a failed upload leaves no microphone capture running', async () => { + const s = setup(); const pending = s.recorder.onstop(); + s.reject(); await assert.rejects(pending, /offline/); + assert.ok(s.tracks.every((track) => !track.live)); +}); + +test('releasing capture preserves the recorded blob and successful transcription', async () => { + const s = setup(); const pending = s.recorder.onstop(); + assert.equal(await s.body().get('file').text(), 'recorded audio'); + s.resolve(); await pending; + assert.deepEqual(s.inserted, ['hello']); +}); From 65722dd7899b44194afaacf0b35f03207a96fe3b Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 17:57:02 -0400 Subject: [PATCH 7/9] test(grok-voice): use a real dictation HTTP response Signed-off-by: Rudy Celekli (cherry picked from commit 677c8934819a958697d67917d046f810f0998c79) --- grok-voice/tests/dictation-capture.test.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grok-voice/tests/dictation-capture.test.cjs b/grok-voice/tests/dictation-capture.test.cjs index 4282cad8f..d76df84a5 100644 --- a/grok-voice/tests/dictation-capture.test.cjs +++ b/grok-voice/tests/dictation-capture.test.cjs @@ -24,7 +24,7 @@ function setup() { vm.runInContext(stripTypeScriptTypes(snippet), context); recorder.ondataavailable({ data: new Blob(['recorded audio']) }); return { tracks, recorder, inserted, body: () => body, - resolve() { resolveRequest({ async json() { return { text: 'hello' }; } }); }, + resolve() { resolveRequest(new Response(JSON.stringify({ text: 'hello' }), { status: 200 })); }, reject() { rejectRequest(new Error('offline')); }, }; } From 55bbf37da05bf6499fb285d3a1b163fee8c5035d Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 17:35:00 -0400 Subject: [PATCH 8/9] fix(grok-voice): reject failed dictation uploads Signed-off-by: Rudy Celekli (cherry picked from commit a77509c8e31291ee7812864c4b2a4e978079ed77) --- grok-voice/skills/add-dictation/SKILL.md | 4 ++- grok-voice/tests/dictation-http.test.cjs | 42 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 grok-voice/tests/dictation-http.test.cjs diff --git a/grok-voice/skills/add-dictation/SKILL.md b/grok-voice/skills/add-dictation/SKILL.md index 4d420e08a..8e1a0f142 100644 --- a/grok-voice/skills/add-dictation/SKILL.md +++ b/grok-voice/skills/add-dictation/SKILL.md @@ -80,7 +80,9 @@ rec.onstop = async () => { stream.getTracks().forEach((track) => track.stop()); // MediaRecorder.stop() does not release the microphone const fd = new FormData(); fd.append("file", new Blob(parts, { type: mime }), "dictation"); - const { text } = await (await fetch("/api/dictation", { method: "POST", body: fd })).json(); + const res = await fetch("/api/dictation", { method: "POST", body: fd }); + if (!res.ok) throw new Error(`Dictation ${res.status}`); // surface through the app's existing error UI + const { text } = await res.json(); insertAtCursor(text); }; rec.start(); // second tap: rec.stop() diff --git a/grok-voice/tests/dictation-http.test.cjs b/grok-voice/tests/dictation-http.test.cjs new file mode 100644 index 000000000..6eaebce0a --- /dev/null +++ b/grok-voice/tests/dictation-http.test.cjs @@ -0,0 +1,42 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { stripTypeScriptTypes } = require('node:module'); +const test = require('node:test'); +const vm = require('node:vm'); + +function setup(status) { + const skill = fs.readFileSync(path.join(__dirname, '../skills/add-dictation/SKILL.md'), 'utf8'); + const snippet = skill.match(/```ts\n\/\/ client[^\n]*\n([\s\S]*?)\n```/)[1]; + let recorder, bodyReads = 0; + const inserted = []; + class Recorder { + static isTypeSupported() { return true; } + constructor() { recorder = this; } + start() {} + } + const context = vm.createContext({ + MediaRecorder: Recorder, stream: { getTracks: () => [] }, Blob, FormData, + async fetch() { return { ok: status >= 200 && status < 300, status, + async json() { bodyReads++; return status === 200 ? { text: 'spoken words' } : { error: 'request failed' }; } }; }, + insertAtCursor(text) { inserted.push(text); }, + }); + vm.runInContext(stripTypeScriptTypes(snippet), context); + recorder.ondataavailable({ data: new Blob(['audio']) }); + return { recorder, inserted, bodyReads: () => bodyReads }; +} + +for (const status of [401, 429, 503]) { + test(`HTTP ${status} rejects without inserting an error response into the composer`, async () => { + const s = setup(status); + await assert.rejects(s.recorder.onstop(), new RegExp(`Dictation ${status}`)); + assert.deepEqual(s.inserted, []); + assert.equal(s.bodyReads(), 0); + }); +} + +test('successful dictation still inserts the returned transcript once', async () => { + const s = setup(200); await s.recorder.onstop(); + assert.deepEqual(s.inserted, ['spoken words']); + assert.equal(s.bodyReads(), 1); +}); From f15223f96df9beacf2771112c9bd0a49d2aab7f3 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 17:37:58 -0400 Subject: [PATCH 9/9] fix(grok-voice): contain browser relay transport errors Signed-off-by: Rudy Celekli (cherry picked from commit b1138287519f819fad362de3b9af53e905018ac3) --- grok-voice/skills/add-dictation/SKILL.md | 1 + grok-voice/skills/add-read-aloud/SKILL.md | 1 + grok-voice/tests/relay-client-errors.test.cjs | 46 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 grok-voice/tests/relay-client-errors.test.cjs diff --git a/grok-voice/skills/add-dictation/SKILL.md b/grok-voice/skills/add-dictation/SKILL.md index 8e1a0f142..3ac47bd68 100644 --- a/grok-voice/skills/add-dictation/SKILL.md +++ b/grok-voice/skills/add-dictation/SKILL.md @@ -119,6 +119,7 @@ new WebSocketServer({ port: 8788 }).on("connection", (client) => { } }); // audio + finalize/audio.done up.on("close", end); up.on("error", end); client.on("close", end); + client.on("error", end); // contain browser transport failures }); ``` diff --git a/grok-voice/skills/add-read-aloud/SKILL.md b/grok-voice/skills/add-read-aloud/SKILL.md index ae587d7a5..2b6b09744 100644 --- a/grok-voice/skills/add-read-aloud/SKILL.md +++ b/grok-voice/skills/add-read-aloud/SKILL.md @@ -144,6 +144,7 @@ new WebSocketServer({ port: 8789 }).on("connection", (client) => { } }); up.on("close", end); up.on("error", end); client.on("close", end); + client.on("error", end); // contain browser transport failures }); ``` diff --git a/grok-voice/tests/relay-client-errors.test.cjs b/grok-voice/tests/relay-client-errors.test.cjs new file mode 100644 index 000000000..94360c59e --- /dev/null +++ b/grok-voice/tests/relay-client-errors.test.cjs @@ -0,0 +1,46 @@ +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const { stripTypeScriptTypes } = require('node:module'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const vm = require('node:vm'); + +function setup(skillName) { + const skill = fs.readFileSync(path.join(__dirname, `../skills/${skillName}/SKILL.md`), 'utf8'); + const snippet = skill.match(/```ts\n(import \{ WebSocketServer, WebSocket \} from "ws";[\s\S]*?)\n```/)[1]; + let server, upstream; + // ws sockets inherit EventEmitter: an unhandled 'error' throws synchronously. + class Socket extends EventEmitter { + static CONNECTING = 0; static OPEN = 1; static CLOSED = 3; + constructor() { super(); this.readyState = Socket.OPEN; this.sent = []; upstream = this; } + send(data) { this.sent.push(data); } + close() { this.readyState = Socket.CLOSED; } + } + class Server extends EventEmitter { constructor() { super(); server = this; } } + const context = vm.createContext({ WebSocketServer: Server, WebSocket: Socket, URLSearchParams, Buffer, process: { env: { XAI_API_KEY: 'offline-test' } } }); + vm.runInContext(stripTypeScriptTypes(snippet.replace('import { WebSocketServer, WebSocket } from "ws";', '')), context); + const client = new Socket(); + server.emit('connection', client); + return { client, upstream }; +} + +for (const skill of ['add-dictation', 'add-read-aloud']) { + test(`${skill}: browser transport errors are contained and close both sockets`, () => { + const s = setup(skill); + assert.doesNotThrow(() => s.client.emit('error', Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }))); + assert.equal(s.client.readyState, 3); + assert.equal(s.upstream.readyState, 3); + }); + test(`${skill}: upstream errors still close both sockets`, () => { + const s = setup(skill); + assert.doesNotThrow(() => s.upstream.emit('error', new Error('upstream unavailable'))); + assert.equal(s.client.readyState, 3); + assert.equal(s.upstream.readyState, 3); + }); + test(`${skill}: ordinary transcript/audio messages still reach the browser`, () => { + const s = setup(skill); + s.upstream.emit('message', Buffer.from('{"type":"event"}')); + assert.deepEqual(s.client.sent, ['{"type":"event"}']); + }); +}