grok-voice: six community fixes (stale playback, handshake buffering, mic and source release, failed uploads, relay errors)

grok-voice/skills/add-dictation/SKILL.md

@@ -71,15 +71,18 @@ export async function transcribe(blob: Blob, filename: string) {
 ```
 
 ```ts
-// client
+// client (stream is a dedicated microphone stream owned by this recording)
 const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : "audio/mp4";
 const rec = new MediaRecorder(stream, { mimeType: mime });
 const parts: BlobPart[] = [];
 rec.ondataavailable = (e) => parts.push(e.data);
 rec.onstop = async () => {
+  stream.getTracks().forEach((track) => track.stop()); // MediaRecorder.stop() does not release the microphone
   const fd = new FormData();
   fd.append("file", new Blob(parts, { type: mime }), "dictation");
-  const { text } = await (await fetch("/api/dictation", { method: "POST", body: fd })).json();
+  const res = await fetch("/api/dictation", { method: "POST", body: fd });
+  if (!res.ok) throw new Error(`Dictation ${res.status}`); // surface through the app's existing error UI
+  const { text } = await res.json();
   insertAtCursor(text);
 };
 rec.start(); // second tap: rec.stop()
@@ -95,9 +98,28 @@ new WebSocketServer({ port: 8788 }).on("connection", (client) => {
   const q = new URLSearchParams({ model: "grok-voice-transcribe-2.0", sample_rate: "16000", encoding: "pcm", interim_results: "true", language: "en" });
   const up = new WebSocket(`wss://api.x.ai/v1/stt?${q}`, { headers: { Authorization: `Bearer ${process.env.XAI_API_KEY}` } });
   up.on("message", (d) => client.send(d.toString()));                       // transcript.* and error events
-  client.on("message", (d, isBinary) => up.readyState === WebSocket.OPEN && up.send(d, { binary: isBinary })); // audio + finalize/audio.done
-  const end = () => { client.close(); up.close(); };
+  let pending: { data: Buffer; isBinary: boolean }[] = [], pendingBytes = 0;
+  const end = () => { pending = []; pendingBytes = 0; client.close(); up.close(); };
+  up.on("open", () => {
+    if (client.readyState !== WebSocket.OPEN) return;
+    pending.forEach(({ data, isBinary }) => up.send(data, { binary: isBinary }));
+    pending = []; pendingBytes = 0;
+  });
+  client.on("message", (d, isBinary) => {
+    // ws RawData is Buffer, ArrayBuffer, or Buffer[]; preserve bytes and frame kind.
+    const data = Array.isArray(d) ? Buffer.concat(d) : Buffer.from(d);
+    if (up.readyState === WebSocket.OPEN) up.send(data, { binary: isBinary });
+    else if (up.readyState === WebSocket.CONNECTING) {
+      pendingBytes += data.byteLength;
+      if (pendingBytes > 64 * 1024) {
+        pending = []; pendingBytes = 0;
+        client.close(1009, "Upstream not ready; pending audio limit exceeded");
+        up.close();
+      } else pending.push({ data, isBinary });
+    }
+  }); // audio + finalize/audio.done
   up.on("close", end); up.on("error", end); client.on("close", end);
+  client.on("error", end); // contain browser transport failures
 });
 ```
 

grok-voice/skills/add-read-aloud/SKILL.md

@@ -77,18 +77,36 @@ export async function speak(text: string, voice_id = "eve", language = "auto") {
 
 ```ts
 // client
+// Keep this player in the shared store, not in each message button.
 let current: HTMLAudioElement | null = null;
+let currentUrl: string | null = null;
+let generation = 0;
+function stop() {
+  generation++; // invalidate pending requests as well as active playback
+  current?.pause(); current = null;
+  if (currentUrl) URL.revokeObjectURL(currentUrl);
+  currentUrl = null;
+}
 async function readAloud(text: string, voiceId = "eve") {
-  current?.pause(); current = null;                       // one utterance at a time
+  stop();
+  const request = generation;
   const res = await fetch("/api/tts", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text, voice_id: voiceId }) });
+  if (request !== generation) return;
   if (!res.ok) throw new Error("TTS request failed");
-  const url = URL.createObjectURL(await res.blob());
+  const blob = await res.blob();
+  if (request !== generation) return; // stop/new click may occur while reading the body
+  const url = URL.createObjectURL(blob);
   const audio = new Audio(url);
-  audio.addEventListener("ended", () => URL.revokeObjectURL(url)); // avoid blob leaks
-  current = audio;
-  await audio.play();                                      // call from the click handler’s promise chain
+  current = audio; currentUrl = url;
+  audio.addEventListener("ended", () => { if (current === audio) stop(); });
+  try {
+    await audio.play(); // call from the click handler's promise chain
+  } catch (error) {
+    if (request !== generation) return; // stop/replacement can reject a pending play()
+    if (current === audio) stop();
+    throw error;
+  }
 }
-function stop() { current?.pause(); current = null; }
 ```
 
    - Safari: `audio.duration` is `Infinity` on blob URLs. If you need a progress bar, decode with `AudioContext.decodeAudioData(buf.slice(0))` or request `with_timestamps: true` and read `duration` from the JSON envelope (audio is then base64 in `audio`).
@@ -104,9 +122,29 @@ new WebSocketServer({ port: 8789 }).on("connection", (client) => {
   const q = new URLSearchParams({ language: "en", voice: "eve", codec: "pcm", sample_rate: "24000" /* optimize_streaming_latency: "1" */ });
   const up = new WebSocket(`wss://api.x.ai/v1/tts?${q}`, { headers: { Authorization: `Bearer ${process.env.XAI_API_KEY}` } });
   up.on("message", (d) => client.send(d.toString()));                                   // audio.delta, audio.done, audio.clear, session.updated, error
-  client.on("message", (d) => up.readyState === WebSocket.OPEN && up.send(d.toString())); // text.delta, text.done, text.clear, session.update
-  const end = () => { client.close(); up.close(); };
+  // The browser connection opens before the upstream handshake completes.
+  // Preserve early text.delta/text.done order, with a bounded pre-open queue.
+  let pending: string[] = [], pendingBytes = 0;
+  const end = () => { pending = []; pendingBytes = 0; client.close(); up.close(); };
+  up.on("open", () => {
+    if (client.readyState !== WebSocket.OPEN) return;
+    pending.forEach((message) => up.send(message));
+    pending = []; pendingBytes = 0;
+  });
+  client.on("message", (d) => {
+    const message = d.toString();
+    if (up.readyState === WebSocket.OPEN) up.send(message);
+    else if (up.readyState === WebSocket.CONNECTING) {
+      pendingBytes += Buffer.byteLength(message);
+      if (pendingBytes > 64 * 1024) {
+        pending = []; pendingBytes = 0;
+        client.close(1009, "Upstream not ready; pending text limit exceeded");
+        up.close();
+      } else pending.push(message);
+    }
+  });
   up.on("close", end); up.on("error", end); client.on("close", end);
+  client.on("error", end); // contain browser transport failures
 });
 ```
 
@@ -126,6 +164,10 @@ ws.addEventListener("message", (e) => {
     const ch = buf.getChannelData(0);
     for (let i = 0; i < pcm.length; i++) ch[i] = pcm[i] / 32768;
     const src = ctx.createBufferSource(); src.buffer = buf; src.connect(ctx.destination);
+    src.onended = () => {
+      src.disconnect();
+      sources = sources.filter((source) => source !== src); // retain queued/playing audio only
+    };
     playhead = Math.max(playhead, ctx.currentTime + 0.15);   // ~150 ms lead so chunks butt together
     src.start(playhead); playhead += buf.duration; sources.push(src);
   } else if (ev.type === "audio.done") { /* utterance finished; socket stays open */ }
@@ -170,9 +212,9 @@ r.raise_for_status(); audio_bytes = r.content  # audio/mpeg
 
 7. **Smoke**
    - Batch: `curl -X POST https://api.x.ai/v1/tts -H "Authorization: Bearer $XAI_API_KEY" -H "Content-Type: application/json" -d '{"text":"Hello from read aloud.","voice_id":"eve","language":"en"}' --output /tmp/hello.mp3` → 200 `audio/mpeg` (MP3, 24 kHz, 128 kbps, mono), plays. Omit `language` → 422 (observed; the docs’ table only lists 400). `voice_id: "nope"` → 404.
-   - In the app: no speaker while a reply streams; it appears when the reply ends. Click → spinner → stop square → back to speaker when audio ends; click again mid-play → stops at once; click a second reply while the first plays → first stops, second plays. Safari: first play works from the click, `URL.revokeObjectURL` fires on `ended`.
+   - In the app: no speaker while a reply streams; it appears when the reply ends. Click → spinner → stop square → back to speaker when audio ends; click again mid-play → stops at once; click a second reply while the first plays → first stops, second plays. Click two replies before either fetch resolves, resolving the older one last → only the latest plays. Stop during fetch or body download → nothing starts afterward; stopped/replaced audio releases its blob URL. Safari: first play works from the click, `URL.revokeObjectURL` fires on `ended`.
    - Text prep: a reply with a fenced block, a table, a `[1]` citation and a stray `[laugh]` → spoken as “Code block omitted”, one sentence per row, the citation intact, the tag gone. Unit-test this; it is pure.
-   - Streaming: send a two-sentence reply token by token; audio should start before `text.done`. Send `text.clear` mid-utterance → `audio.clear`, playback stops with nothing stale. Second utterance on the same socket → fresh `audio.delta`s, no bleed from the first.
+   - Streaming: send a two-sentence reply token by token; audio should start before `text.done`. Send `text.clear` mid-utterance → `audio.clear`, playback stops with nothing stale. Second utterance on the same socket → fresh `audio.delta`s, no bleed from the first. After playback drains, the active-source collection is empty; finished chunks and their PCM buffers are released between utterances.
    - Search the client bundle for `XAI_API_KEY`; it must not be there. Against a running dev server, fetch `/`, collect the `/_next/static/chunks/*.js` (or equivalent) URLs it references, and grep each; do not rely on a production build you have not made.
    - Debug from logs with `/debug-voice`; swap its hook points to `audio.delta` (byte counts), `audio.done`, `audio.clear`, `error`.
 

grok-voice/tests/dictation-capture.test.cjs

@@ -0,0 +1,49 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const { stripTypeScriptTypes } = require('node:module');
+const test = require('node:test');
+const vm = require('node:vm');
+
+function setup() {
+  const skill = fs.readFileSync(path.join(__dirname, '../skills/add-dictation/SKILL.md'), 'utf8');
+  const snippet = skill.match(/```ts\n\/\/ client[^\n]*\n([\s\S]*?)\n```/)[1];
+  const tracks = [{ live: true, stop() { this.live = false; } }, { live: true, stop() { this.live = false; } }];
+  let recorder, resolveRequest, rejectRequest, body;
+  const inserted = [];
+  class Recorder {
+    static isTypeSupported() { return true; }
+    constructor(stream) { this.stream = stream; recorder = this; }
+    start() {}
+  }
+  const context = vm.createContext({
+    MediaRecorder: Recorder, stream: { getTracks: () => tracks }, Blob, FormData,
+    fetch(url, options) { body = options.body; return new Promise((resolve, reject) => { resolveRequest = resolve; rejectRequest = reject; }); },
+    insertAtCursor(text) { inserted.push(text); },
+  });
+  vm.runInContext(stripTypeScriptTypes(snippet), context);
+  recorder.ondataavailable({ data: new Blob(['recorded audio']) });
+  return { tracks, recorder, inserted, body: () => body,
+    resolve() { resolveRequest(new Response(JSON.stringify({ text: 'hello' }), { status: 200 })); },
+    reject() { rejectRequest(new Error('offline')); },
+  };
+}
+
+test('recording stop releases every microphone track before upload completes', async () => {
+  const s = setup(); const pending = s.recorder.onstop();
+  assert.equal(s.tracks.filter((track) => track.live).length, 0);
+  s.resolve(); await pending;
+});
+
+test('a failed upload leaves no microphone capture running', async () => {
+  const s = setup(); const pending = s.recorder.onstop();
+  s.reject(); await assert.rejects(pending, /offline/);
+  assert.ok(s.tracks.every((track) => !track.live));
+});
+
+test('releasing capture preserves the recorded blob and successful transcription', async () => {
+  const s = setup(); const pending = s.recorder.onstop();
+  assert.equal(await s.body().get('file').text(), 'recorded audio');
+  s.resolve(); await pending;
+  assert.deepEqual(s.inserted, ['hello']);
+});

grok-voice/tests/dictation-http.test.cjs

@@ -0,0 +1,42 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const { stripTypeScriptTypes } = require('node:module');
+const test = require('node:test');
+const vm = require('node:vm');
+
+function setup(status) {
+  const skill = fs.readFileSync(path.join(__dirname, '../skills/add-dictation/SKILL.md'), 'utf8');
+  const snippet = skill.match(/```ts\n\/\/ client[^\n]*\n([\s\S]*?)\n```/)[1];
+  let recorder, bodyReads = 0;
+  const inserted = [];
+  class Recorder {
+    static isTypeSupported() { return true; }
+    constructor() { recorder = this; }
+    start() {}
+  }
+  const context = vm.createContext({
+    MediaRecorder: Recorder, stream: { getTracks: () => [] }, Blob, FormData,
+    async fetch() { return { ok: status >= 200 && status < 300, status,
+      async json() { bodyReads++; return status === 200 ? { text: 'spoken words' } : { error: 'request failed' }; } }; },
+    insertAtCursor(text) { inserted.push(text); },
+  });
+  vm.runInContext(stripTypeScriptTypes(snippet), context);
+  recorder.ondataavailable({ data: new Blob(['audio']) });
+  return { recorder, inserted, bodyReads: () => bodyReads };
+}
+
+for (const status of [401, 429, 503]) {
+  test(`HTTP ${status} rejects without inserting an error response into the composer`, async () => {
+    const s = setup(status);
+    await assert.rejects(s.recorder.onstop(), new RegExp(`Dictation ${status}`));
+    assert.deepEqual(s.inserted, []);
+    assert.equal(s.bodyReads(), 0);
+  });
+}
+
+test('successful dictation still inserts the returned transcript once', async () => {
+  const s = setup(200); await s.recorder.onstop();
+  assert.deepEqual(s.inserted, ['spoken words']);
+  assert.equal(s.bodyReads(), 1);
+});

grok-voice/tests/read-aloud.test.cjs

@@ -0,0 +1,127 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const { stripTypeScriptTypes } = require('node:module');
+const test = require('node:test');
+const vm = require('node:vm');
+
+function deferred() {
+  let resolve, reject;
+  const promise = new Promise((r, j) => { resolve = r; reject = j; });
+  return { promise, resolve, reject };
+}
+
+function setup({ deferPlay = false, playError } = {}) {
+  const skill = fs.readFileSync(path.join(__dirname, '../skills/add-read-aloud/SKILL.md'), 'utf8');
+  const snippet = skill.match(/```ts\n\/\/ client\n([\s\S]*?)\n```/)[1];
+  const requests = [], playing = new Set(), urls = new Set(), audios = [];
+  let nextUrl = 0;
+  class Audio {
+    constructor(url) { this.url = url; this.listeners = {}; audios.push(this); }
+    addEventListener(name, fn) { this.listeners[name] = fn; }
+    pause() { playing.delete(this); this.pendingPlay?.reject(Object.assign(new Error("Playback cancelled"), { name: "AbortError" })); }
+    async play() {
+      if (playError) throw playError;
+      playing.add(this);
+      if (deferPlay) { this.pendingPlay = deferred(); await this.pendingPlay.promise; }
+    }
+    end() { playing.delete(this); this.listeners.ended(); }
+  }
+  const context = vm.createContext({
+    Audio,
+    URL: {
+      createObjectURL() { const url = `blob:${++nextUrl}`; urls.add(url); return url; },
+      revokeObjectURL(url) { urls.delete(url); },
+    },
+    fetch() { const request = deferred(); requests.push(request); return request.promise; },
+  });
+  vm.runInContext(stripTypeScriptTypes(snippet), context);
+  const finish = (index) => requests[index].resolve({ ok: true, async blob() { return new Blob(['audio']); } });
+  return { context, requests, playing, urls, audios, finish };
+}
+
+test('latest request wins when responses arrive out of order', async () => {
+  const s = setup();
+  const old = s.context.readAloud('old');
+  const latest = s.context.readAloud('latest');
+  s.finish(1); await latest;
+  const latestAudio = s.audios[0];
+  s.finish(0); await old;
+  assert.equal(s.playing.size, 1);
+  assert.ok(s.playing.has(latestAudio));
+});
+
+test('stop while a request is pending prevents delayed playback', async () => {
+  const s = setup();
+  const pending = s.context.readAloud('pending');
+  s.context.stop();
+  s.finish(0); await pending;
+  assert.equal(s.playing.size, 0);
+  assert.equal(s.urls.size, 0);
+});
+
+test('stopping active playback releases its blob URL', async () => {
+  const s = setup();
+  const pending = s.context.readAloud('active');
+  s.finish(0); await pending;
+  s.context.stop();
+  assert.equal(s.playing.size, 0);
+  assert.equal(s.urls.size, 0);
+});
+
+test('replacing active playback releases its blob URL', async () => {
+  const s = setup();
+  const first = s.context.readAloud('first'); s.finish(0); await first;
+  const second = s.context.readAloud('second'); s.finish(1); await second;
+  assert.equal(s.playing.size, 1);
+  assert.equal(s.urls.size, 1);
+});
+
+test('natural completion releases its blob URL', async () => {
+  const s = setup();
+  const pending = s.context.readAloud('normal'); s.finish(0); await pending;
+  s.audios[0].end();
+  assert.equal(s.urls.size, 0);
+});
+
+test('stop during body download prevents delayed playback', async () => {
+  const s = setup();
+  const body = deferred();
+  const pending = s.context.readAloud('downloading');
+  s.requests[0].resolve({ ok: true, blob: () => body.promise });
+  await new Promise(setImmediate);
+  s.context.stop();
+  body.resolve(new Blob(['audio']));
+  await pending;
+  assert.equal(s.playing.size, 0);
+  assert.equal(s.urls.size, 0);
+});
+
+
+test('stop while play is pending treats pause rejection as cancellation', async () => {
+  const s = setup({ deferPlay: true });
+  const pending = s.context.readAloud('pending playback'); s.finish(0);
+  await new Promise(setImmediate);
+  s.context.stop();
+  await assert.doesNotReject(pending);
+  assert.equal(s.urls.size, 0);
+});
+
+test('replacing pending playback does not reject the cancelled request', async () => {
+  const s = setup({ deferPlay: true });
+  const first = s.context.readAloud('first'); s.finish(0);
+  await new Promise(setImmediate);
+  const second = s.context.readAloud('second'); s.finish(1);
+  await assert.doesNotReject(first);
+  await new Promise(setImmediate);
+  s.audios[1].pendingPlay.resolve();
+  await second;
+  assert.equal(s.playing.size, 1);
+});
+
+test('an active playback failure still rejects and releases its URL', async () => {
+  const s = setup({ playError: new Error('Audio decode failed') });
+  const pending = s.context.readAloud('broken'); s.finish(0);
+  await assert.rejects(pending, /Audio decode failed/);
+  assert.equal(s.urls.size, 0);
+});