diff --git a/CHANGELOG.md b/CHANGELOG.md index 58499e2..82220d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [4.41.2] + +- Add opt-in `acknowledgeSilence` streaming param — when enabled, the server emits a `Silence` message roughly once per second while no speech is transcribed, surfaced via the new `silence` event with `start_ms` and `end_ms` (session-audio positions on the same clock as word timestamps). Requires a U3Pro streaming model, and is also settable mid-stream via `updateConfiguration({ acknowledge_silence })` + ## [4.41.1] - Add `universal-3-6` to the streaming `speechModel` options (`StreamingSpeechModel`) diff --git a/package.json b/package.json index 1171871..340af01 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "assemblyai", - "version": "4.41.1", + "version": "4.41.2", "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription.", "engines": { "node": ">=18" diff --git a/src/services/streaming/service.ts b/src/services/streaming/service.ts index db7d63c..ffa3d8a 100644 --- a/src/services/streaming/service.ts +++ b/src/services/streaming/service.ts @@ -21,6 +21,7 @@ import { StreamingKeepAlive, WarningEvent, HeartbeatEvent, + SilenceEvent, } from "../.."; import type { VadDetector, VadFrame } from "../../types/streaming/dual-channel"; import { EnergyVad } from "./energy-vad"; @@ -318,6 +319,13 @@ export class StreamingTranscriber { ); } + if (this.params.acknowledgeSilence !== undefined) { + searchParams.set( + "acknowledge_silence", + this.params.acknowledgeSilence.toString(), + ); + } + if (this.params.encoding) { searchParams.set("encoding", this.params.encoding.toString()); } @@ -510,6 +518,7 @@ export class StreamingTranscriber { ): void; on(event: "warning", listener: (event: WarningEvent) => void): void; on(event: "heartbeat", listener: (event: HeartbeatEvent) => void): void; + on(event: "silence", listener: (event: SilenceEvent) => void): void; on(event: "vad", listener: (event: VadFrame) => void): void; on(event: "error", listener: (error: Error) => void): void; on(event: "close", listener: (code: number, reason: string) => void): void; @@ -736,6 +745,10 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c this.listeners.heartbeat?.(message); break; } + case "Silence": { + this.listeners.silence?.(message); + break; + } case "Termination": { this.resolveSessionTermination(); break; diff --git a/src/types/streaming/index.ts b/src/types/streaming/index.ts index 479f879..2757179 100644 --- a/src/types/streaming/index.ts +++ b/src/types/streaming/index.ts @@ -109,6 +109,12 @@ export type StreamingTranscriberParams = { vadThreshold?: number; formatTurns?: boolean; sessionHeartbeat?: boolean; + /** + * Opt in to `Silence` messages. While no speech is being transcribed, the + * server emits a `Silence` message roughly once per second, surfaced via the + * `silence` event. Requires a U3Pro streaming model. + */ + acknowledgeSilence?: boolean; filterProfanity?: boolean; keyterms?: string[]; keytermsPrompt?: string[]; @@ -186,6 +192,7 @@ export type StreamingEvents = | "speakerRevision" | "warning" | "heartbeat" + | "silence" | "vad" | "error"; @@ -198,6 +205,7 @@ export type StreamingListeners = { speakerRevision?: (event: SpeakerRevisionEvent) => void; warning?: (event: WarningEvent) => void; heartbeat?: (event: HeartbeatEvent) => void; + silence?: (event: SilenceEvent) => void; vad?: (event: VadFrame) => void; error?: (error: Error) => void; }; @@ -376,6 +384,7 @@ export type StreamingUpdateConfiguration = { vad_threshold?: number; format_turns?: boolean; session_heartbeat?: boolean; + acknowledge_silence?: boolean; keyterms_prompt?: string[]; prompt?: string; agent_context?: string; @@ -418,6 +427,22 @@ export type HeartbeatEvent = { max_speech_probability: number; }; +export type SilenceEvent = { + type: "Silence"; + /** + * Start of the silent stretch, in milliseconds of session audio on the same + * clock as word timestamps. + */ + start_ms: number; + /** + * End of the silent stretch, in milliseconds of session audio on the same + * clock as word timestamps. A range may overlap audio already reported as + * speech, because the gateway replays a short window of audio after a worker + * reconnect; track a running maximum and trim ranges against it. + */ + end_ms: number; +}; + export type LLMGatewayResponseEvent = { type: "LLMGatewayResponse"; turn_order: number; @@ -458,7 +483,8 @@ export type StreamingEventMessage = | SpeakerRevisionEvent | ErrorEvent | WarningEvent - | HeartbeatEvent; + | HeartbeatEvent + | SilenceEvent; export type StreamingOperationMessage = | StreamingUpdateConfiguration diff --git a/tests/unit/streaming.test.ts b/tests/unit/streaming.test.ts index 608a578..b642fdf 100644 --- a/tests/unit/streaming.test.ts +++ b/tests/unit/streaming.test.ts @@ -892,4 +892,98 @@ describe("streaming", () => { }), ); }); + + it("should include acknowledge_silence=true in connection URL", async () => { + await cleanup(); + WS.clean(); + + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&acknowledge_silence=true`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + acknowledgeSilence: true, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + + expect(connectUrlParams(rt).get("acknowledge_silence")).toBe("true"); + }); + + it("should include acknowledge_silence=false in connection URL when explicitly disabled", async () => { + await cleanup(); + WS.clean(); + + // Serialization uses `!== undefined`, so an explicit `false` is still emitted. + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&acknowledge_silence=false`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + acknowledgeSilence: false, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + + expect(connectUrlParams(rt).get("acknowledge_silence")).toBe("false"); + }); + + it("should omit acknowledge_silence from connection URL when unset", async () => { + await cleanup(); + WS.clean(); + + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + + expect(connectUrlParams(rt).has("acknowledge_silence")).toBe(false); + }); + + it("should parse and dispatch Silence event", async () => { + const handler = jest.fn(); + const silencePromise = new Promise<{ + type: string; + start_ms: number; + end_ms: number; + }>((resolve) => { + rt.on("silence", (event) => { + handler(event); + resolve(event); + }); + }); + + const silence = { + type: "Silence", + start_ms: 12000, + end_ms: 13000, + }; + + server.send(JSON.stringify(silence)); + + const event = await silencePromise; + expect(handler).toHaveBeenCalledTimes(1); + // snake_case wire payload is passed through verbatim. + expect(event).toEqual(silence); + }); + + it("should include acknowledge_silence in updateConfiguration message", async () => { + rt.updateConfiguration({ acknowledge_silence: true }); + await expect(server).toReceiveMessage( + JSON.stringify({ + type: "UpdateConfiguration", + acknowledge_silence: true, + }), + ); + }); });