From b2e194080f678748b91c5be8807efeea8180d1d6 Mon Sep 17 00:00:00 2001 From: David Feng Date: Sat, 19 Sep 2026 18:12:13 +0900 Subject: [PATCH] feat(confucius4_r2t2): add bounded VAD endpointing for long streaming ASR Port VAD endpointing and boundary hardening onto the upstream confucius4_r2t2 family while preserving graph reuse. Process aligned VAD frames, bound segment audio including gap context, preserve continuous VAD state, and expose finalized segment events through CLI and server. Includes model options, UI configuration, documentation, and regression tests for packet-size independence, strict span limits, and event/result consistency. --- app/cli/main.cpp | 10 +- app/server/example.json | 13 + app/server/runtime.cpp | 62 ++- docs/community_models/r2t2.md | 76 ++- .../confucius4_r2t2/session.h | 64 +++ model_specs/confucius4_r2t2.json | 56 +++ .../confucius4_r2t2/session.cpp | 439 ++++++++++++++++-- tests/confucius4_r2t2/README.md | 23 + tests/confucius4_r2t2/long_stream_test.py | 218 +++++++++ .../test_confucius4_r2t2_transcription.cpp | 108 ++++- webui/configs/model_params.json | 9 +- 11 files changed, 1035 insertions(+), 43 deletions(-) create mode 100644 tests/confucius4_r2t2/long_stream_test.py diff --git a/app/cli/main.cpp b/app/cli/main.cpp index d794ecafe..2e3b61cb5 100644 --- a/app/cli/main.cpp +++ b/app/cli/main.cpp @@ -576,7 +576,15 @@ void run_streaming( std::cout << "speech_segment"; break; } - std::cout << " sample=" << activity.sample << " probability=" << activity.probability << "\n"; + std::cout << " sample=" << activity.sample << " probability=" << activity.probability; + if (activity.segment.has_value()) { + std::cout << " start_sample=" << activity.segment->span.start_sample + << " end_sample=" << activity.segment->span.end_sample; + if (!activity.segment->text.empty()) { + std::cout << " text=" << activity.segment->text; + } + } + std::cout << "\n"; } }; diff --git a/app/server/example.json b/app/server/example.json index 7b2e73350..b12da2c21 100644 --- a/app/server/example.json +++ b/app/server/example.json @@ -54,6 +54,19 @@ "confucius4_r2t2.chunk_size_ms": "320", "confucius4_r2t2.max_tokens": "32" } + }, + { + "id": "r2t2-asr-live", + "family": "confucius4_r2t2", + "path": "../../models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf", + "task": "asr", + "mode": "streaming", + "session_options": { + "confucius4_r2t2.chunk_size_ms": "320", + "confucius4_r2t2.max_tokens": "32", + "confucius4_r2t2.endpointing": "true", + "confucius4_r2t2.max_segment_seconds": "20" + } } ] } diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index b50656f5f..42d694e4b 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -876,6 +876,37 @@ std::string stream_event_json(const engine::runtime::StreamEvent & event) { << ",\"language\":" << json_quote(event.partial_text->language) << "}"; } + if (!event.voice_activity.empty()) { + field("voice_activity"); + out << "["; + for (size_t i = 0; i < event.voice_activity.size(); ++i) { + const auto & activity = event.voice_activity[i]; + if (i != 0) { + out << ","; + } + const char * kind = "speech_segment"; + using Kind = engine::runtime::VoiceActivityEvent::Kind; + if (activity.kind == Kind::SpeechStart) { + kind = "speech_start"; + } else if (activity.kind == Kind::SpeechEnd) { + kind = "speech_end"; + } + out << "{\"kind\":" << json_quote(kind) + << ",\"sample\":" << activity.sample + << ",\"probability\":" << activity.probability; + if (activity.segment.has_value()) { + out << ",\"segment\":{\"start_sample\":" << activity.segment->span.start_sample + << ",\"end_sample\":" << activity.segment->span.end_sample + << ",\"confidence\":" << activity.segment->confidence; + if (!activity.segment->text.empty()) { + out << ",\"text\":" << json_quote(activity.segment->text); + } + out << "}"; + } + out << "}"; + } + out << "]"; + } if (event.audio_output.has_value()) { const auto wav = encode_pcm16_wav(*event.audio_output); field("audio"); @@ -3017,14 +3048,31 @@ HttpResponse ServerState::handle_transcription_live(const HttpRequest & request) task_request, audio, [&](const engine::runtime::StreamEvent & event) { - if (!event.partial_text.has_value() || event.partial_text->text.empty()) { - return; + // Segment-final deltas must precede the boundary: the + // client first appends the rollback tail, then commits the + // segment and starts a new line (reset semantics, mirroring + // the reference ws_server integrator). + if (event.partial_text.has_value() && !event.partial_text->text.empty()) { + write_sse( + writer, + "{\"type\":\"transcript.text.delta\",\"delta\":" + + json_quote(event.partial_text->text) + + "}"); + } + for (const auto & activity : event.voice_activity) { + using Kind = engine::runtime::VoiceActivityEvent::Kind; + if (activity.kind != Kind::SpeechEnd || !activity.segment.has_value()) { + continue; + } + const auto & segment = *activity.segment; + std::ostringstream out; + out << "{\"type\":\"transcript.segment.end\",\"text\":" + << json_quote(segment.text) + << ",\"start_sample\":" << segment.span.start_sample + << ",\"end_sample\":" << segment.span.end_sample + << ",\"reset\":true}"; + write_sse(writer, out.str()); } - write_sse( - writer, - "{\"type\":\"transcript.text.delta\",\"delta\":" + - json_quote(event.partial_text->text) + - "}"); }, busy_timeout_ms); if (!timed_result.result.text_output.has_value()) { diff --git a/docs/community_models/r2t2.md b/docs/community_models/r2t2.md index bc1939f31..dce66d2c8 100644 --- a/docs/community_models/r2t2.md +++ b/docs/community_models/r2t2.md @@ -116,6 +116,14 @@ quality. | `confucius4_r2t2.unfixed_token_num` | integer | `5` | Tokens rolled back from the accumulated text before it is used as the prefix prompt. | | `confucius4_r2t2.rollback_punctuation` | `true`, `false` | `false` | Keep trailing text uncommitted when it already ends with punctuation instead of rolling back tokens. | | `confucius4_r2t2.max_tokens` | integer | `32` | Greedy decode budget per chunk and for the final flush. | +| `confucius4_r2t2.endpointing` | `true`, `false` | `false` | VAD-driven endpointing: split the stream into speech segments so sessions run indefinitely (see below). Streaming only. | +| `confucius4_r2t2.vad_model_path` | path | `assets/framework/models/silero_vad` | Silero VAD weights used by endpointing. | +| `confucius4_r2t2.vad_threshold` | 0-1 | `0.4` | Speech probability threshold for VAD decisions. Lower values hold speech open through quiet tails at the cost of stickier segmentation. | +| `confucius4_r2t2.vad_min_speech_ms` | integer | `100` | Shorter bursts (coughs, clicks) are absorbed into the surrounding segment instead of closing it. | +| `confucius4_r2t2.vad_min_silence_ms` | integer | `200` | Silence that closes a segment; drives finalize latency after the speaker stops. | +| `confucius4_r2t2.vad_speech_pad_ms` | integer | `50` | Context kept ahead of a detected onset so first syllables are not clipped. | +| `confucius4_r2t2.vad_gap_keep_ms` | integer | `2000` | Rolling window of non-speech audio prepended to the next segment, so words on the quiet side of a pause survive. | +| `confucius4_r2t2.max_segment_seconds` | 0-110 | `20` | Force a boundary after this much segment audio even without a VAD pause. | | `confucius4_r2t2.audio_encoder_weight_type` | `native`, `f32`, `f16` | `native` | Audio tower weight storage. | | `confucius4_r2t2.thinker_weight_type` (alias `confucius4_r2t2.weight_type`) | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Thinker weight storage. | | `confucius4_r2t2.audio_encoder_graph_arena_mb` | MB | `128` | Audio tower graph arena. | @@ -164,11 +172,64 @@ prefix emits nothing. The uncommitted tail is delivered in the final result The reference also ships a rolling-window variant for unbounded streams ("no reset": keep 16 s of audio, discard the oldest 8 s and the matching text). This -port implements the standard variant used by the upstream WebSocket server, -which bounds audio per utterance with VAD. Because the audio tower uses 1500 -positions, a single unsegmented stream is limited to roughly 110 s of -accumulated audio; segment longer streams (as the reference server does) or add -the rolling-window variant. +port implements the standard variant used by the upstream WebSocket server. +Because the audio tower uses 1500 positions, a single unsegmented stream is +limited to roughly 110 s of accumulated audio; enable endpointing (below) for +longer sessions. + +### Endpointing: long-form dictation (VAD segmentation) + +Dictation is unbounded: a session may run for minutes, past the audio tower's +1500-frame position table (115.40 s), and per-chunk decode cost grows with the +accumulated audio. With `confucius4_r2t2.endpointing=true` the session embeds a Silero +VAD and splits the stream into speech segments: + +* On each accepted speech end (silence of `vad_min_silence_ms` after speech of + at least `vad_min_speech_ms`) the session runs the authoritative final flush, + publishes the segment text, and re-opens a fresh LSP segment. Segment audio is + bounded by `max_segment_seconds`, far inside the position table. +* Between segments the session keeps a rolling `vad_gap_keep_ms` window of + non-speech audio and prepends it to the next segment, so words on the quiet + side of a pause are not clipped; all other silence is dropped, so idle time + costs nothing. +* A speaker who never pauses is force-split at `max_segment_seconds`. The + limit includes retained gap audio and is enforced before decoding, even + when one input packet contains several segments. The retained gap is capped + at one sample less than this limit; `vad_gap_keep_ms=0` disables it. +* Input packets are buffered into 32 ms VAD frames. Segment boundaries and + decoding are independent of transport packet sizes; finalization consumes + any shorter tail. Boundary spans describe the audio actually decoded, + including retained context and detection latency, in input sample-rate units. + Silero state remains continuous across recognizer segment resets. +* The language/context request options survive segment resets; segments decode + independently, exactly like the reference WebSocket server's per-utterance + resets. + +Boundaries need no framework changes: they travel in the existing stream-event +voice-activity list as `SpeechEnd` events whose segment carries the final text +and span. On the server's live route each boundary is emitted as + +```json +{"type": "transcript.segment.end", "text": "...", "start_sample": 7904, + "end_sample": 47392, "reset": true} +``` + +Metadata-only rollback prefixes (such as `language English` before +``) are suppressed in deltas. This intentionally corrects the +reference integrator's metadata leak; historical committed-delta and +`fixed_text` comparisons below describe the earlier reference-compatible behavior. + +Committed deltas keep flowing append-only inside a segment; on `reset` the +client commits the segment (the authoritative text replaces the delta-assembled +buffer) and starts a new one. `transcript.text.done` carries all segments joined +with spaces, and the result's `segments` array lists every segment span+text. + +Historical measurements before graph reuse and VAD-frame scheduling +(M3/Metal, 320 ms chunks, 14 s utterances separated by 1 s gaps): +without endpointing a 134.6 s stream fails safely at 115.40 s after 360 chunks +with per-chunk encoder cost growing 22 -> 906 ms; with endpointing the same +stream completes with 36 segments, per-chunk cost flat at 22 -> 28 ms (max +51 ms) and a maximum of 512 encoder frames per decode. ### Streaming graph reuse @@ -247,6 +308,11 @@ ffmpeg -f avfoundation -i ":0" -ar 16000 -ac 1 -f s16le - \ 'http://127.0.0.1:8488/v1/audio/transcriptions/live?model=r2t2-asr-stream&sample_rate=16000&channels=1&sample_format=s16le' ``` +For unbounded dictation, point the live route at a model configured with +`confucius4_r2t2.endpointing=true` (the example configuration's `r2t2-asr-live`). The +stream then emits `transcript.segment.end` events with `reset:true` at every +pause; clients commit each segment and start a new one. + ## GGUF checkpoints GGUF is supported for Q8_0 and higher precision. `f16`, `q8_0`, and the native diff --git a/include/engine/community_models/confucius4_r2t2/session.h b/include/engine/community_models/confucius4_r2t2/session.h index 1e9d319da..5e4b3a1ef 100644 --- a/include/engine/community_models/confucius4_r2t2/session.h +++ b/include/engine/community_models/confucius4_r2t2/session.h @@ -1,5 +1,6 @@ #pragma once +#include "engine/framework/runtime/session.h" #include "engine/framework/runtime/session_base.h" #include "engine/framework/runtime/model.h" #include "engine/community_models/confucius4_r2t2/assets.h" @@ -11,10 +12,16 @@ #include #include +#include #include #include #include +namespace engine::models::silero_vad { +class SileroRuntime; +struct SileroVADConfig; +} + namespace engine::community_models::confucius4_r2t2 { /// Spec-backed loader factory (schema-v1 contract): the framework derives @@ -32,6 +39,27 @@ struct R2T2ASRStreamConfig { int64_t max_new_tokens = 32; }; +/// VAD-driven endpointing for long streaming sessions (dictation / input +/// method). Defaults mirror the reference ws_server.py FireRed VAD intent +/// (0.4 speech threshold, 200 ms minimum silence, 50 ms onset pad, 20 s +/// maximum speech frame) expressed in Silero VAD terms. When enabled, a +/// speech end closes the current segment: the session runs the authoritative +/// final flush, publishes a VoiceActivityEvent::SpeechEnd carrying the +/// segment text, and re-opens a fresh LSP segment so audio towers never see +/// more than one segment worth of audio (far below the 1500-frame position +/// table). A bounded non-speech context window is retained between segments; +/// it also counts toward the segment cap. +struct R2T2ASREndpointingConfig { + bool enabled = false; + std::filesystem::path vad_model_path = "assets/framework/models/silero_vad"; + float threshold = 0.4f; + int min_speech_ms = 100; + int min_silence_ms = 200; + int speech_pad_ms = 50; + int gap_keep_ms = 2000; + double max_segment_seconds = 20.0; +}; + /// Confucius4-R2T2 streaming ASR session. /// /// This family owns its full Qwen3-ASR-derived graph (audio tower, thinker, @@ -84,6 +112,42 @@ class R2T2ASRSession final std::string decode_rollback_prefix(const std::vector & ids, int64_t rollback) const; void publish_stream_delta(const std::string & fixed_text, runtime::StreamEvent & event); + // Endpointed streaming: the family embeds a Silero VAD runtime and splits + // the incoming stream into speech segments. See R2T2ASREndpointingConfig. + void ensure_vad_runtime(); + /// Steps the VAD over the chunk; returns true when the current segment + /// must end (accepted speech end). + void process_endpoint_frame(const runtime::AudioChunk & chunk, runtime::StreamEvent & event); + bool feed_vad(const runtime::AudioChunk & chunk); + /// Runs the authoritative final flush for the open segment, publishes the + /// segment boundary event, and re-opens a fresh LSP segment. + void flush_segment(runtime::StreamEvent & event, bool from_vad); + void begin_new_segment(); + void append_stream_text(const std::string & segment_text); + std::string joined_stream_text(const std::string & current_segment_text) const; + int64_t to_stream_samples(int64_t vad_samples) const; + const models::silero_vad::SileroVADConfig & vad_config() const; + + R2T2ASREndpointingConfig endpointing_; + std::unique_ptr vad_runtime_; + std::unique_ptr vad_config_; + // Session-global VAD bookkeeping: survives segment resets so spans stay + // monotonic across the whole stream. + std::vector endpoint_input_; + std::vector vad_remainder_; + int64_t vad_consumed_samples_ = 0; + // Gap-context window (interleaved, stream format) retained before onset. + std::vector vad_seed_; + bool in_speech_ = false; + bool segment_has_audio_ = false; + runtime::VoiceActivityEvent pending_speech_end_; + int64_t segment_start_stream_sample_ = 0; + int64_t segment_stream_frames_ = 0; + int64_t max_segment_stream_frames_ = 0; + int64_t stream_frames_consumed_ = 0; + std::vector> completed_segments_; + int64_t segment_index_ = 0; + runtime::TaskSpec task_; std::shared_ptr assets_; R2T2ASRStreamConfig stream_config_; diff --git a/model_specs/confucius4_r2t2.json b/model_specs/confucius4_r2t2.json index fdf8b33af..6da6b48f2 100644 --- a/model_specs/confucius4_r2t2.json +++ b/model_specs/confucius4_r2t2.json @@ -288,6 +288,62 @@ "required": false, "default": 64, "description": "Thinker weight context in MB." + }, + { + "name": "endpointing", + "type": "bool", + "required": false, + "default": false, + "description": "VAD-driven endpointing for long streaming sessions: close and re-open the LSP segment on speech ends so sessions run past the 115.40 s audio-tower limit with bounded cost. Streaming only." + }, + { + "name": "vad_model_path", + "type": "string", + "required": false, + "default": "assets/framework/models/silero_vad", + "description": "Silero VAD model directory (or safetensors file) used by endpointing." + }, + { + "name": "vad_threshold", + "type": "float", + "required": false, + "default": 0.4, + "description": "Speech probability threshold for endpointing VAD decisions (0-1)." + }, + { + "name": "vad_min_speech_ms", + "type": "int", + "required": false, + "default": 100, + "description": "Minimum speech-burst duration; shorter bursts are absorbed into the surrounding segment instead of triggering a boundary." + }, + { + "name": "vad_min_silence_ms", + "type": "int", + "required": false, + "default": 200, + "description": "Silence duration that closes a segment; drives finalize latency after the speaker stops (~200 ms plus one decode)." + }, + { + "name": "vad_speech_pad_ms", + "type": "int", + "required": false, + "default": 50, + "description": "Context prepended to a detected speech onset so first syllables are not clipped." + }, + { + "name": "vad_gap_keep_ms", + "type": "int", + "required": false, + "default": 2000, + "description": "Rolling window of non-speech audio prepended to the next segment, capped below the segment limit. Zero disables retention." + }, + { + "name": "max_segment_seconds", + "type": "float", + "required": false, + "default": 20.0, + "description": "Force a segment boundary after this much segment audio even without a VAD pause; must stay in (0, 110] because the audio tower position table holds 1500 frames (115.40 s)." } ], "load": [] diff --git a/src/community_models/confucius4_r2t2/session.cpp b/src/community_models/confucius4_r2t2/session.cpp index 68853c0a0..2a21a4963 100644 --- a/src/community_models/confucius4_r2t2/session.cpp +++ b/src/community_models/confucius4_r2t2/session.cpp @@ -1,10 +1,13 @@ #include "engine/community_models/confucius4_r2t2/session.h" #include "engine/framework/audio/chunking.h" +#include "engine/framework/audio/conversion.h" #include "engine/framework/debug/profiler.h" #include "engine/framework/runtime/options.h" #include "engine/framework/runtime/spec_backed_model.h" #include "engine/community_models/confucius4_r2t2/text_postprocess.h" +#include "engine/models/silero_vad/assets.h" +#include "engine/models/silero_vad/runtime.h" #include #include @@ -125,7 +128,7 @@ R2T2ASRSession::R2T2ASRSession( if (const auto value = runtime::parse_int_option(options.options, {"confucius4_r2t2.max_tokens"})) { stream_config_.max_new_tokens = *value; } - if (stream_config_.chunk_seconds <= 0.0) { + if (!std::isfinite(stream_config_.chunk_seconds) || stream_config_.chunk_seconds <= 0.0) { throw std::runtime_error("confucius4_r2t2.chunk_size_ms must be positive"); } if (stream_config_.unfixed_chunk_num < 0 || stream_config_.unfixed_token_num < 0) { @@ -134,6 +137,46 @@ R2T2ASRSession::R2T2ASRSession( if (stream_config_.max_new_tokens <= 0) { throw std::runtime_error("confucius4_r2t2.max_tokens must be positive"); } + if (const auto value = runtime::find_option(options.options, {"confucius4_r2t2.endpointing"})) { + endpointing_.enabled = runtime::parse_bool_option(*value, "confucius4_r2t2.endpointing"); + } + if (const auto value = runtime::find_option(options.options, {"confucius4_r2t2.vad_model_path"})) { + if (!value->empty()) { + endpointing_.vad_model_path = *value; + } + } + if (const auto value = runtime::parse_float_option(options.options, {"confucius4_r2t2.vad_threshold"})) { + endpointing_.threshold = *value; + } + if (const auto value = runtime::parse_int_option(options.options, {"confucius4_r2t2.vad_min_speech_ms"})) { + endpointing_.min_speech_ms = *value; + } + if (const auto value = runtime::parse_int_option(options.options, {"confucius4_r2t2.vad_min_silence_ms"})) { + endpointing_.min_silence_ms = *value; + } + if (const auto value = runtime::parse_int_option(options.options, {"confucius4_r2t2.vad_speech_pad_ms"})) { + endpointing_.speech_pad_ms = *value; + } + if (const auto value = runtime::parse_int_option(options.options, {"confucius4_r2t2.vad_gap_keep_ms"})) { + endpointing_.gap_keep_ms = *value; + } + if (const auto value = runtime::parse_float_option(options.options, {"confucius4_r2t2.max_segment_seconds"})) { + endpointing_.max_segment_seconds = static_cast(*value); + } + if (!std::isfinite(endpointing_.threshold) || endpointing_.threshold <= 0.0f || endpointing_.threshold >= 1.0f) { + throw std::runtime_error("confucius4_r2t2.vad_threshold must be in (0, 1)"); + } + if (endpointing_.min_speech_ms < 0 || endpointing_.min_silence_ms <= 0 || + endpointing_.speech_pad_ms < 0 || endpointing_.gap_keep_ms < 0) { + throw std::runtime_error( + "confucius4_r2t2.vad_min_speech_ms, confucius4_r2t2.vad_speech_pad_ms and confucius4_r2t2.vad_gap_keep_ms " + "must be non-negative and confucius4_r2t2.vad_min_silence_ms must be positive"); + } + if (!std::isfinite(endpointing_.max_segment_seconds) || endpointing_.max_segment_seconds <= 0.0 || endpointing_.max_segment_seconds > 110.0) { + throw std::runtime_error( + "confucius4_r2t2.max_segment_seconds must be in (0, 110]: the audio tower position table " + "holds 1500 frames (115.40 s) and every segment must stay well inside it"); + } for (const auto & [key, value] : options.options) { (void) value; if (key.rfind("confucius4_r2t2.", 0) == 0 && @@ -148,7 +191,15 @@ R2T2ASRSession::R2T2ASRSession( key != "confucius4_r2t2.unfixed_chunk_num" && key != "confucius4_r2t2.unfixed_token_num" && key != "confucius4_r2t2.rollback_punctuation" && - key != "confucius4_r2t2.max_tokens") { + key != "confucius4_r2t2.max_tokens" && + key != "confucius4_r2t2.endpointing" && + key != "confucius4_r2t2.vad_model_path" && + key != "confucius4_r2t2.vad_threshold" && + key != "confucius4_r2t2.vad_min_speech_ms" && + key != "confucius4_r2t2.vad_min_silence_ms" && + key != "confucius4_r2t2.vad_speech_pad_ms" && + key != "confucius4_r2t2.vad_gap_keep_ms" && + key != "confucius4_r2t2.max_segment_seconds") { throw std::runtime_error("unknown R2T2 ASR session option: " + key); } } @@ -310,7 +361,7 @@ std::string R2T2ASRSession::build_stream_prefix(bool final_flush) const { if (final_flush) { // finish_streaming_transcribe uses a fixed rollback without the // replacement-character loop and never rolls back past the first token. - const int64_t end_index = std::max(1, static_cast(ids.size()) - stream_config_.unfixed_token_num); + const int64_t end_index = std::min(ids.size(), std::max(1, static_cast(ids.size()) - stream_config_.unfixed_token_num)); return truncate_at_pipe(sanitize_utf8_lossy(tokenizer_.decode(std::vector(ids.begin(), ids.begin() + static_cast(end_index))))); } int64_t k = stream_config_.unfixed_token_num; @@ -399,9 +450,16 @@ R2T2ASRSession::StreamOutcome R2T2ASRSession::decode_stream_chunk(bool final_flu } void R2T2ASRSession::publish_stream_delta(const std::string & fixed_text, runtime::StreamEvent & event) { - // fixed_text contains transcript text only; metadata must never advance - // this code-point offset. Stable transcript prefixes may still shrink - // between chunks, so only publish newly committed code points. + // Mirrors the reference WebSocket integrator, which slices the committed + // text by the previously published length (in code points): + // + // if len(fixed) > len(last_fixed): emit fixed[len(last_fixed):] + // + // The stable prefix can regress between chunks, and the reference does + // not rewrite what it already sent. Metadata-only prefixes are suppressed + // before reaching this method. The authoritative + // transcript is delivered in the final result, so consumers that need exact + // text use that. const size_t length = utf8_codepoint_count(fixed_text); if (length <= published_codepoints_) { return; @@ -431,6 +489,11 @@ void R2T2ASRSession::start_stream(const runtime::TaskRequest & request) { } reset(); streaming_request_ = request; + if (endpointing_.enabled) { + // Load eagerly so a misconfigured VAD path fails at start_stream, not + // at the first chunk. + ensure_vad_runtime(); + } if (streaming_request_.audio_input.has_value()) { streaming_request_.audio_input->samples.clear(); } @@ -473,6 +536,22 @@ void R2T2ASRSession::reset() { stream_channels_ = 1; stream_started_ = false; stream_wall_start_ = {}; + if (vad_runtime_ != nullptr) { + vad_runtime_->reset(1); + } + endpoint_input_.clear(); + vad_remainder_.clear(); + vad_consumed_samples_ = 0; + vad_seed_.clear(); + in_speech_ = false; + segment_has_audio_ = false; + pending_speech_end_ = {}; + segment_start_stream_sample_ = 0; + segment_stream_frames_ = 0; + max_segment_stream_frames_ = 0; + stream_frames_consumed_ = 0; + completed_segments_.clear(); + segment_index_ = 0; } runtime::StreamEvent R2T2ASRSession::process_audio_chunk(const runtime::AudioChunk & chunk) { @@ -493,16 +572,96 @@ runtime::StreamEvent R2T2ASRSession::process_audio_chunk(const runtime::AudioChu chunk_size_samples_ = std::max( 1, static_cast(std::llround(stream_config_.chunk_seconds * static_cast(chunk.sample_rate)))); + max_segment_stream_frames_ = std::max(1, static_cast( + endpointing_.max_segment_seconds * static_cast(chunk.sample_rate))); } else if (chunk.sample_rate != stream_sample_rate_ || chunk.channels != stream_channels_) { // Chunk boundaries are counted in frames of the stream's first chunk; // a mid-stream format change would silently corrupt the slicing. throw std::runtime_error( "R2T2 ASR streaming audio format changed mid-stream (sample rate or channel count); start a new stream instead"); } - buffer_.insert(buffer_.end(), chunk.samples.begin(), chunk.samples.end()); + if (!endpointing_.enabled) { + buffer_.insert(buffer_.end(), chunk.samples.begin(), chunk.samples.end()); + + runtime::StreamEvent event; + event.is_final = false; + const size_t channel_stride = static_cast(chunk.channels); + while (buffer_.size() >= static_cast(chunk_size_samples_) * channel_stride) { + const size_t take_values = static_cast(chunk_size_samples_) * channel_stride; + audio_accum_.insert(audio_accum_.end(), buffer_.begin(), buffer_.begin() + static_cast(take_values)); + buffer_.erase(buffer_.begin(), buffer_.begin() + static_cast(take_values)); + const auto outcome = decode_stream_chunk(/*final_flush=*/false); + if (!outcome.fixed_text.empty()) { + publish_stream_delta(outcome.fixed_text, event); + } + append_stream_text(outcome.text); + if (stream_event_sink_ != nullptr && event.partial_text.has_value()) { + stream_event_sink_(event); + event.partial_text.reset(); + } + } + if (stream_event_sink_ != nullptr && event.partial_text.has_value()) { + stream_event_sink_(event); + event.partial_text.reset(); + } + return event; + } + // Buffer transport packets into globally aligned 32 ms VAD frames. This + // makes endpoint decisions independent of the caller's packet sizes. runtime::StreamEvent event; - event.is_final = false; + size_t offset = 0; + while (offset < chunk.samples.size()) { + const int64_t next_frame = to_stream_samples(vad_consumed_samples_ + 512); + const size_t frame_values = static_cast(std::max( + 1, next_frame - stream_frames_consumed_)) * chunk.channels; + const size_t take = std::min(frame_values - endpoint_input_.size(), chunk.samples.size() - offset); + endpoint_input_.insert(endpoint_input_.end(), chunk.samples.begin() + offset, + chunk.samples.begin() + offset + take); + offset += take; + if (endpoint_input_.size() == frame_values) { + runtime::AudioChunk frame; + frame.sample_rate = stream_sample_rate_; + frame.channels = stream_channels_; + frame.samples.swap(endpoint_input_); + process_endpoint_frame(frame, event); + } + } + return event; +} + +void R2T2ASRSession::process_endpoint_frame(const runtime::AudioChunk & chunk, runtime::StreamEvent & event) { + const bool vad_segment_end = feed_vad(chunk); + const int64_t chunk_frames = static_cast(chunk.samples.size() / chunk.channels); + if (in_speech_ || segment_has_audio_ || vad_segment_end) { + int64_t offset = 0; + while (offset < chunk_frames) { + if (!segment_has_audio_) { + segment_has_audio_ = true; + segment_start_stream_sample_ = stream_frames_consumed_ + offset; + } + const int64_t take = std::min(chunk_frames - offset, + max_segment_stream_frames_ - segment_stream_frames_); + buffer_.insert(buffer_.end(), chunk.samples.begin() + offset * chunk.channels, + chunk.samples.begin() + (offset + take) * chunk.channels); + segment_stream_frames_ += take; + offset += take; + if (segment_stream_frames_ == max_segment_stream_frames_) { + flush_segment(event, vad_segment_end && offset == chunk_frames); + } + } + } else { + vad_seed_.insert(vad_seed_.end(), chunk.samples.begin(), chunk.samples.end()); + const int64_t seed_frames = std::min(max_segment_stream_frames_ - 1, + static_cast(endpointing_.gap_keep_ms) * stream_sample_rate_ / 1000); + const size_t keep = std::min(vad_seed_.size(), static_cast(seed_frames) * chunk.channels); + vad_seed_.erase(vad_seed_.begin(), vad_seed_.end() - static_cast(keep)); + } + stream_frames_consumed_ += chunk_frames; + if (vad_segment_end && segment_has_audio_) { + flush_segment(event, true); + } + const size_t channel_stride = static_cast(chunk.channels); while (buffer_.size() >= static_cast(chunk_size_samples_) * channel_stride) { const size_t take_values = static_cast(chunk_size_samples_) * channel_stride; @@ -512,14 +671,7 @@ runtime::StreamEvent R2T2ASRSession::process_audio_chunk(const runtime::AudioChu if (!outcome.fixed_text.empty()) { publish_stream_delta(outcome.fixed_text, event); } - if (!streaming_result_.text_output.has_value()) { - streaming_result_.text_output = runtime::Transcript{outcome.text, language_}; - } else { - streaming_result_.text_output->text = outcome.text; - if (!language_.empty()) { - streaming_result_.text_output->language = language_; - } - } + append_stream_text(outcome.text); if (stream_event_sink_ != nullptr && event.partial_text.has_value()) { stream_event_sink_(event); event.partial_text.reset(); @@ -529,7 +681,7 @@ runtime::StreamEvent R2T2ASRSession::process_audio_chunk(const runtime::AudioChu stream_event_sink_(event); event.partial_text.reset(); } - return event; + } runtime::TaskResult R2T2ASRSession::finish_stream() { @@ -545,21 +697,53 @@ runtime::TaskResult R2T2ASRSession::finalize() { if (!stream_started_) { throw std::runtime_error("R2T2 ASR finalize() requires start_stream"); } - if (!buffer_.empty()) { - audio_accum_.insert(audio_accum_.end(), buffer_.begin(), buffer_.end()); - buffer_.clear(); - const auto outcome = decode_stream_chunk(/*final_flush=*/true); + if (endpointing_.enabled) { + if (!endpoint_input_.empty()) { + runtime::AudioChunk tail; + tail.sample_rate = stream_sample_rate_; + tail.channels = stream_channels_; + tail.samples.swap(endpoint_input_); + runtime::StreamEvent event; + process_endpoint_frame(tail, event); + } + // Anything still in the non-speech lead-in window gets one final + // decode: quiet speech the VAD ended on would otherwise be dropped. + // The window is bounded (vad_gap_keep_ms), so this costs at most one + // short decode at stream end. + if (!in_speech_ && !segment_has_audio_ && !vad_seed_.empty()) { + buffer_ = std::move(vad_seed_); + vad_seed_.clear(); + segment_stream_frames_ = static_cast(buffer_.size() / static_cast(std::max(1, stream_channels_))); + segment_start_stream_sample_ = stream_frames_consumed_ - segment_stream_frames_; + segment_has_audio_ = true; + } + // Close the open segment (if any) with the same authoritative final + // flush a VAD-driven boundary would use. + if (in_speech_ || segment_has_audio_) { + runtime::StreamEvent boundary_event; + flush_segment(boundary_event, /*from_vad=*/false); + } if (!streaming_result_.text_output.has_value()) { - streaming_result_.text_output = runtime::Transcript{outcome.text, language_}; + streaming_result_.text_output = runtime::Transcript{joined_stream_text(std::string()), language_}; } else { - streaming_result_.text_output->text = outcome.text; + streaming_result_.text_output->text = joined_stream_text(std::string()); if (!language_.empty()) { streaming_result_.text_output->language = language_; } } - } - if (!streaming_result_.text_output.has_value()) { - streaming_result_.text_output = runtime::Transcript{text_, language_}; + for (const auto & [span, segment_text] : completed_segments_) { + streaming_result_.speech_segments.push_back(runtime::SpeechSegment{span, 1.0f, segment_text}); + } + } else { + if (!buffer_.empty()) { + audio_accum_.insert(audio_accum_.end(), buffer_.begin(), buffer_.end()); + buffer_.clear(); + const auto outcome = decode_stream_chunk(/*final_flush=*/true); + append_stream_text(outcome.text); + } + if (!streaming_result_.text_output.has_value()) { + streaming_result_.text_output = runtime::Transcript{text_, language_}; + } } if (stream_event_sink_ != nullptr) { // The final transcript travels in the task result (the server emits it @@ -579,6 +763,209 @@ runtime::TaskResult R2T2ASRSession::finalize() { return streaming_result_; } +void R2T2ASRSession::ensure_vad_runtime() { + if (vad_runtime_ != nullptr) { + return; + } + namespace sv = engine::models::silero_vad; + const auto paths = sv::resolve_silero_assets(endpointing_.vad_model_path); + auto weights = sv::load_silero_weights_cached(paths.checkpoint_path); + vad_runtime_ = std::make_unique( + std::move(weights), execution_context(), engine::assets::TensorStorageType::Native); + vad_runtime_->prepare(16000); + vad_config_ = std::make_unique(); + vad_config_->threshold = endpointing_.threshold; + vad_config_->min_silence_duration_ms = endpointing_.min_silence_ms; + vad_config_->speech_pad_ms = endpointing_.speech_pad_ms; + // neg_threshold stays on the runtime default (threshold - 0.15); the + // min-speech gate is enforced on the segment span below, mirroring the + // reference VAD's min_speech_frame semantics. +} + +const engine::models::silero_vad::SileroVADConfig & R2T2ASRSession::vad_config() const { + return *vad_config_; +} + +int64_t R2T2ASRSession::to_stream_samples(int64_t vad_samples) const { + if (stream_sample_rate_ == 16000 || vad_samples == 0) { + return vad_samples; + } + return static_cast(std::llround( + static_cast(vad_samples) * static_cast(stream_sample_rate_) / 16000.0)); +} + +bool R2T2ASRSession::feed_vad(const runtime::AudioChunk & chunk) { + ensure_vad_runtime(); + auto mono = engine::audio::convert_interleaved_audio_to_mono_linear_resampled( + chunk.samples, chunk.sample_rate, chunk.channels, 16000); + // Rounded source-frame boundaries can resample to 511/513 values at + // unusual rates. A complete VAD frame always advances exactly 512 ticks. + const int64_t expected_frames = std::max(1, + to_stream_samples(vad_consumed_samples_ + 512) - stream_frames_consumed_); + if (static_cast(chunk.samples.size() / chunk.channels) == expected_frames) { + mono.resize(512, mono.empty() ? 0.0f : mono.back()); + } + vad_remainder_.insert(vad_remainder_.end(), mono.begin(), mono.end()); + + bool end_requested = false; + constexpr int64_t kVadFrameSamples = 512; + while (vad_remainder_.size() >= static_cast(kVadFrameSamples)) { + runtime::AudioChunk frame; + frame.sample_rate = 16000; + frame.channels = 1; + frame.start_sample = vad_consumed_samples_; + frame.samples.assign(vad_remainder_.begin(), vad_remainder_.begin() + kVadFrameSamples); + vad_remainder_.erase(vad_remainder_.begin(), vad_remainder_.begin() + kVadFrameSamples); + vad_consumed_samples_ += kVadFrameSamples; + + const auto vad_event = vad_runtime_->process_chunk(frame, vad_config()); + for (const auto & activity : vad_event.voice_activity) { + using Kind = runtime::VoiceActivityEvent::Kind; + if (activity.kind == Kind::SpeechStart) { + if (!segment_has_audio_) { + // Fresh segment: anchor its span at the lead-in it is + // about to receive and seed the decoder buffer with the + // bounded non-speech window so boundary words survive. + segment_has_audio_ = true; + if (!vad_seed_.empty()) { + buffer_.insert(buffer_.end(), vad_seed_.begin(), vad_seed_.end()); + segment_stream_frames_ += + static_cast(vad_seed_.size() / static_cast(std::max(1, stream_channels_))); + vad_seed_.clear(); + } + segment_start_stream_sample_ = stream_frames_consumed_ - segment_stream_frames_; + } + in_speech_ = true; + } else if (activity.kind == Kind::SpeechEnd) { + in_speech_ = false; + auto end_activity = activity; + end_activity.sample = to_stream_samples(activity.sample); + if (end_activity.segment.has_value()) { + end_activity.segment->span.start_sample = to_stream_samples(end_activity.segment->span.start_sample); + end_activity.segment->span.end_sample = to_stream_samples(end_activity.segment->span.end_sample); + } + const int64_t span_frames = end_activity.segment.has_value() + ? end_activity.segment->span.end_sample - end_activity.segment->span.start_sample + : 0; + const int64_t min_speech_frames = static_cast( + static_cast(endpointing_.min_speech_ms) * stream_sample_rate_ / 1000.0); + if (span_frames >= min_speech_frames) { + pending_speech_end_ = std::move(end_activity); + end_requested = true; + } + // Rejected bursts (coughs, clicks below min_speech_ms) stay + // absorbed in the open segment; the next onset continues it. + } + } + } + return end_requested; +} + +void R2T2ASRSession::flush_segment(runtime::StreamEvent & event, bool from_vad) { + if (!buffer_.empty()) { + audio_accum_.insert(audio_accum_.end(), buffer_.begin(), buffer_.end()); + buffer_.clear(); + } + const runtime::TimeSpan span{ + segment_start_stream_sample_, + segment_start_stream_sample_ + segment_stream_frames_, + }; + std::string segment_text; + if (!audio_accum_.empty()) { + const auto outcome = decode_stream_chunk(/*final_flush=*/true); + segment_text = outcome.text; + if (!outcome.fixed_text.empty()) { + publish_stream_delta(outcome.fixed_text, event); + } + } + if (!segment_text.empty()) { + completed_segments_.push_back({span, segment_text}); + } + // The current segment is already in completed_segments_; pass empty so the + // joined view is not appended twice. + append_stream_text(std::string()); + + runtime::VoiceActivityEvent boundary; + if (from_vad && pending_speech_end_.kind == runtime::VoiceActivityEvent::Kind::SpeechEnd) { + boundary = std::move(pending_speech_end_); + pending_speech_end_ = {}; + } else { + boundary.kind = runtime::VoiceActivityEvent::Kind::SpeechEnd; + boundary.sample = span.end_sample; + boundary.probability = 0.0f; + boundary.segment = runtime::SpeechSegment{span, 0.0f, {}}; + } + boundary.sample = span.end_sample; + boundary.segment = runtime::SpeechSegment{span, boundary.probability, segment_text}; + if (boundary.segment.has_value()) { + // Authoritative segment text travels with the boundary so clients can + // replace their delta-assembled buffer instead of appending to it. + boundary.segment->text = segment_text; + } + if (!segment_text.empty()) { + // Empty boundaries (silence-only flushes, e.g. the final lead-in + // window decode) carry no commit action; skip them rather than make + // clients filter reset events with nothing to commit. + event.voice_activity.push_back(boundary); + } + debug::trace_log_scalar("confucius4_r2t2.stream.segment_index", segment_index_); + debug::trace_log_scalar("confucius4_r2t2.stream.segment_frames", segment_stream_frames_); + debug::trace_log_scalar("confucius4_r2t2.stream.segment_text", segment_text); + if (stream_event_sink_ != nullptr) { + stream_event_sink_(event); + event.voice_activity.clear(); + event.partial_text.reset(); + } + begin_new_segment(); +} + +void R2T2ASRSession::begin_new_segment() { + // Only reset the recognizer. VAD recurrent state, its clock, the input + // remainder and speech state must survive both natural and forced cuts. + text_.clear(); + raw_decoded_.clear(); + buffer_.clear(); + audio_accum_.clear(); + chunk_id_ = 0; + published_codepoints_ = 0; + segment_has_audio_ = false; + segment_stream_frames_ = 0; + ++segment_index_; +} + +void R2T2ASRSession::append_stream_text(const std::string & segment_text) { + const std::string joined = joined_stream_text(segment_text); + if (!streaming_result_.text_output.has_value()) { + streaming_result_.text_output = runtime::Transcript{joined, language_}; + } else { + streaming_result_.text_output->text = joined; + if (!language_.empty()) { + streaming_result_.text_output->language = language_; + } + } +} + +std::string R2T2ASRSession::joined_stream_text(const std::string & current_segment_text) const { + std::ostringstream joined; + bool first = true; + auto append = [&](const std::string & text) { + if (text.empty()) { + return; + } + if (!first) { + joined << ' '; + } + first = false; + joined << text; + }; + for (const auto & [span, segment_text] : completed_segments_) { + (void) span; + append(segment_text); + } + append(current_segment_text); + return joined.str(); +} + // Loading adapter: confucius4_r2t2 uses the schema-v1 spec-backed loader, so the loader // wiring stays beside the session it constructs (no per-model loader.{h,cpp}). std::shared_ptr make_confucius4_r2t2_loader() { diff --git a/tests/confucius4_r2t2/README.md b/tests/confucius4_r2t2/README.md index 068dfb02a..c8a64be91 100644 --- a/tests/confucius4_r2t2/README.md +++ b/tests/confucius4_r2t2/README.md @@ -9,6 +9,7 @@ implementation in the Confucius4-R2T2 repository (see `docs/community_models/r2t |---|---| | `make_golden.py` | Runs the Python reference (`R2T2ASRModel` on MPS) for an audio file and writes offline text plus per-chunk streaming `fixed_text`, `raw_decoded`, and `text` to a golden JSON. | | `compare.py` | Runs `audiocpp_cli` offline and streaming with `--log-file`, parses the per-chunk trace, and diffs everything against a golden. | +| `long_stream_test.py` | Long-session behavior: without endpointing a >115.4 s stream must fail safely on the audio-tower position table (and records the per-chunk cost curve); with `--endpointing` it must complete, emit segment boundaries, and keep per-chunk cost flat. | | `test_confucius4_r2t2_transcription.cpp` | Repo-native smoke test: offline + final streaming transcripts against the golden, plus a check that Auto-language deltas form a nonempty prefix of the expected transcript for `assets/resources/sample_16k.wav`. Skips with exit code 125 when the model or audio is missing. Also exposes `--encode ` to dump token ids for tokenizer diffing. | | `golden*.json` | Recorded reference outputs. | @@ -49,6 +50,23 @@ python3 tests/confucius4_r2t2/compare.py \ --backend metal ``` +## Long-session / endpointing + +```bash +# Baseline: must fail safely past 115.40 s (position table), prints cost curve +python3 tests/confucius4_r2t2/long_stream_test.py \ + --cli build/macos-metal-release/bin/audiocpp_cli \ + --model models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf + +# Endpointed: must complete past the limit with flat per-chunk cost +python3 tests/confucius4_r2t2/long_stream_test.py \ + --cli build/macos-metal-release/bin/audiocpp_cli \ + --model models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf --endpointing +``` + +The script assembles its audio by looping `assets/resources/sample_16k.wav` +with silence gaps, so it needs no external fixtures. + The comparison checks four things: the offline transcript, the committed delta stream, every per-chunk committed `fixed_text`, and the final streaming transcript. Metadata-only rollback prefixes are filtered from the unmodified @@ -64,6 +82,11 @@ build/macos-metal-release/bin/test_confucius4_r2t2_transcription \ --encode "language EnglishSome text 22,500" ``` +The smoke test also compares 60 s and 17 ms transport packets with a 3.013 s +segment cap and a 5 s gap window. It checks identical text and spans, ordered +boundaries, input-range timestamps, and the strict cap including gap audio. +The English fixture also rejects language metadata in transcript deltas. + ## Graph reuse regression ```bash diff --git a/tests/confucius4_r2t2/long_stream_test.py b/tests/confucius4_r2t2/long_stream_test.py new file mode 100644 index 000000000..17978eaad --- /dev/null +++ b/tests/confucius4_r2t2/long_stream_test.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Long-session streaming behavior test for the confucius4_r2t2 family. + +The audio tower position table caps a single decode at 1500 frames +(13 tokens/s => 115.40 s). This script pins down what happens past that +limit and, once VAD endpointing is enabled, verifies that segmentation +keeps long dictation sessions alive with bounded per-chunk cost. + +Usage: + python3 tests/confucius4_r2t2/long_stream_test.py \ + --cli build/macos-metal-release/bin/audiocpp_cli \ + --model models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf \ + [--source-audio assets/resources/sample_16k.wav] \ + [--target-seconds 130] [--gap-seconds 1.0] [--chunk-ms 320] + [--endpointing] [--report /tmp/r2t2_long_report.json] + +Without --endpointing the run must fail safely once the stream exceeds the +position-table limit: nonzero exit, an explicit max_source_positions error, +and no crash. With --endpointing the run must complete, emit segment.end +boundaries, and keep per-chunk encoder cost bounded by the segment cap +instead of the whole stream length. +""" + +import argparse +import json +import math +import re +import subprocess +import sys +import tempfile +import wave +from pathlib import Path + +TRACE_RE = re.compile(r"^\[(?:TRACE|TIMING) [^\]]*\] (?P\S+)\s?(?P.*)$") + +# Per-chunk timing keys (grouped by confucius4_r2t2.stream.chunk_id). The greedy +# decoder does not expose compute timings (framework gap), so encoder + +# frontend cost is the instrumented proxy for per-chunk decode cost. +CHUNK_TIMING_KEYS = ( + "confucius4_r2t2.frontend.normalize_ms", + "confucius4_r2t2.frontend.log_mel_ms", + "confucius4_r2t2.audio_encoder.input_upload_ms", + "confucius4_r2t2.audio_encoder.graph.compute_ms", + "confucius4_r2t2.audio_encoder.output_read_ms", +) + + +def build_long_wav(source: Path, target_seconds: float, gap_seconds: float, out: Path) -> float: + """Concatenate source speech with silence gaps until target_seconds.""" + with wave.open(str(source), "rb") as w: + rate, channels, width, frames = w.getframerate(), w.getnchannels(), w.getsampwidth(), w.getnframes() + speech = w.readframes(frames) + gap_frames = int(rate * gap_seconds) + silence = b"\x00" * (gap_frames * channels * width) + speech_seconds = frames / rate + block_seconds = speech_seconds + gap_seconds + repeats = max(1, math.ceil(target_seconds / block_seconds)) + with wave.open(str(out), "wb") as w: + w.setnchannels(channels) + w.setsampwidth(width) + w.setframerate(rate) + for i in range(repeats): + w.writeframes(speech) + if i + 1 < repeats and gap_seconds > 0: + w.writeframes(silence) + duration = (frames * repeats + gap_frames * max(0, repeats - 1)) / rate + return duration + + +def parse_trace(path: Path): + """Group per-chunk trace scalars by chunk_id.""" + chunks = [] + pending = None + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + m = TRACE_RE.match(line) + if not m: + continue + name, value = m.group("name"), m.group("value") + if name == "confucius4_r2t2.stream.chunk_id": + if pending is not None: + chunks.append(pending) + pending = {"chunk_id": int(value), "final_flush": 0, "timings": {}, "frames": None} + elif pending is None: + continue + elif name == "confucius4_r2t2.stream.final_flush": + pending["final_flush"] = int(value) + elif name == "confucius4_r2t2.audio_encoder.frames": + try: + pending["frames"] = int(value) + except ValueError: + pass + elif name in CHUNK_TIMING_KEYS: + try: + pending["timings"][name] = pending["timings"].get(name, 0.0) + float(value) + except ValueError: + pass + if pending is not None: + chunks.append(pending) + return chunks + + +def chunk_cost_ms(chunk) -> float: + return sum(chunk["timings"].get(k, 0.0) for k in CHUNK_TIMING_KEYS) + + +def run_cli(args, audio_path: Path, trace_path: Path): + cmd = [ + args.cli, "--task", "asr", "--mode", "streaming", "--family", "confucius4_r2t2", + "--model", args.model, "--backend", args.backend, "--audio", str(audio_path), + "--session-option", f"confucius4_r2t2.chunk_size_ms={args.chunk_ms}", + "--session-option", f"confucius4_r2t2.max_tokens={args.max_tokens}", + "--log-file", str(trace_path), + ] + if args.endpointing: + cmd += [ + "--session-option", "confucius4_r2t2.endpointing=true", + "--session-option", f"confucius4_r2t2.max_segment_seconds={args.max_segment_seconds}", + ] + return subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cli", required=True) + ap.add_argument("--model", required=True) + ap.add_argument("--backend", default="metal") + ap.add_argument("--source-audio", default="assets/resources/sample_16k.wav") + ap.add_argument("--target-seconds", type=float, default=130.0, + help="must exceed the 115.40 s position-table limit") + ap.add_argument("--gap-seconds", type=float, default=1.0) + ap.add_argument("--chunk-ms", type=int, default=320) + ap.add_argument("--max-tokens", type=int, default=32) + ap.add_argument("--max-segment-seconds", type=float, default=20.0) + ap.add_argument("--endpointing", action="store_true") + ap.add_argument("--report", default=None) + args = ap.parse_args() + + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + audio_path = tmp / "long.wav" + duration = build_long_wav(Path(args.source_audio), args.target_seconds, args.gap_seconds, audio_path) + print(f"audio: {duration:.1f}s assembled from {args.source_audio} " + f"(gap {args.gap_seconds}s, limit 115.40s, chunk {args.chunk_ms}ms)") + + trace = tmp / "trace.log" + proc = run_cli(args, audio_path, trace) + chunks = parse_trace(trace) if trace.exists() else [] + per_chunk = [c for c in chunks if not c["final_flush"]] + costs = [chunk_cost_ms(c) for c in per_chunk] + frames = [c["frames"] for c in per_chunk if c["frames"] is not None] + + segments = [ + line for line in proc.stdout.splitlines() + if line.startswith("event=speech_end") + ] + final_text = next( + (line[len("text_output="):] for line in proc.stdout.splitlines() + if line.startswith("text_output=")), + None, + ) + + report = { + "mode": "endpointing" if args.endpointing else "baseline", + "audio_seconds": round(duration, 2), + "chunk_ms": args.chunk_ms, + "exit_code": proc.returncode, + "chunks_decoded": len(per_chunk), + "cost_proxy": { + "note": "frontend+encoder ms per chunk; decoder compute is not instrumented", + "first5_avg_ms": round(sum(costs[:5]) / max(1, len(costs[:5])), 1), + "last5_avg_ms": round(sum(costs[-5:]) / max(1, len(costs[-5:])), 1), + "max_ms": round(max(costs), 1) if costs else None, + "max_encoder_frames": max(frames) if frames else None, + }, + "segments": len(segments), + "final_text_chars": len(final_text or ""), + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + + failures = [] + if args.endpointing: + if proc.returncode != 0: + failures.append(f"endpointed long run failed ({proc.returncode})") + if not segments: + failures.append("endpointed long run produced no speech_end segment events") + if not final_text: + failures.append("endpointed long run produced no final transcript") + if frames and max(frames) > 1500: + failures.append(f"encoder frames {max(frames)} exceeded position table") + else: + if proc.returncode == 0: + failures.append("baseline run unexpectedly survived past the position-table limit") + else: + combined = proc.stderr + proc.stdout + if "max_source_positions" not in combined: + failures.append( + "baseline failure is not the safe max_source_positions error:\n" + + combined[-2000:]) + else: + print("baseline OK: safe max_source_positions failure past 115.40 s") + + if failures: + for f in failures: + print(f"FAIL: {f}") + sys.stderr.write(proc.stdout[-3000:]) + sys.stderr.write(proc.stderr[-3000:]) + return 1 + if args.endpointing: + print(f"endpointed OK: {len(segments)} segments, " + f"{len(per_chunk)} chunks, no position-table error") + if args.report: + Path(args.report).write_text(json.dumps(report, indent=2, ensure_ascii=False)) + print(f"report: {args.report}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp b/tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp index 378f78a2d..5a74e7359 100644 --- a/tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp +++ b/tests/confucius4_r2t2/test_confucius4_r2t2_transcription.cpp @@ -8,6 +8,7 @@ #include "engine/community_models/confucius4_r2t2/tokenizer_text.h" #include +#include #include #include #include @@ -36,6 +37,19 @@ const char * kExpectedStreamFinal = constexpr int64_t kStreamingChunkMs = 320; constexpr int64_t kStreamingMaxNewTokens = 32; +// Segments decode independently, so a boundary legitimately rewrites casing +// and sentence punctuation ("nature, others" vs "nature. Others"). Endpointed +// comparisons normalize both sides to words+digits. +std::string normalize_words(const std::string & text) { + std::string out; + for (const char c : text) { + if (std::isalnum(static_cast(c))) { + out.push_back(static_cast(std::tolower(static_cast(c)))); + } + } + return out; +} + std::filesystem::path repo_path(const std::string & relative) { return std::filesystem::path(ENGINE_REPO_ROOT) / relative; } @@ -91,7 +105,10 @@ std::string run_offline( std::string run_streaming( engine::runtime::ILoadedVoiceModel & model, const engine::runtime::AudioBuffer & audio, - const engine::runtime::SessionOptions & options) { + const engine::runtime::SessionOptions & options, + size_t * segment_count = nullptr, + int64_t input_chunk_ms = kStreamingChunkMs, + std::vector * spans = nullptr) { auto session = model.create_task_session( engine::runtime::TaskSpec{engine::runtime::VoiceTaskKind::Asr, engine::runtime::RunMode::Streaming}, options); @@ -104,17 +121,45 @@ std::string run_streaming( streaming->prepare(engine::runtime::build_preparation_request(request)); std::string committed; + std::string segment_text; + size_t segments = 0; + int64_t previous_end = 0; streaming->set_stream_event_sink([&](const engine::runtime::StreamEvent & event) { if (event.partial_text.has_value()) { + if (event.partial_text->text.find("language") != std::string::npos || + event.partial_text->text.find("") != std::string::npos) { + throw std::runtime_error("language metadata leaked into transcript delta"); + } committed += event.partial_text->text; } + for (const auto & activity : event.voice_activity) { + if (activity.kind == engine::runtime::VoiceActivityEvent::Kind::SpeechEnd && + activity.segment.has_value() && + !activity.segment->text.empty()) { + const auto span = activity.segment->span; + const int64_t frames = audio.samples.size() / audio.channels; + const double cap = options.options.count("confucius4_r2t2.max_segment_seconds") + ? std::stod(options.options.at("confucius4_r2t2.max_segment_seconds")) : 20.0; + if (span.start_sample < previous_end || span.end_sample > frames || + span.end_sample <= span.start_sample || + span.end_sample - span.start_sample > static_cast(cap * audio.sample_rate) || + activity.sample != span.end_sample) { + throw std::runtime_error("invalid endpoint segment span"); + } + if (!segment_text.empty()) segment_text += ' '; + segment_text += activity.segment->text; + previous_end = span.end_sample; + if (spans) spans->push_back(span); + ++segments; + } + } }); request.options["language"] = "Auto"; streaming->start_stream(request); const int64_t chunk_frames = std::max( 1, - static_cast(audio.sample_rate) * kStreamingChunkMs / 1000); + static_cast(audio.sample_rate) * input_chunk_ms / 1000); const int64_t frames = static_cast(audio.samples.size() / static_cast(audio.channels)); for (int64_t start = 0; start < frames; start += chunk_frames) { const int64_t take = std::min(chunk_frames, frames - start); @@ -127,13 +172,22 @@ std::string run_streaming( streaming->process_audio_chunk(chunk); } const auto result = streaming->finish_stream(); + if (segment_count != nullptr) { + *segment_count = segments; + } if (!result.text_output.has_value()) { throw std::runtime_error("streaming run produced no text"); } std::cout << "Committed stream: " << committed << "\n"; // finish_stream() returns the final tail separately; the emitted deltas // must form a nonempty transcript prefix, never consume metadata offsets. - if (committed.empty() || std::string(kExpectedStreamFinal).compare(0, committed.size(), committed) != 0) { + const bool endpointed = options.options.count("confucius4_r2t2.endpointing") && + options.options.at("confucius4_r2t2.endpointing") == "true"; + if (endpointed) { + if (segment_text != result.text_output->text || segments != result.speech_segments.size()) { + throw std::runtime_error("segment events do not reconstruct the final transcript"); + } + } else if (committed.empty() || std::string(kExpectedStreamFinal).compare(0, committed.size(), committed) != 0) { throw std::runtime_error("committed deltas are not a prefix of the expected transcript: " + committed); } return result.text_output->text; @@ -209,6 +263,54 @@ int main(int argc, char ** argv) { return kExitFail; } + // Endpointing smoke: a misconfigured VAD path must fail fast at + // start_stream with an actionable error, and a real run must emit at + // least one non-empty segment boundary whose text joins into the + // final transcript. + try { + engine::runtime::SessionOptions bad = options; + bad.options["confucius4_r2t2.endpointing"] = "true"; + bad.options["confucius4_r2t2.vad_model_path"] = "assets/definitely/missing/silero_vad"; + run_streaming(*model, audio, bad); + std::cerr << "FAIL: endpointing with a missing VAD model should have thrown\n"; + return kExitFail; + } catch (const std::exception & error) { + std::cout << "Endpointing bad-path check: rejected as expected (" << error.what() << ")\n"; + } + + engine::runtime::SessionOptions endpointed = options; + endpointed.options["confucius4_r2t2.endpointing"] = "true"; + size_t segments = 0; + const std::string endpointed_final = run_streaming(*model, audio, endpointed, &segments); + std::cout << "Endpointed final: " << endpointed_final << " (" << segments << " segments)\n"; + if (segments == 0) { + std::cerr << "FAIL: endpointed run emitted no segment boundaries\n"; + return kExitFail; + } + if (normalize_words(endpointed_final) != normalize_words(kExpectedStreamFinal)) { + std::cerr << "FAIL: endpointed transcript mismatch\n" + << " expected: " << kExpectedStreamFinal << "\n" + << " actual: " << endpointed_final << "\n"; + return kExitFail; + } + + // One transport packet contains several VAD boundaries. Its output + // must equal irregular small packets, including exact sample spans. + endpointed.options["confucius4_r2t2.max_segment_seconds"] = "3.013"; + endpointed.options["confucius4_r2t2.vad_gap_keep_ms"] = "5000"; + std::vector large_spans, small_spans; + const auto large = run_streaming(*model, audio, endpointed, nullptr, 60000, &large_spans); + const auto small = run_streaming(*model, audio, endpointed, nullptr, 17, &small_spans); + if (large != small || large_spans.size() != small_spans.size() || large_spans.size() < 2) { + throw std::runtime_error("endpoint output depends on transport packet size"); + } + for (size_t i = 0; i < large_spans.size(); ++i) { + if (large_spans[i].start_sample != small_spans[i].start_sample || + large_spans[i].end_sample != small_spans[i].end_sample) { + throw std::runtime_error("endpoint spans depend on transport packet size"); + } + } + std::cout << "PASS: Confucius4-R2T2 offline and streaming transcripts match the MPS golden.\n"; return kExitPass; } catch (const std::exception & error) { diff --git a/webui/configs/model_params.json b/webui/configs/model_params.json index 222e7de6b..abc88e125 100644 --- a/webui/configs/model_params.json +++ b/webui/configs/model_params.json @@ -116,7 +116,14 @@ {"name": "unfixed_chunk_num", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.unfixed_chunk_num", "label": "unfixed_chunk_num（前 N 块不用稳定前缀）", "label_en": "unfixed_chunk_num (leading chunks without prefix)", "default": 2, "minimum": 0, "maximum": 10, "step": 1, "precision": 0, "info": "开头若干块不使用已识别文本作为前缀提示。", "info_en": "Leading chunks that decode without a stable-prefix prompt."}, {"name": "unfixed_token_num", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.unfixed_token_num", "label": "unfixed_token_num（回滚 token 数）", "label_en": "unfixed_token_num (rollback tokens)", "default": 5, "minimum": 0, "maximum": 20, "step": 1, "precision": 0, "info": "作为前缀前从累积文本回滚的 token 数，用于降低边界抖动。", "info_en": "Tokens rolled back from the accumulated text before it is used as the prefix prompt."}, {"name": "rollback_punctuation", "type": "bool", "scope": "session", "session_option": "confucius4_r2t2.rollback_punctuation", "label": "rollback_punctuation（句末标点不回滚）", "label_en": "rollback_punctuation (keep trailing punctuation)", "default": false, "info": "输出已以标点结尾时不再回滚 token。", "info_en": "Do not roll back tokens when the output already ends with punctuation."}, - {"name": "max_new_tokens", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.max_tokens", "label": "max_new_tokens（每分块解码上限）", "label_en": "max_new_tokens (per-chunk decode budget)", "default": 32, "minimum": 1, "maximum": 256, "step": 1, "precision": 0, "info": "每个流式分块的贪婪解码上限（会话级，区别于离线 max_tokens）。", "info_en": "Greedy decode budget per streaming chunk (session-scoped; distinct from offline max_tokens)."} + {"name": "max_new_tokens", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.max_tokens", "label": "max_new_tokens（每分块解码上限）", "label_en": "max_new_tokens (per-chunk decode budget)", "default": 32, "minimum": 1, "maximum": 256, "step": 1, "precision": 0, "info": "每个流式分块的贪婪解码上限（会话级，区别于离线 max_tokens）。", "info_en": "Greedy decode budget per streaming chunk (session-scoped; distinct from offline max_tokens)."}, + {"name": "endpointing", "type": "bool", "scope": "session", "session_option": "confucius4_r2t2.endpointing", "label": "endpointing（VAD 自动分段）", "label_en": "endpointing (VAD auto segmentation)", "default": false, "info": "长时听写：语音停顿自动分段并定稿，会话可无限长（否则 115.4s 触限）。", "info_en": "Long-form dictation: close and re-open the segment on speech pauses so sessions run indefinitely (otherwise capped at 115.4 s)."}, + {"name": "vad_threshold", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.vad_threshold", "label": "vad_threshold（语音判定阈值）", "label_en": "vad_threshold (speech probability)", "default": 0.4, "minimum": 0.05, "maximum": 0.95, "step": 0.05, "precision": 2, "info": "越低越不容易丢句尾弱音，但分段更迟钝。", "info_en": "Lower keeps quiet trailing syllables but segments less eagerly."}, + {"name": "vad_min_speech_ms", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.vad_min_speech_ms", "label": "vad_min_speech_ms（最短语音）", "label_en": "vad_min_speech_ms (min speech burst)", "default": 100, "minimum": 0, "maximum": 1000, "step": 10, "precision": 0, "info": "短于此的突发声（咳嗽/碰撞）不触发分段。", "info_en": "Bursts shorter than this (coughs, clicks) do not close a segment."}, + {"name": "vad_min_silence_ms", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.vad_min_silence_ms", "label": "vad_min_silence_ms（停顿判定）", "label_en": "vad_min_silence_ms (pause to finalize)", "default": 200, "minimum": 60, "maximum": 2000, "step": 10, "precision": 0, "info": "静默多久判定一句话结束；决定停顿后的定稿延迟。", "info_en": "Silence duration that closes a segment; drives finalize latency after you stop speaking."}, + {"name": "vad_speech_pad_ms", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.vad_speech_pad_ms", "label": "vad_speech_pad_ms（起始保护）", "label_en": "vad_speech_pad_ms (onset pad)", "default": 50, "minimum": 0, "maximum": 500, "step": 10, "precision": 0, "info": "段起始前保留的音频上下文，避免首字被削。", "info_en": "Audio context prepended before a detected onset so first syllables survive."}, + {"name": "vad_gap_keep_ms", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.vad_gap_keep_ms", "label": "vad_gap_keep_ms（间隙保留）", "label_en": "vad_gap_keep_ms (gap keep window)", "default": 2000, "minimum": 0, "maximum": 10000, "step": 100, "precision": 0, "info": "停顿音频滚动保留给下一段作前导，防止停顿边界的字被丢弃。", "info_en": "Rolling window of pause audio prepended to the next segment so boundary words survive."}, + {"name": "max_segment_seconds", "type": "number", "scope": "session", "session_option": "confucius4_r2t2.max_segment_seconds", "label": "max_segment_seconds（单段上限）", "label_en": "max_segment_seconds (force split)", "default": 20.0, "minimum": 2.0, "maximum": 110.0, "step": 1.0, "precision": 1, "info": "即使没有停顿也强制分段；需远小于 115.4s 声学限位。", "info_en": "Force a segment boundary without a pause; must stay well below the 115.4 s acoustic limit."} ], "pocket_tts": [