feat(confucius4_r2t2): add bounded VAD endpointing for long streaming ASR

app/cli/main.cpp

@@ -576,7 +576,15 @@ void run_streaming(
                     std::cout << "speech_segment";
                     break;
                 }
-                std::cout << " sample=" << activity.sample << " probability=" << activity.probability << "\n";
+                std::cout << " sample=" << activity.sample << " probability=" << activity.probability;
+                if (activity.segment.has_value()) {
+                    std::cout << " start_sample=" << activity.segment->span.start_sample
+                              << " end_sample=" << activity.segment->span.end_sample;
+                    if (!activity.segment->text.empty()) {
+                        std::cout << " text=" << activity.segment->text;
+                    }
+                }
+                std::cout << "\n";
             }
         };
 

app/server/example.json

@@ -54,6 +54,19 @@
         "confucius4_r2t2.chunk_size_ms": "320",
         "confucius4_r2t2.max_tokens": "32"
       }
+    },
+    {
+      "id": "r2t2-asr-live",
+      "family": "confucius4_r2t2",
+      "path": "../../models/Confucius4-R2T2-GGUF/r2t2-q8_0.gguf",
+      "task": "asr",
+      "mode": "streaming",
+      "session_options": {
+        "confucius4_r2t2.chunk_size_ms": "320",
+        "confucius4_r2t2.max_tokens": "32",
+        "confucius4_r2t2.endpointing": "true",
+        "confucius4_r2t2.max_segment_seconds": "20"
+      }
     }
   ]
 }

app/server/runtime.cpp

@@ -876,6 +876,37 @@ std::string stream_event_json(const engine::runtime::StreamEvent & event) {
             << ",\"language\":" << json_quote(event.partial_text->language)
             << "}";
     }
+    if (!event.voice_activity.empty()) {
+        field("voice_activity");
+        out << "[";
+        for (size_t i = 0; i < event.voice_activity.size(); ++i) {
+            const auto & activity = event.voice_activity[i];
+            if (i != 0) {
+                out << ",";
+            }
+            const char * kind = "speech_segment";
+            using Kind = engine::runtime::VoiceActivityEvent::Kind;
+            if (activity.kind == Kind::SpeechStart) {
+                kind = "speech_start";
+            } else if (activity.kind == Kind::SpeechEnd) {
+                kind = "speech_end";
+            }
+            out << "{\"kind\":" << json_quote(kind)
+                << ",\"sample\":" << activity.sample
+                << ",\"probability\":" << activity.probability;
+            if (activity.segment.has_value()) {
+                out << ",\"segment\":{\"start_sample\":" << activity.segment->span.start_sample
+                    << ",\"end_sample\":" << activity.segment->span.end_sample
+                    << ",\"confidence\":" << activity.segment->confidence;
+                if (!activity.segment->text.empty()) {
+                    out << ",\"text\":" << json_quote(activity.segment->text);
+                }
+                out << "}";
+            }
+            out << "}";
+        }
+        out << "]";
+    }
     if (event.audio_output.has_value()) {
         const auto wav = encode_pcm16_wav(*event.audio_output);
         field("audio");
@@ -3017,14 +3048,31 @@ HttpResponse ServerState::handle_transcription_live(const HttpRequest & request)
                 task_request,
                 audio,
                 [&](const engine::runtime::StreamEvent & event) {
-                    if (!event.partial_text.has_value() || event.partial_text->text.empty()) {
-                        return;
+                    // Segment-final deltas must precede the boundary: the
+                    // client first appends the rollback tail, then commits the
+                    // segment and starts a new line (reset semantics, mirroring
+                    // the reference ws_server integrator).
+                    if (event.partial_text.has_value() && !event.partial_text->text.empty()) {
+                        write_sse(
+                            writer,
+                            "{\"type\":\"transcript.text.delta\",\"delta\":" +
+                                json_quote(event.partial_text->text) +
+                                "}");
+                    }
+                    for (const auto & activity : event.voice_activity) {
+                        using Kind = engine::runtime::VoiceActivityEvent::Kind;
+                        if (activity.kind != Kind::SpeechEnd || !activity.segment.has_value()) {
+                            continue;
+                        }
+                        const auto & segment = *activity.segment;
+                        std::ostringstream out;
+                        out << "{\"type\":\"transcript.segment.end\",\"text\":"
+                            << json_quote(segment.text)
+                            << ",\"start_sample\":" << segment.span.start_sample
+                            << ",\"end_sample\":" << segment.span.end_sample
+                            << ",\"reset\":true}";
+                        write_sse(writer, out.str());
                     }
-                    write_sse(
-                        writer,
-                        "{\"type\":\"transcript.text.delta\",\"delta\":" +
-                            json_quote(event.partial_text->text) +
-                            "}");
                 },
                 busy_timeout_ms);
             if (!timed_result.result.text_output.has_value()) {

docs/community_models/r2t2.md

@@ -116,6 +116,14 @@ quality.
 | `confucius4_r2t2.unfixed_token_num` | integer | `5` | Tokens rolled back from the accumulated text before it is used as the prefix prompt. |
 | `confucius4_r2t2.rollback_punctuation` | `true`, `false` | `false` | Keep trailing text uncommitted when it already ends with punctuation instead of rolling back tokens. |
 | `confucius4_r2t2.max_tokens` | integer | `32` | Greedy decode budget per chunk and for the final flush. |
+| `confucius4_r2t2.endpointing` | `true`, `false` | `false` | VAD-driven endpointing: split the stream into speech segments so sessions run indefinitely (see below). Streaming only. |
+| `confucius4_r2t2.vad_model_path` | path | `assets/framework/models/silero_vad` | Silero VAD weights used by endpointing. |
+| `confucius4_r2t2.vad_threshold` | 0-1 | `0.4` | Speech probability threshold for VAD decisions. Lower values hold speech open through quiet tails at the cost of stickier segmentation. |
+| `confucius4_r2t2.vad_min_speech_ms` | integer | `100` | Shorter bursts (coughs, clicks) are absorbed into the surrounding segment instead of closing it. |
+| `confucius4_r2t2.vad_min_silence_ms` | integer | `200` | Silence that closes a segment; drives finalize latency after the speaker stops. |
+| `confucius4_r2t2.vad_speech_pad_ms` | integer | `50` | Context kept ahead of a detected onset so first syllables are not clipped. |
+| `confucius4_r2t2.vad_gap_keep_ms` | integer | `2000` | Rolling window of non-speech audio prepended to the next segment, so words on the quiet side of a pause survive. |
+| `confucius4_r2t2.max_segment_seconds` | 0-110 | `20` | Force a boundary after this much segment audio even without a VAD pause. |
 | `confucius4_r2t2.audio_encoder_weight_type` | `native`, `f32`, `f16` | `native` | Audio tower weight storage. |
 | `confucius4_r2t2.thinker_weight_type` (alias `confucius4_r2t2.weight_type`) | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Thinker weight storage. |
 | `confucius4_r2t2.audio_encoder_graph_arena_mb` | MB | `128` | Audio tower graph arena. |
@@ -164,11 +172,64 @@ prefix emits nothing. The uncommitted tail is delivered in the final result
 
 The reference also ships a rolling-window variant for unbounded streams ("no
 reset": keep 16 s of audio, discard the oldest 8 s and the matching text). This
-port implements the standard variant used by the upstream WebSocket server,
-which bounds audio per utterance with VAD. Because the audio tower uses 1500
-positions, a single unsegmented stream is limited to roughly 110 s of
-accumulated audio; segment longer streams (as the reference server does) or add
-the rolling-window variant.
+port implements the standard variant used by the upstream WebSocket server.
+Because the audio tower uses 1500 positions, a single unsegmented stream is
+limited to roughly 110 s of accumulated audio; enable endpointing (below) for
+longer sessions.
+
+### Endpointing: long-form dictation (VAD segmentation)
+
+Dictation is unbounded: a session may run for minutes, past the audio tower's
+1500-frame position table (115.40 s), and per-chunk decode cost grows with the
+accumulated audio. With `confucius4_r2t2.endpointing=true` the session embeds a Silero
+VAD and splits the stream into speech segments:
+
+* On each accepted speech end (silence of `vad_min_silence_ms` after speech of
+  at least `vad_min_speech_ms`) the session runs the authoritative final flush,
+  publishes the segment text, and re-opens a fresh LSP segment. Segment audio is
+  bounded by `max_segment_seconds`, far inside the position table.
+* Between segments the session keeps a rolling `vad_gap_keep_ms` window of
+  non-speech audio and prepends it to the next segment, so words on the quiet
+  side of a pause are not clipped; all other silence is dropped, so idle time
+  costs nothing.
+* A speaker who never pauses is force-split at `max_segment_seconds`. The
+  limit includes retained gap audio and is enforced before decoding, even
+  when one input packet contains several segments. The retained gap is capped
+  at one sample less than this limit; `vad_gap_keep_ms=0` disables it.
+* Input packets are buffered into 32 ms VAD frames. Segment boundaries and
+  decoding are independent of transport packet sizes; finalization consumes
+  any shorter tail. Boundary spans describe the audio actually decoded,
+  including retained context and detection latency, in input sample-rate units.
+  Silero state remains continuous across recognizer segment resets.
+* The language/context request options survive segment resets; segments decode
+  independently, exactly like the reference WebSocket server's per-utterance
+  resets.
+
+Boundaries need no framework changes: they travel in the existing stream-event
+voice-activity list as `SpeechEnd` events whose segment carries the final text
+and span. On the server's live route each boundary is emitted as
+
+```json
+{"type": "transcript.segment.end", "text": "...", "start_sample": 7904,
+ "end_sample": 47392, "reset": true}
+```
+
+Metadata-only rollback prefixes (such as `language English` before
+`<asr_text>`) are suppressed in deltas. This intentionally corrects the
+reference integrator's metadata leak; historical committed-delta and
+`fixed_text` comparisons below describe the earlier reference-compatible behavior.
+
+Committed deltas keep flowing append-only inside a segment; on `reset` the
+client commits the segment (the authoritative text replaces the delta-assembled
+buffer) and starts a new one. `transcript.text.done` carries all segments joined
+with spaces, and the result's `segments` array lists every segment span+text.
+
+Historical measurements before graph reuse and VAD-frame scheduling
+(M3/Metal, 320 ms chunks, 14 s utterances separated by 1 s gaps):
+without endpointing a 134.6 s stream fails safely at 115.40 s after 360 chunks
+with per-chunk encoder cost growing 22 -> 906 ms; with endpointing the same
+stream completes with 36 segments, per-chunk cost flat at 22 -> 28 ms (max
+51 ms) and a maximum of 512 encoder frames per decode.
 
 ### Streaming graph reuse
 
@@ -247,6 +308,11 @@ ffmpeg -f avfoundation -i ":0" -ar 16000 -ac 1 -f s16le - \
       'http://127.0.0.1:8488/v1/audio/transcriptions/live?model=r2t2-asr-stream&sample_rate=16000&channels=1&sample_format=s16le'
 ```
 
+For unbounded dictation, point the live route at a model configured with
+`confucius4_r2t2.endpointing=true` (the example configuration's `r2t2-asr-live`). The
+stream then emits `transcript.segment.end` events with `reset:true` at every
+pause; clients commit each segment and start a new one.
+
 ## GGUF checkpoints
 
 GGUF is supported for Q8_0 and higher precision. `f16`, `q8_0`, and the native

include/engine/community_models/confucius4_r2t2/session.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "engine/framework/runtime/session.h"
 #include "engine/framework/runtime/session_base.h"
 #include "engine/framework/runtime/model.h"
 #include "engine/community_models/confucius4_r2t2/assets.h"
@@ -11,10 +12,16 @@
 
 #include <chrono>
 #include <cstdint>
+#include <filesystem>
 #include <memory>
 #include <string>
 #include <vector>
 
+namespace engine::models::silero_vad {
+class SileroRuntime;
+struct SileroVADConfig;
+}
+
 namespace engine::community_models::confucius4_r2t2 {
 
 /// Spec-backed loader factory (schema-v1 contract): the framework derives
@@ -32,6 +39,27 @@ struct R2T2ASRStreamConfig {
     int64_t max_new_tokens = 32;
 };
 
+/// VAD-driven endpointing for long streaming sessions (dictation / input
+/// method). Defaults mirror the reference ws_server.py FireRed VAD intent
+/// (0.4 speech threshold, 200 ms minimum silence, 50 ms onset pad, 20 s
+/// maximum speech frame) expressed in Silero VAD terms. When enabled, a
+/// speech end closes the current segment: the session runs the authoritative
+/// final flush, publishes a VoiceActivityEvent::SpeechEnd carrying the
+/// segment text, and re-opens a fresh LSP segment so audio towers never see
+/// more than one segment worth of audio (far below the 1500-frame position
+/// table). A bounded non-speech context window is retained between segments;
+/// it also counts toward the segment cap.
+struct R2T2ASREndpointingConfig {
+    bool enabled = false;
+    std::filesystem::path vad_model_path = "assets/framework/models/silero_vad";
+    float threshold = 0.4f;
+    int min_speech_ms = 100;
+    int min_silence_ms = 200;
+    int speech_pad_ms = 50;
+    int gap_keep_ms = 2000;
+    double max_segment_seconds = 20.0;
+};
+
 /// Confucius4-R2T2 streaming ASR session.
 ///
 /// This family owns its full Qwen3-ASR-derived graph (audio tower, thinker,
@@ -84,6 +112,42 @@ class R2T2ASRSession final
     std::string decode_rollback_prefix(const std::vector<int32_t> & ids, int64_t rollback) const;
     void publish_stream_delta(const std::string & fixed_text, runtime::StreamEvent & event);
 
+    // Endpointed streaming: the family embeds a Silero VAD runtime and splits
+    // the incoming stream into speech segments. See R2T2ASREndpointingConfig.
+    void ensure_vad_runtime();
+    /// Steps the VAD over the chunk; returns true when the current segment
+    /// must end (accepted speech end).
+    void process_endpoint_frame(const runtime::AudioChunk & chunk, runtime::StreamEvent & event);
+    bool feed_vad(const runtime::AudioChunk & chunk);
+    /// Runs the authoritative final flush for the open segment, publishes the
+    /// segment boundary event, and re-opens a fresh LSP segment.
+    void flush_segment(runtime::StreamEvent & event, bool from_vad);
+    void begin_new_segment();
+    void append_stream_text(const std::string & segment_text);
+    std::string joined_stream_text(const std::string & current_segment_text) const;
+    int64_t to_stream_samples(int64_t vad_samples) const;
+    const models::silero_vad::SileroVADConfig & vad_config() const;
+
+    R2T2ASREndpointingConfig endpointing_;
+    std::unique_ptr<models::silero_vad::SileroRuntime> vad_runtime_;
+    std::unique_ptr<models::silero_vad::SileroVADConfig> vad_config_;
+    // Session-global VAD bookkeeping: survives segment resets so spans stay
+    // monotonic across the whole stream.
+    std::vector<float> endpoint_input_;
+    std::vector<float> vad_remainder_;
+    int64_t vad_consumed_samples_ = 0;
+    // Gap-context window (interleaved, stream format) retained before onset.
+    std::vector<float> vad_seed_;
+    bool in_speech_ = false;
+    bool segment_has_audio_ = false;
+    runtime::VoiceActivityEvent pending_speech_end_;
+    int64_t segment_start_stream_sample_ = 0;
+    int64_t segment_stream_frames_ = 0;
+    int64_t max_segment_stream_frames_ = 0;
+    int64_t stream_frames_consumed_ = 0;
+    std::vector<std::pair<runtime::TimeSpan, std::string>> completed_segments_;
+    int64_t segment_index_ = 0;
+
     runtime::TaskSpec task_;
     std::shared_ptr<const R2T2ASRAssets> assets_;
     R2T2ASRStreamConfig stream_config_;

model_specs/confucius4_r2t2.json

@@ -288,6 +288,62 @@
         "required": false,
         "default": 64,
         "description": "Thinker weight context in MB."
+      },
+      {
+        "name": "endpointing",
+        "type": "bool",
+        "required": false,
+        "default": false,
+        "description": "VAD-driven endpointing for long streaming sessions: close and re-open the LSP segment on speech ends so sessions run past the 115.40 s audio-tower limit with bounded cost. Streaming only."
+      },
+      {
+        "name": "vad_model_path",
+        "type": "string",
+        "required": false,
+        "default": "assets/framework/models/silero_vad",
+        "description": "Silero VAD model directory (or safetensors file) used by endpointing."
+      },
+      {
+        "name": "vad_threshold",
+        "type": "float",
+        "required": false,
+        "default": 0.4,
+        "description": "Speech probability threshold for endpointing VAD decisions (0-1)."
+      },
+      {
+        "name": "vad_min_speech_ms",
+        "type": "int",
+        "required": false,
+        "default": 100,
+        "description": "Minimum speech-burst duration; shorter bursts are absorbed into the surrounding segment instead of triggering a boundary."
+      },
+      {
+        "name": "vad_min_silence_ms",
+        "type": "int",
+        "required": false,
+        "default": 200,
+        "description": "Silence duration that closes a segment; drives finalize latency after the speaker stops (~200 ms plus one decode)."
+      },
+      {
+        "name": "vad_speech_pad_ms",
+        "type": "int",
+        "required": false,
+        "default": 50,
+        "description": "Context prepended to a detected speech onset so first syllables are not clipped."
+      },
+      {
+        "name": "vad_gap_keep_ms",
+        "type": "int",
+        "required": false,
+        "default": 2000,
+        "description": "Rolling window of non-speech audio prepended to the next segment, capped below the segment limit. Zero disables retention."
+      },
+      {
+        "name": "max_segment_seconds",
+        "type": "float",
+        "required": false,
+        "default": 20.0,
+        "description": "Force a segment boundary after this much segment audio even without a VAD pause; must stay in (0, 110] because the audio tower position table holds 1500 frames (115.40 s)."
       }
     ],
     "load": []