fix(classroom): 智能采集语音识别流式化+merge降级+去重复传输

- 流式ASR：VADMarker添加onSegmentReady回调，语音段完成后即时转写并回填audioText，UI显示'已转写N段语音'进度

- ASR语言修复：ASRWorker支持language构造参数，captureManager传入用户配置，消除硬编码'zh'

- merge降级：mergeNotes失败时本地拼接片段笔记(零网络)，避免显示错误+触发全量重发

- 去重复传输：local-concat结果不显示重试按钮，防止增量已处理帧被全量重发

- sessionAnalyzer：优先使用流式已转写文本(seg.audioText)，仅对未转写段补充ASR

Author: Aparencia

client/src/features/classroom/hooks/useClassroomCapture.ts

@@ -23,6 +23,7 @@ import type {
 } from '@/lib/capture';
 import { analyzeSession, analyzeVideo, analyzePartial, mergeNotes } from '@/lib/ai/sessionAnalyzer';
 import type { AnalyzeResult } from '@/lib/ai/sessionAnalyzer';
+import { aiClient } from '@/lib/http/apiClient';
 
 interface IPCAudioStartResult {
   success: boolean;
@@ -136,6 +137,7 @@ export function useClassroomCapture() {
   const pendingKeyframesRef = useRef<KeyFrame[]>([]);
   const isPartialAnalyzingRef = useRef(false);
   const [partialCount, setPartialCount] = useState(0);
+  const [transcribedCount, setTranscribedCount] = useState(0);
   const INCREMENTAL_BATCH_SIZE = 5;
 
   useEffect(() => {
@@ -173,6 +175,46 @@ export function useClassroomCapture() {
     return () => { offKeyframe(); offBundleReady(); };
   }, [config.language]);
 
+  // ── Path B：流式 ASR — 语音段完成后立即转写 ──
+  useEffect(() => {
+    const offSegmentReady = captureEventBus.on<{ sessionId: string; segment: import('@/lib/capture').AudioSegment }>(
+      'smart:audio_segment_ready',
+      (data) => {
+        const seg = data.segment;
+        // 先将音频段加入 bundle
+        setSmartBundle((prev) => ({
+          ...prev,
+          audioSegments: [...(prev.audioSegments ?? []), seg],
+        }));
+
+        // 后台流式 ASR 转写（不阻塞采集）
+        if (!seg.audioBase64) return;
+        const lang = config.language === 'en' ? 'en' : config.language === 'mixed' ? 'auto' : 'zh';
+        aiClient.post<{ text: string }>('/api/v1/asr/transcribe', {
+          audio_base64: seg.audioBase64,
+          language: lang,
+          sample_rate: 16000,
+          channels: 1,
+        })
+          .then((resp) => {
+            const text = resp.text?.trim() || null;
+            // 将转写结果回填到对应的音频段
+            setSmartBundle((prev) => ({
+              ...prev,
+              audioSegments: (prev.audioSegments ?? []).map((s) =>
+                s.id === seg.id ? { ...s, audioText: text } : s,
+              ),
+            }));
+            if (text) setTranscribedCount((c) => c + 1);
+          })
+          .catch((err) => {
+            console.warn('[useClassroomCapture] 流式 ASR 转写失败:', err);
+          });
+      },
+    );
+    return () => { offSegmentReady(); };
+  }, [config.language]);
+
   // ── Path C：监听录制视频就绪 ──
   useEffect(() => {
     const offVideoReady = captureEventBus.on<{ sessionId: string; videoRecording: VideoRecording }>(
@@ -477,14 +519,22 @@ export function useClassroomCapture() {
         if (partialNotesRef.current.length > 0) {
           setIsAnalyzing(true);
           setAnalysisError(null);
+          const partials = [...partialNotesRef.current];
           try {
-            const result = await mergeNotes(partialNotesRef.current, {
+            const result = await mergeNotes(partials, {
               duration: (smartBundle.duration ?? 0) / 1000,
               language: config.language,
             });
             setAnalysisResult(result);
-          } catch (err) {
-            setAnalysisError(err instanceof Error ? err.message : '笔记合并失败');
+          } catch {
+            // 降级：本地拼接片段笔记（无需 AI，零网络，避免全量重发）
+            const fallbackContent = partials.join('\n\n---\n\n');
+            setAnalysisResult({
+              content: fallbackContent,
+              keyframesAnalyzed: smartBundle.keyframes?.length ?? 0,
+              modelUsed: 'local-concat',
+            });
+            toast({ type: 'warning', message: 'AI 合并不可用，已直接拼接片段笔记' });
           } finally {
             setIsAnalyzing(false);
           }
@@ -597,7 +647,7 @@ export function useClassroomCapture() {
     // 路径
     capturePath, setCapturePath, smartBundle,
     // 分析
-    isAnalyzing, analysisResult, analysisError, partialCount,
+    isAnalyzing, analysisResult, analysisError, partialCount, transcribedCount,
     // 录制
     recordingStatus, videoFilePath,
     // 操作

client/src/features/classroom/pages/ClassroomPage.tsx

@@ -346,6 +346,12 @@ export default function ClassroomPage() {
                 <span className="text-b3 text-brand-600">已增量分析 {capture.partialCount} 段，课后将快速合并生成笔记</span>
               </div>
             )}
+            {capture.transcribedCount > 0 && (
+              <div className="mx-4 mt-2 flex items-center gap-2 px-3 py-2 rounded-kb-md bg-emerald-50/50 border border-emerald-100/50">
+                <Mic className="w-4 h-4 text-emerald-500" strokeWidth={1.5} />
+                <span className="text-b3 text-emerald-600">已转写 {capture.transcribedCount} 段语音</span>
+              </div>
+            )}
           </>
         )}
 
@@ -361,7 +367,7 @@ export default function ClassroomPage() {
             error={capture.analysisError}
             onInsert={() => capture.handleDismissAnalysis()}
             onDismiss={capture.handleDismissAnalysis}
-            onRetry={capture.handleAnalyze}
+            onRetry={capture.analysisResult?.modelUsed === 'local-concat' ? undefined : capture.handleAnalyze}
           />
         )}
       </div>

client/src/lib/ai/asrWorker.ts

@@ -38,6 +38,11 @@ interface TranscribeApiResponse {
 
 export class ASRWorker implements PipelineWorker {
   name = 'asr-worker';
+  private readonly language: string;
+
+  constructor(language: string = 'zh') {
+    this.language = language;
+  }
 
   canProcess(message: PipelineMessage): boolean {
     return message.type === 'audio_chunk';
@@ -49,12 +54,12 @@ export class ASRWorker implements PipelineWorker {
     // ArrayBuffer → base64
     const base64 = arrayBufferToBase64(audioData.audioBuffer);
 
-    // 调用后端 ASR API
+    // 调用后端 ASR API（使用配置的语言）
     const response = await aiClient.post<TranscribeApiResponse>(
       '/api/v1/asr/transcribe',
       {
         audio_base64: base64,
-        language: 'zh',
+        language: this.language,
         sample_rate: audioData.sampleRate,
         channels: audioData.channels,
       },

client/src/lib/ai/sessionAnalyzer.ts

@@ -6,7 +6,8 @@
 import { supabase } from '@/lib/auth/supabaseClient';
 import { getActiveUserKey } from '@/lib/ai/apiKeyManager';
 import { classroomNoteStore } from '@/lib/storage/classroomNoteStore';
-import type { SessionBundle, KeyFrame } from '@/lib/capture/captureTypes';
+import { aiClient } from '@/lib/http/apiClient';
+import type { SessionBundle, KeyFrame, AudioSegment } from '@/lib/capture/captureTypes';
 
 // ================================================================
 // 分析结果类型
@@ -20,6 +21,42 @@ export interface AnalyzeResult {
   source?: 'local' | 'remote';
 }
 
+// ================================================================
+// 音频段转写工具
+// ================================================================
+
+interface TranscribeResponse {
+  text: string;
+  confidence: number;
+  model_used: string;
+}
+
+/**
+ * 将音频段的 audioBase64 通过 ASR API 转写为文本
+ * 失败时静默返回 null，不阻塞整体分析流程
+ */
+async function transcribeSegment(
+  seg: AudioSegment,
+  language: string,
+): Promise<string | null> {
+  if (!seg.audioBase64) return null;
+  try {
+    const resp = await aiClient.post<TranscribeResponse>(
+      '/api/v1/asr/transcribe',
+      {
+        audio_base64: seg.audioBase64,
+        language,
+        sample_rate: 16000,
+        channels: 1,
+      },
+    );
+    return resp.text?.trim() || null;
+  } catch (e) {
+    console.warn('[sessionAnalyzer] 音频段转写失败:', e);
+    return null;
+  }
+}
+
 // ================================================================
 // 分析函数
 // ================================================================
@@ -42,11 +79,16 @@ export async function analyzeSession(
     imageBase64: kf.imageBase64,
     changeType: kf.changeType,
   }));
-  const audioSegments = bundle.audioSegments.map((seg) => ({
-    timestampStart: seg.timestampStart / 1000,
-    timestampEnd: seg.timestampEnd / 1000,
-    audioText: null,
-  }));
+
+  // 转写音频段：优先使用流式 ASR 已转写的文本，仅对未转写的段进行补充转写
+  const lang = options?.language === 'en' ? 'en' : options?.language === 'mixed' ? 'auto' : 'zh';
+  const audioSegments = await Promise.all(
+    bundle.audioSegments.map(async (seg) => ({
+      timestampStart: seg.timestampStart / 1000,
+      timestampEnd: seg.timestampEnd / 1000,
+      audioText: seg.audioText ?? await transcribeSegment(seg, lang),
+    })),
+  );
 
   const result = await window.electronAPI!.invoke('ai_session_analyze', {
     keyframes,

client/src/lib/capture/captureManager.ts

@@ -110,7 +110,7 @@ export class CaptureManager {
     this.capturePath = config.path ?? 'fine';
 
     // ================================================================
-    // Path B 智能模式：跳过 Pipeline/Worker，用轻量采样器替代
+    // Path B 智能模式：跳过 Pipeline/Worker，用轻量采样器 + 流式 ASR 替代
     // ================================================================
     if (this.capturePath === 'smart') {
       const session = await captureStore.createSession({
@@ -129,6 +129,14 @@ export class CaptureManager {
       this.smartSampler = new SmartSampler();
       this.vadMarker = new VADMarker();
 
+      // 流式 ASR：语音段完成后立即发射事件，由上层 Hook 触发转写
+      this.vadMarker.onSegmentReady = (segment) => {
+        captureEventBus.emit('smart:audio_segment_ready', {
+          sessionId: this.sessionId,
+          segment,
+        });
+      };
+
       captureEventBus.emit('session:started', {
         sessionId: this.sessionId,
         config,
@@ -176,9 +184,9 @@ export class CaptureManager {
       uiAutomationAvailable: false, // Electron 环境下后续检测
     });
 
-    // 根据决策动态注册 ASR Worker
+    // 根据决策动态注册 ASR Worker（传入用户配置的语言）
     if (this.lastDecision.audioEnabled && !this.asrWorker) {
-      this.asrWorker = new ASRWorker();
+      this.asrWorker = new ASRWorker(config.language || 'zh');
       this.pipeline.registerWorker(this.asrWorker);
       this.dispatcher.registerWorker(this.asrWorker);
     } else if (!this.lastDecision.audioEnabled && this.asrWorker) {
@@ -411,7 +419,7 @@ export class CaptureManager {
     }
 
     // ================================================================
-    // Path B 智能模式：VADMarker 检测语音段，不送入 ASR
+    // Path B 智能模式：VADMarker 检测语音段，流式触发 ASR 转写
     // ================================================================
     if (this.capturePath === 'smart' && this.vadMarker) {
       this.vadMarker.processChunk(audioData);

client/src/lib/capture/captureTypes.ts

@@ -135,13 +135,15 @@ export interface KeyFrame {
   changeType: 'slide_change' | 'writing' | 'scene_change' | 'periodic';
 }
 
-/** @ai-context VAD 标记器切出的语音段，含编码后的音频数据 */
+/** @ai-context VAD 标记器切出的语音段，含编码后的音频数据，支持流式 ASR 转写 */
 export interface AudioSegment {
   id: string;
   timestampStart: number;
   timestampEnd: number;
   audioBase64: string;
   energy: number;
+  /** 流式 ASR 转写结果（课堂进行中即时填充，无需课后批量转写） */
+  audioText?: string | null;
 }
 
 /** @ai-context 全局时间轴条目，串联关键帧和语音段供后续分析回放 */

client/src/lib/capture/vadMarker.ts

@@ -2,8 +2,8 @@
  * VAD 音频标记器 — Path B 语音段检测与分段
  *
  * @ai-context
- * Path B 不做实时 ASR，而是通过 RMS 能量检测将连续语音切段，
- * 每段打包为 WAV base64 供后续按需转写，降低 AI 调用次数。
+ * Path B 通过 RMS 能量检测将连续语音切段，
+ * 每段完成后立即触发 onSegmentReady 回调，支持流式 ASR 转写。
  */
 
 import type { AudioChunkData, AudioSegment, TimelineEntry } from './captureTypes';
@@ -36,6 +36,9 @@ export class VADMarker {
   private segments: AudioSegment[] = [];
   private timeline: TimelineEntry[] = [];
 
+  /** 语音段完成回调（流式 ASR 触发点） */
+  onSegmentReady: ((segment: AudioSegment) => void) | null = null;
+
   // 当前语音段状态
   private isSpeaking = false;
   private speechStartTime = 0;
@@ -172,6 +175,12 @@ export class VADMarker {
       audioBase64,
       energy: Math.round(avgEnergy * 10000) / 10000,
     });
+
+    // 流式触发：语音段完成后立即通知外部进行 ASR 转写
+    const newSegment = this.segments[this.segments.length - 1];
+    if (this.onSegmentReady) {
+      this.onSegmentReady(newSegment);
+    }
   }
 }
 

server/nginx/nginx.conf

@@ -1,3 +1,4 @@
+
 worker_processes auto;
 
 events {