Skip to content

Commit ffcd193

Browse files
authored
Merge pull request #5 from Aparencia/dev
fix: 修复课堂 ASR 链路故障(降级链断裂 / Redis 未连接 / 音频静音无回退)
2 parents 8ff2818 + dd996ba commit ffcd193

64 files changed

Lines changed: 3049 additions & 164 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎client/electron/ai/handlers/sessionAnalyzeHandler.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ function register(): void {
3737
audioText: string | null;
3838
}>;
3939
duration: number; // 已转为秒
40+
/** 分析模式:full 全量 / partial 增量片段(网关据此选择 Prompt 模板) */
41+
mode?: 'full' | 'partial';
4042
language?: string;
4143
authToken?: string;
4244
userApiKey?: string;
@@ -45,7 +47,7 @@ function register(): void {
4547
const startMs = Date.now();
4648
const kfCount = args.keyframes?.length ?? 0;
4749
const segCount = args.audioSegments?.length ?? 0;
48-
logger.info(`[AI] [session-analyze] IPC received: keyframes=${kfCount}, audioSegments=${segCount}, duration=${args.duration}s, language=${args.language ?? 'zh'}, hasAuth=${!!args.authToken}`);
50+
logger.info(`[AI] [session-analyze] IPC received: keyframes=${kfCount}, audioSegments=${segCount}, duration=${args.duration}s, mode=${args.mode ?? 'full'}, language=${args.language ?? 'zh'}, hasAuth=${!!args.authToken}`);
4951

5052
const reqBody = {
5153
keyframes: args.keyframes.map(kf => ({
@@ -59,6 +61,7 @@ function register(): void {
5961
audio_text: seg.audioText,
6062
})),
6163
duration: args.duration,
64+
mode: args.mode ?? 'full',
6265
language: args.language ?? 'zh',
6366
};
6467

‎client/electron/audioCapture.ts‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@
99
* 5. PCM 数据块通过 IPC 回传主进程,添加单调时间戳后推送给消费者
1010
*
1111
* @ai-context: 系统音频捕获:渲染进程 getDisplayMedia 采集、主进程聚合分块。
12+
*
13+
* TODO(现场课程): 当前仅支持系统音频环回(捕获电脑播放的声音,适配网课场景)。
14+
* 后续「现场课程」需扩展麦克风输入源:listAudioSources 增加枚举
15+
* navigator.mediaDevices 的 audioinput 设备,getUserMedia 直接以 deviceId
16+
* 采集麦克风,并与环回源并列供用户选择(或双路混合)。
1217
*/
1318

1419
import { desktopCapturer, DesktopCapturerSource, BrowserWindow } from 'electron';
@@ -122,14 +127,18 @@ export class AudioCapture {
122127
* 开始音频捕获
123128
*
124129
* 1. 如果未指定 sourceId,自动选择第一个可用的系统音频源
125-
* 2. 向渲染进程发送启动指令(含 sourceId + 配置)
126-
* 3. 渲染进程负责 getUserMedia 和音频切片
130+
* 2. 方案A:若指定的是窗口源(window:xxx),优先尝试窗口级音频捕获,
131+
* 同时解析一个屏幕环回源作为降级候选(Windows 上窗口源音频支持
132+
* 依赖 Chromium 版本,失败时渲染进程自动回退环回源)
133+
* 3. 向渲染进程发送启动指令(含 sourceId + fallbackSourceId + 配置)
134+
* 4. 渲染进程负责 getUserMedia 和音频切片
127135
*/
128136
async start(win: BrowserWindow, sourceId?: string): Promise<void> {
129137
if (this.capturing || this.disposed) return;
130138

131139
// 解析音频源
132140
let resolvedSourceId = sourceId ?? null;
141+
let fallbackSourceId: string | null = null;
133142
if (!resolvedSourceId) {
134143
const sources = await listAudioSources();
135144
if (sources.length === 0) {
@@ -138,6 +147,15 @@ export class AudioCapture {
138147
}
139148
resolvedSourceId = sources[0].id;
140149
logger.info(`[AudioCapture] 自动选择音频源: ${sources[0].name} (${resolvedSourceId})`);
150+
} else if (resolvedSourceId.startsWith('window:')) {
151+
// 方案A:窗口源直采目标应用(如 B站客户端/浏览器)的音频,
152+
// 预解析屏幕环回源作为降级候选,防御窗口级捕获不受支持的环境
153+
const sources = await listAudioSources();
154+
fallbackSourceId = sources[0]?.id ?? null;
155+
logger.info(
156+
`[AudioCapture] 方案A 窗口源音频捕获: ${resolvedSourceId}, ` +
157+
`降级候选: ${fallbackSourceId ?? '无'}`,
158+
);
141159
}
142160

143161
this.capturing = true;
@@ -153,6 +171,7 @@ export class AudioCapture {
153171
if (!win.isDestroyed()) {
154172
win.webContents.send('audio_capture_do_start', {
155173
sourceId: resolvedSourceId,
174+
fallbackSourceId,
156175
options: this.options,
157176
});
158177
}

‎client/electron/cspPolicy.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,10 @@ export function installCspPolicy(isDev: boolean): void {
4343
const extraConnectSrc = buildExtraConnectSrc();
4444
// 开发环境:允许 unsafe-inline/unsafe-eval(Vite HMR 需要),worker-src 需要 blob:
4545
// 生产环境:禁止 unsafe-eval,保留 unsafe-inline(Tailwind 运行时需要)
46+
// img-src 放行 keyframe:(课堂关键帧本地图片自定义协议,见 ipc/keyframeStorage.ts)
4647
const csp = isDev
47-
? `default-src 'self' 'unsafe-inline' 'unsafe-eval' http://localhost:* ws://localhost:*; worker-src 'self' blob: http://localhost:*; connect-src 'self' http://localhost:* ws://localhost:* https://*.supabase.co wss://*.supabase.co https://entropydecrease.com wss://entropydecrease.com${extraConnectSrc}; img-src 'self' data: blob: https://*.supabase.co; font-src 'self' data:;`
48-
: `default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://*.supabase.co; font-src 'self' data:; connect-src 'self' https://*.supabase.co wss://*.supabase.co https://entropydecrease.com wss://entropydecrease.com${extraConnectSrc}; frame-ancestors 'none';`;
48+
? `default-src 'self' 'unsafe-inline' 'unsafe-eval' http://localhost:* ws://localhost:*; worker-src 'self' blob: http://localhost:*; connect-src 'self' http://localhost:* ws://localhost:* https://*.supabase.co wss://*.supabase.co https://entropydecrease.com wss://entropydecrease.com${extraConnectSrc}; img-src 'self' data: blob: keyframe: https://*.supabase.co; font-src 'self' data:;`
49+
: `default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: keyframe: https://*.supabase.co; font-src 'self' data:; connect-src 'self' https://*.supabase.co wss://*.supabase.co https://entropydecrease.com wss://entropydecrease.com${extraConnectSrc}; frame-ancestors 'none';`;
4950

5051
callback({
5152
responseHeaders: {

‎client/electron/ipc/channels.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ export const IPC_CHANNELS = {
5656
// 文件系统
5757
FS_READ_FILE: 'fs:read-file',
5858

59+
// 课堂关键帧图片持久化
60+
KEYFRAME_SAVE: 'keyframe_save',
61+
KEYFRAME_CLEANUP: 'keyframe_cleanup',
62+
5963
// 存储路径
6064
STORAGE_CHANGE_PATH: 'storage:change-path',
6165
STORAGE_GET_ACTIVE_PATH: 'storage:get-active-path',
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/**
2+
* 课堂关键帧图片持久化 IPC + keyframe:// 自定义协议
3+
* Classroom keyframe image persistence IPC handlers and custom
4+
* keyframe:// protocol for renderer-side image loading.
5+
*
6+
* @ai-context: 关键帧 JPEG 本地落盘到 {userData}/captures/{sessionId}/{keyframeId}.jpg,
7+
* 遵守本地优先原则(图片不上传)。渲染进程通过自定义 keyframe:// 协议加载:
8+
* 开发页面源为 http://localhost、生产为 file://,file:// 子资源会被
9+
* webSecurity/CSP 拦截,故注册 standard+secure 专用协议(CSP img-src 已放行 keyframe:)。
10+
* (Custom protocol is used because file:// subresources are blocked by
11+
* webSecurity/CSP under both dev http origin and prod file origin.)
12+
* @ai-context: sessionId/keyframeId 仅允许 UUID/安全字符(防路径穿越);
13+
* keyframe_cleanup 删除整个会话目录,由笔记删除路径调用(失败静默)。
14+
*/
15+
import { app, net, protocol } from 'electron';
16+
import * as path from 'path';
17+
import { mkdir, rm, writeFile } from 'fs/promises';
18+
import { pathToFileURL } from 'url';
19+
import { safeHandle } from '../ipcUtils.js';
20+
import { logger } from '../logger.js';
21+
22+
/** 自定义协议名与固定 host(keyframe://capture/{sessionId}/{keyframeId}.jpg) */
23+
const KEYFRAME_SCHEME = 'keyframe';
24+
const KEYFRAME_HOST = 'capture';
25+
26+
/** 安全 ID 校验:仅允许 UUID/字母数字/下划线/连字符,最长 64(防路径穿越) */
27+
const SAFE_ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
28+
29+
function isSafeId(id: unknown): id is string {
30+
return typeof id === 'string' && SAFE_ID_RE.test(id);
31+
}
32+
33+
/** 关键帧图片根目录:{userData}/captures */
34+
function capturesRoot(): string {
35+
return path.join(app.getPath('userData'), 'captures');
36+
}
37+
38+
/**
39+
* 注册 keyframe:// 为特权 scheme。
40+
* 必须在 app ready 之前调用(main.ts 模块顶层)。
41+
* (Must be called before app ready.)
42+
*/
43+
export function registerKeyframeScheme(): void {
44+
protocol.registerSchemesAsPrivileged([
45+
{
46+
scheme: KEYFRAME_SCHEME,
47+
privileges: { standard: true, secure: true, supportFetchAPI: true, stream: true },
48+
},
49+
]);
50+
}
51+
52+
/**
53+
* 注册 keyframe:// 协议 handler 与 keyframe_save / keyframe_cleanup IPC。
54+
* app ready 后调用一次(main.ts whenReady 中)。
55+
*/
56+
export function registerKeyframeIpcHandlers(): void {
57+
// ---- keyframe:// 协议:映射到本地 captures 目录(只读) ----
58+
if (protocol.isProtocolHandled(KEYFRAME_SCHEME)) {
59+
protocol.unhandle(KEYFRAME_SCHEME);
60+
}
61+
protocol.handle(KEYFRAME_SCHEME, async (request) => {
62+
try {
63+
const url = new URL(request.url);
64+
const [sessionId, fileName] = url.pathname.replace(/^\//, '').split('/');
65+
const keyframeId = fileName?.endsWith('.jpg') ? fileName.slice(0, -4) : undefined;
66+
if (url.hostname !== KEYFRAME_HOST || !isSafeId(sessionId) || !isSafeId(keyframeId)) {
67+
return new Response('Bad Request', { status: 400 });
68+
}
69+
const filePath = path.join(capturesRoot(), sessionId, `${keyframeId}.jpg`);
70+
return await net.fetch(pathToFileURL(filePath).toString());
71+
} catch (err) {
72+
logger.warn(`[Keyframe] Protocol request failed: ${String(err)}`);
73+
return new Response('Not Found', { status: 404 });
74+
}
75+
});
76+
77+
// ---- 保存关键帧 JPEG(返回可渲染 URL)----
78+
safeHandle(
79+
'keyframe_save',
80+
async (_event, args: { sessionId: string; keyframeId: string; imageBase64: string }) => {
81+
const { sessionId, keyframeId, imageBase64 } = args ?? {};
82+
if (!isSafeId(sessionId) || !isSafeId(keyframeId)) {
83+
throw new Error('非法的 sessionId/keyframeId');
84+
}
85+
if (typeof imageBase64 !== 'string' || imageBase64.length === 0) {
86+
throw new Error('imageBase64 不能为空');
87+
}
88+
// 容忍 data URL 前缀(tolerate optional data URL prefix)
89+
const commaIdx = imageBase64.indexOf(',');
90+
const rawBase64 = imageBase64.startsWith('data:') && commaIdx >= 0
91+
? imageBase64.slice(commaIdx + 1)
92+
: imageBase64;
93+
94+
const dir = path.join(capturesRoot(), sessionId);
95+
await mkdir(dir, { recursive: true });
96+
await writeFile(path.join(dir, `${keyframeId}.jpg`), Buffer.from(rawBase64, 'base64'));
97+
return {
98+
success: true,
99+
url: `${KEYFRAME_SCHEME}://${KEYFRAME_HOST}/${sessionId}/${keyframeId}.jpg`,
100+
};
101+
},
102+
);
103+
104+
// ---- 清理会话目录(笔记删除时调用,失败由调用方静默处理)----
105+
safeHandle('keyframe_cleanup', async (_event, args: { sessionId: string }) => {
106+
const sessionId = args?.sessionId;
107+
if (!isSafeId(sessionId)) {
108+
throw new Error('非法的 sessionId');
109+
}
110+
await rm(path.join(capturesRoot(), sessionId), { recursive: true, force: true });
111+
return { success: true };
112+
});
113+
}

‎client/electron/main.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { loadEnvironment } from './envLoader.js';
3535
import { installCspPolicy } from './cspPolicy.js';
3636
import { registerDbIpcHandlers } from './db/dbIpcHandlers.js';
3737
import { registerStorageIpcHandlers } from './storageIpcHandlers.js';
38+
import { registerKeyframeScheme, registerKeyframeIpcHandlers } from './ipc/keyframeStorage.js';
3839

3940
// ================================================================
4041
// 性能优化:启用 GPU 光栅化与零拷贝
@@ -60,6 +61,9 @@ if (process.platform === 'darwin') {
6061
// ================================================================
6162
loadEnvironment(__dirname);
6263

64+
// keyframe:// 特权 scheme 注册(必须在 app ready 前,详见 ipc/keyframeStorage.ts)
65+
registerKeyframeScheme();
66+
6367
// ================================================================
6468
// 模块级状态
6569
// ================================================================
@@ -139,6 +143,8 @@ if (!gotTheLock) {
139143
// 数据访问与存储/备份 IPC(详见 db/dbIpcHandlers.ts、storageIpcHandlers.ts)
140144
registerDbIpcHandlers();
141145
registerStorageIpcHandlers();
146+
// 课堂关键帧图片持久化 IPC + keyframe:// 协议(详见 ipc/keyframeStorage.ts)
147+
registerKeyframeIpcHandlers();
142148

143149
// 隐藏默认 Electron 菜单栏
144150
Menu.setApplicationMenu(null);

‎client/electron/preload.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ const ALLOWED_CHANNELS = [
8686
// AI 流式输出 IPC channel
8787
'ai:stream:start',
8888
'ai:stream:cancel',
89+
// 课堂关键帧图片持久化 IPC channel
90+
'keyframe_save',
91+
'keyframe_cleanup',
8992
] as const;
9093

9194
/** 允许渲染进程监听的事件 channel 白名单(主进程 → 渲染进程推送) */
155 KB
Binary file not shown.
-6.07 KB
Binary file not shown.
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/**
2+
* 课堂音频自动恢复 hook(静音诊断 / 窗口源回退 / 设备变更重启)
3+
*
4+
* @ai-context: 从 useClassroomAudio 拆出的独立恢复层。三条恢复路径:
5+
* ①窗口源静音回退——Windows 上窗口源(window:xxx)getUserMedia 可能"成功但
6+
* 给出纯静音轨"(不抛错,异常降级不会触发),SilenceTracker 检出连续静音后
7+
* 自动 stop→start 以环回源重启(每会话仅回退一次);
8+
* ②输出设备不匹配提示——已在环回源仍持续静音,说明视频声音输出到了
9+
* 非默认设备,提示用户核对默认输出设备;
10+
* ③设备变更重启——默认输出设备切换后环回仍绑定旧设备,devicechange 时
11+
* 自动重启重新绑定。
12+
* @ai-context: 仅依赖 refs 的稳定回调,重启经 restartingRef 互斥防止并发。
13+
*/
14+
import { useEffect, useRef, useCallback } from 'react';
15+
import type { CaptureMode, SessionStatus } from '@/lib/capture';
16+
import {
17+
computeChunkRms, SilenceTracker,
18+
getDefaultOutputDeviceLabel, subscribeDeviceChange,
19+
} from '@/lib/audio/outputDeviceMonitor';
20+
21+
/** 与 useSessionControl 启动参数保持一致的音频采集配置 */
22+
const AUDIO_START_OPTIONS = { chunkDurationMs: 5000, sampleRate: 16000, channels: 1 };
23+
/** 重启前等待渲染端管道清理完成的时间 */
24+
const RESTART_CLEANUP_DELAY_MS = 500;
25+
26+
interface UseAudioRecoveryOptions {
27+
status: SessionStatus;
28+
mode: CaptureMode;
29+
/** 方案A:会话选中窗口的源 ID(回退环回后置空) */
30+
audioSourceId?: string | null;
31+
onNotify: (type: 'warning' | 'error', message: string) => void;
32+
}
33+
34+
export function useAudioRecovery({ status, mode, audioSourceId, onNotify }: UseAudioRecoveryOptions) {
35+
const notifyRef = useRef(onNotify);
36+
notifyRef.current = onNotify;
37+
/** 当前生效的音频源(窗口源回退环回后变为 undefined) */
38+
const effectiveSourceRef = useRef<string | undefined>(audioSourceId ?? undefined);
39+
const silenceTrackerRef = useRef(new SilenceTracker());
40+
/** 本会话是否已执行过窗口源→环回回退 */
41+
const fellBackRef = useRef(false);
42+
const restartingRef = useRef(false);
43+
44+
const audioEnabled = status === 'capturing' && (mode === 'audio' || mode === 'mixed');
45+
46+
// 会话开始时重置恢复状态(audioSourceId 取会话启动瞬间的快照)
47+
useEffect(() => {
48+
if (status !== 'capturing') return;
49+
effectiveSourceRef.current = audioSourceId ?? undefined;
50+
fellBackRef.current = false;
51+
silenceTrackerRef.current.reset();
52+
// eslint-disable-next-line react-hooks/exhaustive-deps -- 仅会话开始时快照 audioSourceId
53+
}, [status]);
54+
55+
/** stop→start 重启音频捕获(互斥),成功返回 true */
56+
const restartCapture = useCallback(async (sourceId?: string): Promise<boolean> => {
57+
if (!window.electronAPI || restartingRef.current) return false;
58+
restartingRef.current = true;
59+
try {
60+
await window.electronAPI.invoke('audio_capture_stop');
61+
await new Promise((r) => setTimeout(r, RESTART_CLEANUP_DELAY_MS));
62+
const result = await window.electronAPI.invoke('audio_capture_start', {
63+
...AUDIO_START_OPTIONS, sourceId,
64+
}) as { success: boolean; error?: string };
65+
if (result.success) silenceTrackerRef.current.reset();
66+
else console.warn('[useAudioRecovery] 音频捕获重启失败:', result.error);
67+
return result.success;
68+
} catch (err) {
69+
console.error('[useAudioRecovery] 音频捕获重启异常:', err);
70+
return false;
71+
} finally {
72+
restartingRef.current = false;
73+
}
74+
}, []);
75+
76+
// 静音诊断:窗口源先自动回退环回;已是环回仍静音则提示核对输出设备
77+
useEffect(() => {
78+
if (!audioEnabled || !window.electronAPI) return;
79+
const off = window.electronAPI.on('audio_capture_chunk', (...args: unknown[]) => {
80+
const chunk = args[0] as { audioBuffer: ArrayBuffer };
81+
if (!silenceTrackerRef.current.push(computeChunkRms(chunk.audioBuffer))) return;
82+
83+
const src = effectiveSourceRef.current;
84+
if (src?.startsWith('window:') && !fellBackRef.current) {
85+
// 方案A 漏洞补偿:窗口源"成功但静音"不抛错,只能靠静音检测触发回退
86+
fellBackRef.current = true;
87+
effectiveSourceRef.current = undefined;
88+
console.warn('[useAudioRecovery] 窗口音频源持续静音,自动回退系统环回');
89+
void restartCapture(undefined).then((ok) => {
90+
if (ok) notifyRef.current('warning', '窗口音频源持续静音,已自动切换为系统环回捕获');
91+
});
92+
return;
93+
}
94+
void getDefaultOutputDeviceLabel().then((label) => {
95+
notifyRef.current('warning',
96+
`持续收到静音音频:请确认视频声音正在播放,且输出到系统默认设备${label ? `「${label}」` : ''}(音频捕获只能录到默认输出设备的声音)`);
97+
});
98+
});
99+
return off;
100+
}, [audioEnabled, restartCapture]);
101+
102+
// 设备变更自动重启:重新绑定新的默认输出设备
103+
useEffect(() => {
104+
if (!audioEnabled || !window.electronAPI) return;
105+
const unsubscribe = subscribeDeviceChange(() => {
106+
console.info('[useAudioRecovery] 检测到音频设备变更,自动重启音频捕获');
107+
void restartCapture(effectiveSourceRef.current).then((ok) => {
108+
if (ok) notifyRef.current('warning', '检测到音频输出设备变更,已自动重新绑定音频捕获');
109+
else notifyRef.current('error', '音频设备变更后重启捕获失败,请停止后重新开始采集');
110+
});
111+
});
112+
return unsubscribe;
113+
}, [audioEnabled, restartCapture]);
114+
}

0 commit comments

Comments
 (0)