Skip to content

Commit 8733c87

Browse files
committed
feat(classroom): 增强课堂会话分析管线(关键帧存储/帧去重/多模态优化)
1 parent 6dd1306 commit 8733c87

48 files changed

Lines changed: 2283 additions & 133 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎client/electron/ai/handlers/sessionAnalyzeHandler.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ function register(): void {
3737
audioText: string | null;
3838
}>;
3939
duration: number; // 已转为秒
40+
/** 分析模式:full 全量 / partial 增量片段(网关据此选择 Prompt 模板) */
41+
mode?: 'full' | 'partial';
4042
language?: string;
4143
authToken?: string;
4244
userApiKey?: string;
@@ -45,7 +47,7 @@ function register(): void {
4547
const startMs = Date.now();
4648
const kfCount = args.keyframes?.length ?? 0;
4749
const segCount = args.audioSegments?.length ?? 0;
48-
logger.info(`[AI] [session-analyze] IPC received: keyframes=${kfCount}, audioSegments=${segCount}, duration=${args.duration}s, language=${args.language ?? 'zh'}, hasAuth=${!!args.authToken}`);
50+
logger.info(`[AI] [session-analyze] IPC received: keyframes=${kfCount}, audioSegments=${segCount}, duration=${args.duration}s, mode=${args.mode ?? 'full'}, language=${args.language ?? 'zh'}, hasAuth=${!!args.authToken}`);
4951

5052
const reqBody = {
5153
keyframes: args.keyframes.map(kf => ({
@@ -59,6 +61,7 @@ function register(): void {
5961
audio_text: seg.audioText,
6062
})),
6163
duration: args.duration,
64+
mode: args.mode ?? 'full',
6265
language: args.language ?? 'zh',
6366
};
6467

‎client/electron/cspPolicy.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,10 @@ export function installCspPolicy(isDev: boolean): void {
4343
const extraConnectSrc = buildExtraConnectSrc();
4444
// 开发环境:允许 unsafe-inline/unsafe-eval(Vite HMR 需要),worker-src 需要 blob:
4545
// 生产环境:禁止 unsafe-eval,保留 unsafe-inline(Tailwind 运行时需要)
46+
// img-src 放行 keyframe:(课堂关键帧本地图片自定义协议,见 ipc/keyframeStorage.ts)
4647
const csp = isDev
47-
? `default-src 'self' 'unsafe-inline' 'unsafe-eval' http://localhost:* ws://localhost:*; worker-src 'self' blob: http://localhost:*; connect-src 'self' http://localhost:* ws://localhost:* https://*.supabase.co wss://*.supabase.co https://entropydecrease.com wss://entropydecrease.com${extraConnectSrc}; img-src 'self' data: blob: https://*.supabase.co; font-src 'self' data:;`
48-
: `default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://*.supabase.co; font-src 'self' data:; connect-src 'self' https://*.supabase.co wss://*.supabase.co https://entropydecrease.com wss://entropydecrease.com${extraConnectSrc}; frame-ancestors 'none';`;
48+
? `default-src 'self' 'unsafe-inline' 'unsafe-eval' http://localhost:* ws://localhost:*; worker-src 'self' blob: http://localhost:*; connect-src 'self' http://localhost:* ws://localhost:* https://*.supabase.co wss://*.supabase.co https://entropydecrease.com wss://entropydecrease.com${extraConnectSrc}; img-src 'self' data: blob: keyframe: https://*.supabase.co; font-src 'self' data:;`
49+
: `default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: keyframe: https://*.supabase.co; font-src 'self' data:; connect-src 'self' https://*.supabase.co wss://*.supabase.co https://entropydecrease.com wss://entropydecrease.com${extraConnectSrc}; frame-ancestors 'none';`;
4950

5051
callback({
5152
responseHeaders: {

‎client/electron/ipc/channels.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ export const IPC_CHANNELS = {
5656
// 文件系统
5757
FS_READ_FILE: 'fs:read-file',
5858

59+
// 课堂关键帧图片持久化
60+
KEYFRAME_SAVE: 'keyframe_save',
61+
KEYFRAME_CLEANUP: 'keyframe_cleanup',
62+
5963
// 存储路径
6064
STORAGE_CHANGE_PATH: 'storage:change-path',
6165
STORAGE_GET_ACTIVE_PATH: 'storage:get-active-path',
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/**
2+
* 课堂关键帧图片持久化 IPC + keyframe:// 自定义协议
3+
* Classroom keyframe image persistence IPC handlers and custom
4+
* keyframe:// protocol for renderer-side image loading.
5+
*
6+
* @ai-context: 关键帧 JPEG 本地落盘到 {userData}/captures/{sessionId}/{keyframeId}.jpg,
7+
* 遵守本地优先原则(图片不上传)。渲染进程通过自定义 keyframe:// 协议加载:
8+
* 开发页面源为 http://localhost、生产为 file://,file:// 子资源会被
9+
* webSecurity/CSP 拦截,故注册 standard+secure 专用协议(CSP img-src 已放行 keyframe:)。
10+
* (Custom protocol is used because file:// subresources are blocked by
11+
* webSecurity/CSP under both dev http origin and prod file origin.)
12+
* @ai-context: sessionId/keyframeId 仅允许 UUID/安全字符(防路径穿越);
13+
* keyframe_cleanup 删除整个会话目录,由笔记删除路径调用(失败静默)。
14+
*/
15+
import { app, net, protocol } from 'electron';
16+
import * as path from 'path';
17+
import { mkdir, rm, writeFile } from 'fs/promises';
18+
import { pathToFileURL } from 'url';
19+
import { safeHandle } from '../ipcUtils.js';
20+
import { logger } from '../logger.js';
21+
22+
/** 自定义协议名与固定 host(keyframe://capture/{sessionId}/{keyframeId}.jpg) */
23+
const KEYFRAME_SCHEME = 'keyframe';
24+
const KEYFRAME_HOST = 'capture';
25+
26+
/** 安全 ID 校验:仅允许 UUID/字母数字/下划线/连字符,最长 64(防路径穿越) */
27+
const SAFE_ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
28+
29+
function isSafeId(id: unknown): id is string {
30+
return typeof id === 'string' && SAFE_ID_RE.test(id);
31+
}
32+
33+
/** 关键帧图片根目录:{userData}/captures */
34+
function capturesRoot(): string {
35+
return path.join(app.getPath('userData'), 'captures');
36+
}
37+
38+
/**
39+
* 注册 keyframe:// 为特权 scheme。
40+
* 必须在 app ready 之前调用(main.ts 模块顶层)。
41+
* (Must be called before app ready.)
42+
*/
43+
export function registerKeyframeScheme(): void {
44+
protocol.registerSchemesAsPrivileged([
45+
{
46+
scheme: KEYFRAME_SCHEME,
47+
privileges: { standard: true, secure: true, supportFetchAPI: true, stream: true },
48+
},
49+
]);
50+
}
51+
52+
/**
53+
* 注册 keyframe:// 协议 handler 与 keyframe_save / keyframe_cleanup IPC。
54+
* app ready 后调用一次(main.ts whenReady 中)。
55+
*/
56+
export function registerKeyframeIpcHandlers(): void {
57+
// ---- keyframe:// 协议:映射到本地 captures 目录(只读) ----
58+
if (protocol.isProtocolHandled(KEYFRAME_SCHEME)) {
59+
protocol.unhandle(KEYFRAME_SCHEME);
60+
}
61+
protocol.handle(KEYFRAME_SCHEME, async (request) => {
62+
try {
63+
const url = new URL(request.url);
64+
const [sessionId, fileName] = url.pathname.replace(/^\//, '').split('/');
65+
const keyframeId = fileName?.endsWith('.jpg') ? fileName.slice(0, -4) : undefined;
66+
if (url.hostname !== KEYFRAME_HOST || !isSafeId(sessionId) || !isSafeId(keyframeId)) {
67+
return new Response('Bad Request', { status: 400 });
68+
}
69+
const filePath = path.join(capturesRoot(), sessionId, `${keyframeId}.jpg`);
70+
return await net.fetch(pathToFileURL(filePath).toString());
71+
} catch (err) {
72+
logger.warn(`[Keyframe] Protocol request failed: ${String(err)}`);
73+
return new Response('Not Found', { status: 404 });
74+
}
75+
});
76+
77+
// ---- 保存关键帧 JPEG(返回可渲染 URL)----
78+
safeHandle(
79+
'keyframe_save',
80+
async (_event, args: { sessionId: string; keyframeId: string; imageBase64: string }) => {
81+
const { sessionId, keyframeId, imageBase64 } = args ?? {};
82+
if (!isSafeId(sessionId) || !isSafeId(keyframeId)) {
83+
throw new Error('非法的 sessionId/keyframeId');
84+
}
85+
if (typeof imageBase64 !== 'string' || imageBase64.length === 0) {
86+
throw new Error('imageBase64 不能为空');
87+
}
88+
// 容忍 data URL 前缀(tolerate optional data URL prefix)
89+
const commaIdx = imageBase64.indexOf(',');
90+
const rawBase64 = imageBase64.startsWith('data:') && commaIdx >= 0
91+
? imageBase64.slice(commaIdx + 1)
92+
: imageBase64;
93+
94+
const dir = path.join(capturesRoot(), sessionId);
95+
await mkdir(dir, { recursive: true });
96+
await writeFile(path.join(dir, `${keyframeId}.jpg`), Buffer.from(rawBase64, 'base64'));
97+
return {
98+
success: true,
99+
url: `${KEYFRAME_SCHEME}://${KEYFRAME_HOST}/${sessionId}/${keyframeId}.jpg`,
100+
};
101+
},
102+
);
103+
104+
// ---- 清理会话目录(笔记删除时调用,失败由调用方静默处理)----
105+
safeHandle('keyframe_cleanup', async (_event, args: { sessionId: string }) => {
106+
const sessionId = args?.sessionId;
107+
if (!isSafeId(sessionId)) {
108+
throw new Error('非法的 sessionId');
109+
}
110+
await rm(path.join(capturesRoot(), sessionId), { recursive: true, force: true });
111+
return { success: true };
112+
});
113+
}

‎client/electron/main.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { loadEnvironment } from './envLoader.js';
3535
import { installCspPolicy } from './cspPolicy.js';
3636
import { registerDbIpcHandlers } from './db/dbIpcHandlers.js';
3737
import { registerStorageIpcHandlers } from './storageIpcHandlers.js';
38+
import { registerKeyframeScheme, registerKeyframeIpcHandlers } from './ipc/keyframeStorage.js';
3839

3940
// ================================================================
4041
// 性能优化:启用 GPU 光栅化与零拷贝
@@ -60,6 +61,9 @@ if (process.platform === 'darwin') {
6061
// ================================================================
6162
loadEnvironment(__dirname);
6263

64+
// keyframe:// 特权 scheme 注册(必须在 app ready 前,详见 ipc/keyframeStorage.ts)
65+
registerKeyframeScheme();
66+
6367
// ================================================================
6468
// 模块级状态
6569
// ================================================================
@@ -139,6 +143,8 @@ if (!gotTheLock) {
139143
// 数据访问与存储/备份 IPC(详见 db/dbIpcHandlers.ts、storageIpcHandlers.ts)
140144
registerDbIpcHandlers();
141145
registerStorageIpcHandlers();
146+
// 课堂关键帧图片持久化 IPC + keyframe:// 协议(详见 ipc/keyframeStorage.ts)
147+
registerKeyframeIpcHandlers();
142148

143149
// 隐藏默认 Electron 菜单栏
144150
Menu.setApplicationMenu(null);

‎client/electron/preload.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ const ALLOWED_CHANNELS = [
8686
// AI 流式输出 IPC channel
8787
'ai:stream:start',
8888
'ai:stream:cancel',
89+
// 课堂关键帧图片持久化 IPC channel
90+
'keyframe_save',
91+
'keyframe_cleanup',
8992
] as const;
9093

9194
/** 允许渲染进程监听的事件 channel 白名单(主进程 → 渲染进程推送) */
155 KB
Binary file not shown.
-6.07 KB
Binary file not shown.

‎client/src/features/classroom/hooks/useClassroomAnalysis.ts‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,13 @@ interface UseClassroomAnalysisOptions {
3232
setSmartBundle: React.Dispatch<React.SetStateAction<Partial<SessionBundle>>>;
3333
videoFilePath: string | null;
3434
recordingStatus: RecordingStatus | null;
35+
/** 真实采集会话 ID(关联笔记与关键帧图片目录),缺省时持久化处退化为随机 UUID */
36+
captureSessionIdRef?: React.MutableRefObject<string | null>;
3537
onWarn: (message: string) => void;
3638
}
3739

3840
export function useClassroomAnalysis({
39-
language, smartBundle, setSmartBundle, videoFilePath, recordingStatus, onWarn,
41+
language, smartBundle, setSmartBundle, videoFilePath, recordingStatus, captureSessionIdRef, onWarn,
4042
}: UseClassroomAnalysisOptions) {
4143
const [isAnalyzing, setIsAnalyzing] = useState(false);
4244
const [analysisResult, setAnalysisResult] = useState<AnalyzeResult | null>(null);
@@ -55,7 +57,10 @@ export function useClassroomAnalysis({
5557
timeline: smartBundle.timeline ?? [],
5658
duration: smartBundle.duration ?? 0,
5759
};
58-
const result = await analyzeSession(fullBundle, { language });
60+
const result = await analyzeSession(fullBundle, {
61+
language,
62+
sessionId: captureSessionIdRef?.current ?? undefined,
63+
});
5964
setAnalysisResult(result);
6065
// 全量分析完成,释放所有 keyframe imageBase64 内存
6166
setSmartBundle((prev) => ({
@@ -67,7 +72,7 @@ export function useClassroomAnalysis({
6772
} finally {
6873
setIsAnalyzing(false);
6974
}
70-
}, [smartBundle, language, setSmartBundle]);
75+
}, [smartBundle, language, setSmartBundle, captureSessionIdRef]);
7176

7277
/** Path C:视频分析 */
7378
const handleVideoAnalyze = useCallback(async (filePath?: string) => {
@@ -97,20 +102,24 @@ export function useClassroomAnalysis({
97102
const result = await mergeNotes(partials, {
98103
duration: durationMs / 1000,
99104
language,
105+
sessionId: captureSessionIdRef?.current ?? undefined,
100106
});
101107
setAnalysisResult(result);
102108
} catch {
103109
// 降级:本地拼接片段笔记(无需 AI,零网络,避免全量重发)
110+
// 为每个片段插入分隔标题,避免拼接后内容边界不清
104111
setAnalysisResult({
105-
content: partials.join('\n\n---\n\n'),
112+
content: partials
113+
.map((p, idx) => `## 片段 ${idx + 1}\n\n${p.trim()}`)
114+
.join('\n\n---\n\n'),
106115
keyframesAnalyzed: keyframeCount,
107116
modelUsed: 'local-concat',
108117
});
109118
onWarn('AI 合并不可用,已直接拼接片段笔记');
110119
} finally {
111120
setIsAnalyzing(false);
112121
}
113-
}, [language, onWarn]);
122+
}, [language, onWarn, captureSessionIdRef]);
114123

115124
const handleDismissAnalysis = useCallback(() => {
116125
setAnalysisResult(null);

‎client/src/features/classroom/hooks/useClassroomCapture.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,11 @@ export function useClassroomCapture() {
110110
setSmartBundle: events.setSmartBundle,
111111
videoFilePath: events.videoFilePath,
112112
recordingStatus: events.recordingStatus,
113+
captureSessionIdRef: events.captureSessionIdRef,
113114
onWarn: (message) => notify('warning', message),
114115
});
115116

116-
const notes = useClassroomNotes(courseMeta);
117+
const notes = useClassroomNotes(courseMeta, events.smartBundle);
117118

118119
/** 开始采集前重置本轮会话数据 */
119120
const resetForStart = useCallback(() => {

0 commit comments

Comments
 (0)