Skip to content

Commit 88bfd92

Browse files
committed
feat(classroom): smart 采样双池去重与增量分析页级切分
1 parent 353b042 commit 88bfd92

5 files changed

Lines changed: 279 additions & 20 deletions

File tree

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/**
2+
* analyzePartial 页级切分测试(P1-5)
3+
*
4+
* @ai-context: 验证批内 slide_change 帧作为新页起点逐页独立分析:纯板书批
5+
* 保持单次调用;多页批按页拆分调用、页内 [图:N] 重映射为批内编号(越界
6+
* 幻觉编号移除)。外部 remapKeyframeMarkers 负责批内→全局的二次映射。
7+
* English: page-level splitting of incremental keyframe batches — pure
8+
* board batches stay single-call; multi-page batches split per slide_change
9+
* with local [图:N] remapped to batch-level indices (out-of-range dropped).
10+
*/
11+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
12+
13+
vi.mock('@/lib/auth/supabaseClient', () => ({
14+
supabase: {
15+
auth: {
16+
getSession: vi.fn().mockResolvedValue({ data: { session: { access_token: 'test-token' } } }),
17+
},
18+
},
19+
}));
20+
21+
import { analyzePartial } from './sessionAnalyzer';
22+
import type { KeyFrame } from '@/lib/capture/captureTypes';
23+
24+
function makeKf(id: string, changeType: KeyFrame['changeType'], ts: number): KeyFrame {
25+
return { id, timestamp: ts, imageBase64: 'img', changeType };
26+
}
27+
28+
describe('analyzePartial 页级切分', () => {
29+
const invokeMock = vi.fn();
30+
31+
beforeEach(() => {
32+
invokeMock.mockReset();
33+
(window as unknown as { electronAPI: unknown }).electronAPI = { invoke: invokeMock };
34+
});
35+
36+
afterEach(() => {
37+
delete (window as unknown as { electronAPI?: unknown }).electronAPI;
38+
});
39+
40+
it('纯板书批(无 slide_change):单次调用,编号不重映射', async () => {
41+
invokeMock.mockResolvedValue({ content: '知识点 A\n[图:1]\n知识点 B' });
42+
const kfs = [
43+
makeKf('a', 'writing', 1000),
44+
makeKf('b', 'writing', 2000),
45+
makeKf('c', 'scene_change', 3000),
46+
];
47+
48+
const md = await analyzePartial(kfs, 0);
49+
50+
expect(invokeMock).toHaveBeenCalledTimes(1);
51+
expect(invokeMock.mock.calls[0][0]).toBe('ai_session_analyze');
52+
expect(invokeMock.mock.calls[0][1].keyframes).toHaveLength(3);
53+
expect(md).toBe('知识点 A\n[图:1]\n知识点 B');
54+
});
55+
56+
it('含翻页批:按 slide_change 切页逐页分析,页内编号重映射为批内编号', async () => {
57+
// 批 = [w1, s1, w2, s2] → 页组 [[w1], [s1, w2], [s2]],3 次调用
58+
invokeMock
59+
.mockResolvedValueOnce({ content: '页1内容 [图:1]' })
60+
.mockResolvedValueOnce({ content: '页2内容 [图:1] [图:2]' })
61+
.mockResolvedValueOnce({ content: '页3内容 [图:1]' });
62+
const kfs = [
63+
makeKf('w1', 'writing', 1000),
64+
makeKf('s1', 'slide_change', 2000),
65+
makeKf('w2', 'writing', 3000),
66+
makeKf('s2', 'slide_change', 4000),
67+
];
68+
69+
const md = await analyzePartial(kfs, 0);
70+
71+
expect(invokeMock).toHaveBeenCalledTimes(3);
72+
// 各页组帧数:1 / 2 / 1(页首 slide_change 帧归入新页);kfPayload 仅含
73+
// timestamp/imageBase64/changeType(无 id),按时间戳断言
74+
expect(invokeMock.mock.calls[0][1].keyframes.map((k: { timestamp: number }) => k.timestamp)).toEqual([1]);
75+
expect(invokeMock.mock.calls[1][1].keyframes.map((k: { timestamp: number }) => k.timestamp)).toEqual([2, 3]);
76+
expect(invokeMock.mock.calls[2][1].keyframes.map((k: { timestamp: number }) => k.timestamp)).toEqual([4]);
77+
// 页内编号 → 批内编号:页2 [图:1]→[图:2]、[图:2]→[图:3];页3 [图:1]→[图:4]
78+
expect(md).toBe('页1内容 [图:1]\n\n页2内容 [图:2] [图:3]\n\n页3内容 [图:4]');
79+
});
80+
81+
it('越界页内编号(模型幻觉)被移除', async () => {
82+
invokeMock
83+
.mockResolvedValueOnce({ content: '页1内容 [图:1]' })
84+
.mockResolvedValueOnce({ content: '幻觉编号 [图:9] [图:0]' });
85+
const kfs = [
86+
makeKf('w1', 'writing', 1000),
87+
makeKf('s1', 'slide_change', 2000),
88+
makeKf('w2', 'writing', 3000),
89+
];
90+
91+
const md = await analyzePartial(kfs, 0);
92+
93+
// 越界编号移除后残留空格被 analyzePartial 的 trim() 收敛
94+
expect(md).toBe('页1内容 [图:1]\n\n幻觉编号');
95+
});
96+
97+
it('批首即翻页帧:首页从 slide_change 帧开始', async () => {
98+
invokeMock
99+
.mockResolvedValueOnce({ content: '页A [图:1]' })
100+
.mockResolvedValueOnce({ content: '页B [图:1]' });
101+
const kfs = [
102+
makeKf('s1', 'slide_change', 1000),
103+
makeKf('w1', 'writing', 2000),
104+
makeKf('s2', 'slide_change', 3000),
105+
];
106+
107+
const md = await analyzePartial(kfs, 0);
108+
109+
// 页组 [[s1, w1], [s2]]:页1 offset 0([图:1] 不变),页2 offset 2([图:1]→[图:3])
110+
expect(invokeMock).toHaveBeenCalledTimes(2);
111+
expect(md).toBe('页A [图:1]\n\n页B [图:3]');
112+
});
113+
});

‎client/src/lib/ai/sessionAnalyzer.ts‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,16 +145,58 @@ export async function analyzeSession(
145145
* 小批次关键帧增量分析,返回 Markdown 片段笔记
146146
* 复用现有 ai_session_analyze IPC(小批次走单 chunk 路径,本身就快)
147147
* @param sessionStartMs 会话开始的 epoch 毫秒,用于换算课程内相对秒数
148+
* @ai-context P1-5 页级切分:批内 slide_change 帧作为新页起点逐页独立分析,
149+
* 摘要粒度从"不确定的 5 帧块"细化为"确定的一页笔记";纯板书批(无翻页)
150+
* 保持单次调用不变。页内 [图:N] 在返回前重映射为批内编号,外部
151+
* remapKeyframeMarkers 再映射为全局编号(精确图对齐,无需时间就近兜底)。
148152
*/
149153
export async function analyzePartial(
150154
keyframes: KeyFrame[],
151155
sessionStartMs: number,
152156
options?: { language?: string },
157+
): Promise<string> {
158+
// P1-5 页级切分:slide_change 帧开启新页(页首帧即新页起始)
159+
const pages: KeyFrame[][] = [];
160+
let current: KeyFrame[] = [];
161+
for (const kf of keyframes) {
162+
if (kf.changeType === 'slide_change' && current.length > 0) {
163+
pages.push(current);
164+
current = [];
165+
}
166+
current.push(kf);
167+
}
168+
if (current.length > 0) pages.push(current);
169+
170+
// 纯板书批(无翻页):保持单次调用,行为与拆分前一致
171+
if (pages.length === 1) {
172+
return analyzePage(pages[0], 0, sessionStartMs, options);
173+
}
174+
175+
// 多页批:逐页独立分析,页内编号重映射为批内编号后拼接
176+
const parts: string[] = [];
177+
let batchOffset = 0;
178+
for (const page of pages) {
179+
const md = await analyzePage(page, batchOffset, sessionStartMs, options);
180+
if (md.trim()) parts.push(md.trim());
181+
batchOffset += page.length;
182+
}
183+
return parts.join('\n\n');
184+
}
185+
186+
/**
187+
* 单页组分析:调用网关 partial 分析,页内 [图:N] → 批内编号 [图:offset+N]
188+
* @param pageOffsetInBatch 页组首帧在整批中的 0-based 偏移
189+
*/
190+
async function analyzePage(
191+
page: KeyFrame[],
192+
pageOffsetInBatch: number,
193+
sessionStartMs: number,
194+
options?: { language?: string },
153195
): Promise<string> {
154196
const { data: { session } } = await supabase.auth.getSession();
155197

156198
// @ai-context 时间戳以会话开始时刻为基准换算为相对秒数(relative seconds)
157-
const kfPayload = keyframes.map((kf) => ({
199+
const kfPayload = page.map((kf) => ({
158200
timestamp: Math.max(0, (kf.timestamp - sessionStartMs) / 1000),
159201
imageBase64: kf.imageBase64,
160202
changeType: kf.changeType,
@@ -163,15 +205,27 @@ export async function analyzePartial(
163205
const result = await window.electronAPI!.invoke('ai_session_analyze', {
164206
keyframes: kfPayload,
165207
audioSegments: [],
166-
duration: keyframes.length > 0
167-
? (keyframes[keyframes.length - 1].timestamp - keyframes[0].timestamp) / 1000
208+
duration: page.length > 0
209+
? (page[page.length - 1].timestamp - page[0].timestamp) / 1000
168210
: 0,
169211
mode: 'partial',
170212
language: options?.language,
171213
authToken: session?.access_token,
172214
}) as { content: string };
173215

174-
return result.content;
216+
// 页内局部编号 → 批内编号(外部 remapKeyframeMarkers 继续映射到全局)
217+
return remapLocalToBatch(result.content, pageOffsetInBatch, page.length);
218+
}
219+
220+
/** 页内 [图:N](1-based)→ 批内编号 [图:offset+N];越界编号(模型幻觉)移除 */
221+
// 兼容半角/全角冒号(与 tipTapImageUtils 的 MARKER_RE 对齐,模型可能输出全角)
222+
const LOCAL_MARKER_RE = /\[图[::](\d+)\]/g;
223+
function remapLocalToBatch(markdown: string, offsetInBatch: number, pageSize: number): string {
224+
return markdown.replace(LOCAL_MARKER_RE, (_m, n: string) => {
225+
const local = parseInt(n, 10);
226+
if (local < 1 || local > pageSize) return '';
227+
return `[图:${offsetInBatch + local}]`;
228+
});
175229
}
176230

177231
// ================================================================

‎client/src/lib/capture/smartSampler.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,4 +218,63 @@ describe('SmartSampler 感知哈希去重', () => {
218218
expect(f2).not.toBeNull();
219219
expect(sampler.getKeyframes()).toHaveLength(2);
220220
});
221+
222+
it('方案 C 双通道独立去重:幻灯片帧与板书帧互不干扰', async () => {
223+
const sampler = new SmartSampler();
224+
// slide 帧(score 0.7)入 slide 池
225+
const f1 = await sampler.processFrame(makeFrame(1, 0.7));
226+
expect(f1?.changeType).toBe('slide_change');
227+
228+
// 相同内容的 scene 帧:board 池为空 → 不受 slide 池影响,正常捕获
229+
const f2 = await sampler.processFrame(makeFrame(1, 0.5));
230+
expect(f2).not.toBeNull();
231+
expect(f2!.changeType).toBe('scene_change');
232+
expect(sampler.getKeyframes()).toHaveLength(2);
233+
234+
// 再次 slide 帧:slide 池命中(距离 0 ≤ 8)→ 跳过
235+
const f3 = await sampler.processFrame(makeFrame(1, 0.7));
236+
expect(f3).toBeNull();
237+
expect(sampler.getKeyframes()).toHaveLength(2);
238+
});
239+
240+
it('方案 C 幻灯片帧放宽阈值:距离 3 ≤ 8 判重跳过(同页动画容忍)', async () => {
241+
const sampler = new SmartSampler();
242+
await sampler.processFrame(makeFrame(1, 0.7));
243+
// seed3 与 seed1 距离 3:slide 阈值 8 判重(旧实现阈值 5 也判重,此处验证通道路由)
244+
const f2 = await sampler.processFrame(makeFrame(3, 0.7));
245+
expect(f2).toBeNull();
246+
expect(sampler.getKeyframes()).toHaveLength(1);
247+
});
248+
249+
it('方案 C 哈希池收集全部通道帧:距最旧帧近但距新帧远时不误判', async () => {
250+
const nowSpy = vi.spyOn(Date, 'now');
251+
const sampler = new SmartSampler();
252+
nowSpy.mockReturnValue(1_000_000);
253+
// 帧 1(seed1)入 board 池
254+
await sampler.processFrame(makeFrame(1, 0.5));
255+
// 帧 2(seed2,距离 64)入池
256+
nowSpy.mockReturnValue(1_000_100);
257+
await sampler.processFrame(makeFrame(2, 0.5));
258+
// 帧 3(seed1 变体 seed3,距离 seed1=3、seed2=61):与池中任一帧距离 3 > 阈值 2?
259+
// scene 帧阈值 5:3 ≤ 5 → 判重跳过(若仅对比上一帧 seed2 则距离 61 会被误采)
260+
nowSpy.mockReturnValue(1_000_200);
261+
const f3 = await sampler.processFrame(makeFrame(3, 0.5));
262+
expect(f3).toBeNull();
263+
expect(sampler.getKeyframes()).toHaveLength(2);
264+
});
265+
266+
it('方案 C 跨池去重:静态 PPT 页 15s 兜底(periodic)不再重复捕获', async () => {
267+
const nowSpy = vi.spyOn(Date, 'now');
268+
const sampler = new SmartSampler();
269+
nowSpy.mockReturnValue(1_000_000);
270+
// slide 帧(score 0.7)入 slide 池
271+
const f1 = await sampler.processFrame(makeFrame(1, 0.7));
272+
expect(f1?.changeType).toBe('slide_change');
273+
274+
// 16s 后同内容静止帧:periodic 兜底触发,跨池命中 slide 池 → 跳过(双池回归护栏)
275+
nowSpy.mockReturnValue(1_016_000);
276+
const f2 = await sampler.processFrame(makeFrame(1, 0, false));
277+
expect(f2).toBeNull();
278+
expect(sampler.getKeyframes()).toHaveLength(1);
279+
});
221280
});

‎client/src/lib/capture/smartSampler.ts‎

Lines changed: 46 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*/
1313

1414
import type { ScreenshotData, KeyFrame } from './captureTypes';
15-
import { computeFrameHash, hammingDistance, isSimilar } from './frameHash';
15+
import { computeFrameHash, hammingDistance } from './frameHash';
1616

1717
// ================================================================
1818
// 配置类型
@@ -36,10 +36,14 @@ const DEFAULT_CONFIG: SmartSamplerConfig = {
3636
maxWidth: 1280,
3737
};
3838

39-
/** 感知哈希去重阈值:与上一关键帧汉明距离 ≤ 5(64 位)视为重复帧,跳过 */
39+
/** 感知哈希去重阈值:与所属通道哈希池任一帧汉明距离 ≤ 阈值视为重复帧,跳过 */
4040
const HASH_DUP_THRESHOLD = 5;
4141
/** 渐进板书帧(变化触发且 0 < score < 0.3)收紧阈值:距离 ≤ 2 才跳过,避免漏采渐进内容 */
4242
const WRITING_HASH_DUP_THRESHOLD = 2;
43+
/** PPT 幻灯片帧(score ≥ 0.6)放宽阈值:距离 ≤ 8 判重,容忍同页动画/光标差异(同页多帧不再重复捕获) */
44+
const SLIDE_HASH_DUP_THRESHOLD = 8;
45+
/** 哈希池容量上限:各通道保留最近 N 帧哈希(FIFO),防止长课堂无界增长 */
46+
const MAX_HASH_POOL_SIZE = 50;
4347
/**
4448
* 内存上限:仅最近 N 个关键帧保留 imageBase64,更早的剥离为空串。
4549
* 单帧 base64 可达数百 KB,长时间采集(1h 约 240 帧)若无界持有会
@@ -56,8 +60,13 @@ export class SmartSampler {
5660
private readonly config: SmartSamplerConfig;
5761
private keyframes: KeyFrame[] = [];
5862
private lastCaptureTime = 0;
59-
/** 上一已捕获关键帧的感知哈希,用于帧间内容去重 */
60-
private lastFrameHash: bigint | null = null;
63+
/**
64+
* 方案 C 双通道哈希池:slide 帧(score ≥ 0.6)与板书/场景帧分池去重,
65+
* 互不干扰——PPT 动画不再误触发板书通道,板书渐进内容也不被 PPT 帧吞掉。
66+
* 各池 FIFO 保留最近 MAX_HASH_POOL_SIZE 帧哈希(对比旧实现的"仅上一帧")。
67+
*/
68+
private slideHashPool: bigint[] = [];
69+
private boardHashPool: bigint[] = [];
6170
/**
6271
* P1-7 强制补帧标志:指令句命中后置真,下一帧跳过变化检测门槛
6372
* 直接进入捕获判定(感知哈希去重仍生效,防止静止画面重复捕获)。
@@ -131,18 +140,32 @@ export class SmartSampler {
131140
new Blob([frameData.imageBuffer], { type: 'image/png' }),
132141
);
133142

134-
// 感知哈希去重:与上一已捕获关键帧比较,定时兜底触发同样走此去重
143+
// 感知哈希去重:与所属通道(slide/board)哈希池中所有帧比较,任一距离 ≤ 阈值视为重复
135144
const hash = await computeFrameHash(bitmap);
136-
if (hash !== null && this.lastFrameHash !== null) {
137-
// 渐进板书帧收紧跳过阈值,避免漏采渐进内容
145+
if (hash !== null) {
146+
const isSlideFrame = changeType === 'slide_change';
147+
const pool = isSlideFrame ? this.slideHashPool : this.boardHashPool;
148+
// 兜底触发帧(periodic)不信任单一通道:静态 PPT 页 15s 兜底时被分类为
149+
// periodic(score≈0),若只比对 board 池会重复捕获同一页(双池回归)——两池都查
150+
const candidates = changeType === 'periodic'
151+
? [...this.boardHashPool, ...this.slideHashPool]
152+
: pool;
153+
// 渐进板书帧收紧跳过阈值,避免漏采渐进内容;幻灯片帧放宽阈值容忍同页动画
138154
const isWritingChange = hasSignificantChange && changeType === 'writing';
139-
const dupThreshold = isWritingChange
140-
? WRITING_HASH_DUP_THRESHOLD
141-
: HASH_DUP_THRESHOLD;
142-
if (isSimilar(hash, this.lastFrameHash, dupThreshold)) {
155+
const dupThreshold = isSlideFrame
156+
? SLIDE_HASH_DUP_THRESHOLD
157+
: isWritingChange
158+
? WRITING_HASH_DUP_THRESHOLD
159+
: HASH_DUP_THRESHOLD;
160+
let minDistance = Infinity;
161+
for (const h of candidates) {
162+
const d = hammingDistance(hash, h);
163+
if (d < minDistance) minDistance = d;
164+
}
165+
if (minDistance <= dupThreshold) {
143166
console.debug(
144167
'[SmartSampler] 跳过帧:感知哈希重复',
145-
`distance=${hammingDistance(hash, this.lastFrameHash)}`,
168+
`distance=${minDistance}`,
146169
`dupThreshold=${dupThreshold}`,
147170
`writing=${isWritingChange}`,
148171
);
@@ -153,7 +176,7 @@ export class SmartSampler {
153176
}
154177
console.debug(
155178
'[SmartSampler] 感知哈希判定为新内容',
156-
`distance=${hammingDistance(hash, this.lastFrameHash)}`,
179+
`distance=${minDistance}`,
157180
);
158181
}
159182

@@ -169,7 +192,14 @@ export class SmartSampler {
169192

170193
this.keyframes.push(keyframe);
171194
this.lastCaptureTime = now;
172-
if (hash !== null) this.lastFrameHash = hash;
195+
// 捕获成功 → 哈希入所属通道池(FIFO 上限裁剪,防长课堂无界增长)
196+
if (hash !== null) {
197+
const pool = changeType === 'slide_change' ? this.slideHashPool : this.boardHashPool;
198+
pool.push(hash);
199+
if (pool.length > MAX_HASH_POOL_SIZE) {
200+
pool.splice(0, pool.length - MAX_HASH_POOL_SIZE);
201+
}
202+
}
173203

174204
// 内存上限:剥离过早关键帧的 base64(已落盘),避免长时间采集内存无界增长
175205
this.trimOldKeyframeImages();
@@ -186,7 +216,8 @@ export class SmartSampler {
186216
reset(): void {
187217
this.keyframes = [];
188218
this.lastCaptureTime = 0;
189-
this.lastFrameHash = null;
219+
this.slideHashPool = [];
220+
this.boardHashPool = [];
190221
this.forceCapturePending = false;
191222
}
192223

‎server/ai-gateway/prompts/session_analyze.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,9 @@ def build_partial_prompt(keyframes_count: int) -> str:
188188
"1. **只保留一个课程概述**:若多个片段各自含有课程概述/课程名称/讲师等全局信息,"
189189
"合并为开头唯一的一段概述,其余全部删除\n"
190190
"2. **严禁重复知识点**:相邻片段的重叠内容必须去重,同一知识点只保留一处最完整的表述\n"
191-
"3. **统一结构**:使用 Markdown 二级标题(##)按知识模块重新组织,标题层级保持一致\n"
191+
"3. **章节组织**:按知识点/教学主题将笔记划分为若干逻辑章节,每章使用 Markdown 二级标题"
192+
"(## 章节名)命名,并在每章标题下一行用斜体给出该章一句话概述(*概述:…*);"
193+
"标题层级保持一致\n"
192194
"4. **统一时间标注**:所有时间标注统一为 MM:SS 格式\n"
193195
"5. **保留细节**:不要丢失任何公式、定义、代码或关键术语\n"
194196
"6. **唯一末尾摘要**:只在笔记最后输出一个「核心知识点摘要」部分(3-5 个知识点),"

0 commit comments

Comments
 (0)