Skip to content

Commit 9a52e84

Browse files
committed
feat(classroom): P1-2 动态热词闭环 + P1-3 修正回写 + P1-8 漏捕检测/手动补截
P1-2: 视觉提取concepts术语注入会话热词(extraction:completed)+ 流式热词IPC同步 P1-3: 编辑修正差异提取(公共前缀/后缀剥离)→ 会话内立即生效 + 词库去重持久化(课程维度绑定) P1-8: 指令句后3s无新帧漏捕提示 + C键手动补截快捷键
1 parent 45caf98 commit 9a52e84

6 files changed

Lines changed: 190 additions & 9 deletions

File tree

‎client/src/features/classroom/hooks/useClassroomCapture.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ export function useClassroomCapture() {
147147
onNotify: notify,
148148
streamingAsrActive,
149149
windowTitle: selectedWindow?.title,
150+
courseName: courseMeta.courseName,
150151
});
151152

152153
// 转写/提取段最新值 ref 桥:停止收尾(useSessionControl)在事件回调
@@ -321,6 +322,13 @@ export function useClassroomCapture() {
321322
notify('success', `已标记重点 (${new Date(now).toLocaleTimeString()})`);
322323
}, [status, notify, captureManager, setSmartBundle]);
323324

325+
// ── P1-8 手动补截:快捷键触发强制补帧(复用 P1-7 机制,下一帧跳过变化检测门槛)──
326+
const handleManualCapture = useCallback(() => {
327+
if (status !== 'capturing') return;
328+
captureManager.requestForceCapture();
329+
notify('info', '已手动捕捉当前画面');
330+
}, [status, notify, captureManager]);
331+
324332
return {
325333
// 窗口
326334
...windowWatcher,
@@ -362,6 +370,8 @@ export function useClassroomCapture() {
362370
handleRetryMerge: analysis.handleRetryMerge,
363371
handleDismissAnalysis: analysis.handleDismissAnalysis,
364372
handleGenerateCards, handleBookmark, bookmarks,
373+
// P1-8 手动补截(漏捕检测提示的补救操作)
374+
handleManualCapture,
365375
// M2 自动锚点(时间线展示)
366376
autoAnchors,
367377
// 笔记持久化

‎client/src/features/classroom/hooks/useClassroomEvents.ts‎

Lines changed: 63 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,11 @@ import { detectCourseFromFrame } from '@/lib/ai/courseDetector';
3333
import { remapKeyframeMarkers } from '../utils/tipTapImageUtils';
3434
import { persistKeyframeImage } from '../utils/keyframePersistence';
3535
import { transcribeWithRetry, toAsrLanguage, useAsrSemaphore, isLocalAsrReady, setOnAsrFallback } from '../utils/asrTranscriber';
36-
import { applySessionReplaces, getSessionHotwordsString, addDynamicBoosts } from '../utils/hotwordRuntime';
36+
import { applySessionReplaces, getSessionHotwordsString, addDynamicBoosts, addSessionReplace } from '../utils/hotwordRuntime';
3737
import { cleanAsrResult, dedupeAcrossFinals, estimateAsrConfidence } from '@/lib/capture/asrFilters';
3838
import { classifyContent, hasCommandCue, type ContentKind } from '@/lib/capture/contentClassifier';
39+
import { hotwordStore } from '@/lib/storage/hotwordStore';
40+
import { extractCorrection } from '../utils/transcriptCorrection';
3941

4042
/** 触发一次增量分析所需的关键帧数 */
4143
const INCREMENTAL_BATCH_SIZE = 5;
@@ -67,6 +69,8 @@ interface UseClassroomEventsOptions {
6769
streamingAsrActive: boolean;
6870
/** P1-6:目标窗口标题(内容类型分类的标题信号) */
6971
windowTitle?: string;
72+
/** P1-3:当前课程名(修正回写词库的 courseId 绑定) */
73+
courseName?: string;
7074
}
7175

7276
/** P1-6 分类尝试上限(转写样本仍无信号则停止,unknown 不再重试) */
@@ -75,10 +79,12 @@ const MAX_CLASSIFY_ATTEMPTS = 5;
7579
const FORCE_CAPTURE_MIN_INTERVAL_MS = 1000;
7680
/** P1-6 分类转写样本上限(字符,FIFO 追加防无界) */
7781
const MAX_CLASSIFY_SAMPLE_CHARS = 500;
82+
/** P1-8 漏捕检测延迟:指令句后该时长内无新关键帧 → 提示(ms) */
83+
const MISSED_CAPTURE_CHECK_MS = 3000;
7884

7985
export function useClassroomEvents({
8086
captureManager, status, capturePath, language, aiDetectEnabled, setCourseMeta, onNotify,
81-
streamingAsrActive, windowTitle,
87+
streamingAsrActive, windowTitle, courseName,
8288
}: UseClassroomEventsOptions) {
8389
const [segments, setSegments] = useState<ExtractedSegment[]>([]);
8490
const [stats, setStats] = useState({ frames: 0, extracted: 0 });
@@ -105,6 +111,8 @@ export function useClassroomEvents({
105111
const transcriptSampleRef = useRef('');
106112
/** P1-7 指令补帧节流时间戳 */
107113
const lastForceCaptureAtRef = useRef(0);
114+
/** P1-8 最近一次关键帧时间戳(漏捕检测依据:指令后无新帧即提示) */
115+
const latestKeyframeTsRef = useRef(0);
108116
/** @ai-context 会话时间基准(epoch ms):记录首帧 timestamp,供 analyzePartial 换算相对秒数 */
109117
const sessionStartMsRef = useRef<number | null>(null);
110118
/** 采集会话 ID(smart:keyframe 事件携带),供笔记持久化关联与关键帧图片清理 */
@@ -148,13 +156,20 @@ export function useClassroomEvents({
148156
console.info(`[useClassroomCapture] 内容类型识别: ${result.kind}(${result.source})`);
149157
}
150158
}
151-
// 指令句补帧(技能场景操作瞬间捕捉)
159+
// 指令句补帧(技能场景操作瞬间捕捉)+ 漏捕检测
152160
if (hasCommandCue(text)) {
153161
const now = Date.now();
154162
if (now - lastForceCaptureAtRef.current >= FORCE_CAPTURE_MIN_INTERVAL_MS) {
155163
lastForceCaptureAtRef.current = now;
156164
captureManager.requestForceCapture();
157165
}
166+
// P1-8 漏捕检测:指令后 3s 无新关键帧 → 提示手动补截
167+
const commandAt = now;
168+
window.setTimeout(() => {
169+
if (latestKeyframeTsRef.current < commandAt) {
170+
toast({ type: 'warning', silent: true, message: '这一步的画面可能没捕捉到,可按 C 键手动截图' });
171+
}
172+
}, MISSED_CAPTURE_CHECK_MS);
158173
}
159174
};
160175

@@ -176,7 +191,7 @@ export function useClassroomEvents({
176191
useEffect(() => {
177192
const offCompleted = captureEventBus.on<{
178193
sessionId: string | null;
179-
result: { text: string; confidence: number; source: 'vision' | 'audio' | 'ui_automation' };
194+
result: { text: string; confidence: number; source: 'vision' | 'audio' | 'ui_automation'; structured?: Record<string, unknown> };
180195
segment: { id: string; timestamp: Date };
181196
extractedCount: number;
182197
}>('extraction:completed', (data) => {
@@ -190,6 +205,20 @@ export function useClassroomEvents({
190205
setSegments((prev) => [...prev, uiSegment]);
191206
setStats((prev) => ({ ...prev, extracted: data.extractedCount }));
192207
setExtractionError(null);
208+
209+
// P1-2 动态热词闭环:视觉提取的概念术语注入会话热词(弱 boost,
210+
// 不持久化);流式运行中同步更新主进程会话热词(下一断句生效)
211+
if (data.result.source === 'vision') {
212+
const concepts = data.result.structured?.concepts;
213+
if (Array.isArray(concepts)) {
214+
const terms = concepts.filter((c): c is string => typeof c === 'string');
215+
if (terms.length > 0) {
216+
addDynamicBoosts(terms);
217+
window.electronAPI?.local_asr_stream_set_hotwords({ hotwords: getSessionHotwordsString() })
218+
.catch(() => { /* 静默 */ });
219+
}
220+
}
221+
}
193222
});
194223

195224
const offError = captureEventBus.on<{ message: string }>(
@@ -209,6 +238,8 @@ export function useClassroomEvents({
209238
if (sessionStartMsRef.current === null) {
210239
sessionStartMsRef.current = data.keyframe.timestamp;
211240
}
241+
// P1-8:更新最近关键帧时间戳(漏捕检测依据)
242+
latestKeyframeTsRef.current = data.keyframe.timestamp;
212243
captureSessionIdRef.current = data.sessionId;
213244
setSmartBundle((prev) => ({
214245
...prev,
@@ -512,7 +543,8 @@ export function useClassroomEvents({
512543
return off;
513544
}, [status, captureManager]);
514545

515-
// P1-2:转写编辑回调——修正直播转录文本,audioText 存修正后文本供下游消费
546+
// P1-2:转写编辑回调——修正直播转录文本,audioText 存修正后文本供下游消费;
547+
// P1-3:修正差异自动回写本地词库(replace 词条,课程维度绑定,下次识别生效)
516548
const handleEditTranscript = useCallback((id: string, newText: string) => {
517549
setLiveTranscripts((prev) =>
518550
prev.map((t) => (t.id === id ? { ...t, editedText: newText } : t)),
@@ -524,7 +556,32 @@ export function useClassroomEvents({
524556
s.id === id ? { ...s, audioText: newText, audioTextRaw: s.audioText } : s,
525557
),
526558
}));
527-
}, []);
559+
560+
// ── P1-3 修正回写(讯飞「用户修正回写」/飞书「改字同步」闭环)──
561+
const original = liveTranscripts.find((t) => t.id === id)?.text ?? '';
562+
const correction = extractCorrection(original, newText);
563+
if (correction) {
564+
// 会话内立即生效(后续转写应用替换)
565+
addSessionReplace(correction.term, correction.target);
566+
// 持久化词库(去重后写入;失败静默,不影响编辑结果)
567+
hotwordStore.listForCourse(courseName).then((entries) => {
568+
const exists = entries.some(
569+
(e) => e.kind === 'replace' && e.term === correction.term && (e.target ?? '') === correction.target,
570+
);
571+
if (!exists) {
572+
return hotwordStore.add({
573+
term: correction.term,
574+
target: correction.target,
575+
kind: 'replace',
576+
courseId: courseName,
577+
enabled: true,
578+
});
579+
}
580+
return null;
581+
}).catch(() => { /* 词库写入失败静默 */ });
582+
toast({ type: 'info', silent: true, message: `已记住修正:「${correction.term}」→「${correction.target}」` });
583+
}
584+
}, [liveTranscripts, courseName, toast]);
528585

529586
return {
530587
segments, setSegments, stats, setStats, extractionError, setExtractionError,

‎client/src/features/classroom/pages/ClassroomPage.tsx‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,16 +60,22 @@ export default function ClassroomPage() {
6060
setShowNoteDialog(true);
6161
}, [capture]);
6262

63-
// M 快捷键:课中标记重点
64-
const { handleBookmark } = capture;
63+
// M 快捷键:课中标记重点;C 快捷键:手动补截当前画面(P1-8)
64+
const { handleBookmark, handleManualCapture } = capture;
6565
const handleKeyDown = useCallback((e: KeyboardEvent) => {
6666
if (e.key === 'm' || e.key === 'M') {
6767
// 避免在输入框中触发
6868
const tag = (e.target as HTMLElement)?.tagName;
6969
if (tag === 'INPUT' || tag === 'TEXTAREA') return;
7070
handleBookmark();
7171
}
72-
}, [handleBookmark]);
72+
if (e.key === 'c' || e.key === 'C') {
73+
// 避免在输入框中触发(快捷键截图同样跳过编辑态)
74+
const tag = (e.target as HTMLElement)?.tagName;
75+
if (tag === 'INPUT' || tag === 'TEXTAREA') return;
76+
handleManualCapture();
77+
}
78+
}, [handleBookmark, handleManualCapture]);
7379

7480
useEffect(() => {
7581
window.addEventListener('keydown', handleKeyDown);

‎client/src/features/classroom/utils/hotwordRuntime.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,15 @@ export function addDynamicBoosts(terms: string[]): void {
9090
if (cleaned.length === 0) return;
9191
activeBoosts = [...new Set([...activeBoosts, ...cleaned])].slice(0, 30);
9292
}
93+
94+
/**
95+
* P1-3:会话内立即生效的替换词条(用户修正回写)。
96+
* 与持久化词库(hotwordStore)区分:仅当前会话生效(后续转写应用),
97+
* 持久化由调用方写 hotwordStore;去重防止重复追加。
98+
*/
99+
export function addSessionReplace(term: string, target: string): void {
100+
const t = term.trim();
101+
if (!t || t.length > 20) return;
102+
if (activeReplaces.some((r) => r.term === t && (r.target ?? '') === target)) return;
103+
activeReplaces = [...activeReplaces, { term: t, target: target ?? '' }];
104+
}
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/**
2+
* extractCorrection 单元测试(P1-3 修正回写差异提取)
3+
*
4+
* @ai-context: 锁定差异提取契约:中段替换/词尾替换/删词;单字差异、
5+
* 完全相同、超长片段、空输入不入库(返回 null)。
6+
*/
7+
import { describe, it, expect } from 'vitest';
8+
import { extractCorrection } from './transcriptCorrection';
9+
10+
describe('extractCorrection', () => {
11+
it('中段替换:错误词 → 正确词', () => {
12+
expect(extractCorrection('特征值限量', '特征向量')).toEqual({ term: '值限', target: '向' });
13+
expect(extractCorrection('梯度下减法', '梯度下降法')).toEqual({ term: '减', target: '降' });
14+
});
15+
16+
it('词尾替换', () => {
17+
expect(extractCorrection('特征值向量', '特征值分解')).toEqual({ term: '向量', target: '分解' });
18+
});
19+
20+
it('删词(target 更短)', () => {
21+
expect(extractCorrection('就是这样的一个', '就是这样')).toEqual({ term: '的一个', target: '' });
22+
});
23+
24+
it('完全相同 → null', () => {
25+
expect(extractCorrection('今天讲矩阵', '今天讲矩阵')).toBeNull();
26+
});
27+
28+
it('单字错字纠正可入库(用户主动修正即强信号)', () => {
29+
expect(extractCorrection('我们来看一下', '我们来看一看')).toEqual({ term: '下', target: '看' });
30+
});
31+
32+
it('超长片段(>20 字)→ null(整句重写非词条修正)', () => {
33+
expect(extractCorrection('一二三四五六七八九十甲乙丙丁戊己庚辛壬癸子丑寅卯', '完全不同的一整句内容长度超过二十个字符的改写')).toBeNull();
34+
});
35+
36+
it('空输入 → null', () => {
37+
expect(extractCorrection('', '内容')).toBeNull();
38+
expect(extractCorrection('内容', '')).toBeNull();
39+
expect(extractCorrection('', '')).toBeNull();
40+
});
41+
42+
it('空白差异归一化(trim 后相同 → null)', () => {
43+
expect(extractCorrection(' 今天讲矩阵 ', '今天讲矩阵')).toBeNull();
44+
});
45+
});
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/**
2+
* 转写修正差异提取(P1-3 修正回写)
3+
*
4+
* @ai-context: 从「原始转写 → 用户修正文本」提取最小差异片段(公共前缀/
5+
* 后缀剥离),生成 replace 词条(term=错误形态 → target=修正形态)写入
6+
* 本地词库。单字差异不入库(误伤面大);片段超长(>20 字)不入库(整句
7+
* 重写不是词条修正,交给笔记层)。
8+
* @ai-context EN: Minimal-diff extraction between original transcript and
9+
* user correction (common prefix/suffix stripping) to produce a replace
10+
* vocabulary entry. Single-char and over-long diffs are not persisted.
11+
*/
12+
13+
export interface Correction {
14+
/** 原文差异片段(ASR 错误形态) */
15+
term: string;
16+
/** 修正后片段(目标形态) */
17+
target: string;
18+
}
19+
20+
/**
21+
* 提取修正词条。
22+
* @returns 差异片段(term/target),无可入库差异时 null
23+
*/
24+
export function extractCorrection(original: string, corrected: string): Correction | null {
25+
const a = (original ?? '').trim();
26+
const b = (corrected ?? '').trim();
27+
if (!a || !b || a === b) return null;
28+
29+
// 公共前缀
30+
let prefix = 0;
31+
const maxPrefix = Math.min(a.length, b.length);
32+
while (prefix < maxPrefix && a[prefix] === b[prefix]) prefix++;
33+
34+
// 公共后缀(不与前缀重叠)
35+
let suffix = 0;
36+
const maxSuffix = Math.min(a.length - prefix, b.length - prefix);
37+
while (
38+
suffix < maxSuffix
39+
&& a[a.length - 1 - suffix] === b[b.length - 1 - suffix]
40+
) suffix++;
41+
42+
const term = a.slice(prefix, a.length - suffix);
43+
const target = b.slice(prefix, b.length - suffix);
44+
// term 必须有内容;target 允许为空串(替换词条语义:删除误词)
45+
if (!term) return null;
46+
// 片段长度上限:整句重写不是词条修正
47+
if (term.length > 20 || target.length > 20) return null;
48+
// 单字差异允许入库:用户主动修正本身是强信号,单字错字纠正
49+
// (如「减」→「降」)正是最高频的术语修正形态
50+
return { term, target };
51+
}

0 commit comments

Comments
 (0)