Skip to content

Commit 45caf98

Browse files
committed
feat(classroom): P1-6 内容类型感知 + P1-7 指令句补帧 + P1-9 实时截图流
P1-6: 规则分类器(标题关键词优先/转写证据投票/讲座场景优先于软件)→ 技能类采样参数收紧(阈值0.05/间隔5s);SmartSampler.setConfig 运行中调参 P1-7: hasCommandCue 指令模式检测 → SmartSampler.forceNextCapture 强制补帧(1s节流/哈希去重仍生效) P1-9: UnifiedTimeline 最近6帧截图缩略流(识别过程可见) 新增 22 例单测(分类契约/参数驱动/强制补帧/setConfig)
1 parent 980ea39 commit 45caf98

9 files changed

Lines changed: 440 additions & 4 deletions

File tree

‎client/src/features/classroom/components/UnifiedTimeline.tsx‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ export function UnifiedTimeline({ bundle, liveTranscripts, autoAnchors = [], par
8383
const audioSegments = bundle.audioSegments ?? [];
8484
const sessionStartMs = timeline[0]?.timestamp ?? Date.now();
8585

86+
// P1-9 实时截图流:最近 6 帧仍有 imageBase64 的关键帧(增序展示)
87+
const recentThumbs = (bundle.keyframes ?? [])
88+
.filter((kf) => kf.imageBase64)
89+
.slice(-6);
90+
8691
// 事件 + 锚点 + 转写按时间戳合并排序
8792
const rows: Row[] = [
8893
...timeline.map((entry) => ({ kind: 'event' as const, ts: entry.timestamp, entry })),
@@ -281,6 +286,22 @@ export function UnifiedTimeline({ bundle, liveTranscripts, autoAnchors = [], par
281286
)}
282287
</div>
283288

289+
{/* P1-9 实时截图流:最近 6 帧缩略横条(识别过程可见性,imageBase64 已被分析清空的帧跳过) */}
290+
{recentThumbs.length > 0 && (
291+
<div className="flex items-center gap-1.5 px-3 py-1.5 border-t border-border/20 bg-bg-secondary/30 overflow-x-auto flex-shrink-0">
292+
<Camera className="w-3 h-3 text-text-quaternary flex-shrink-0" strokeWidth={1.5} />
293+
{recentThumbs.map((kf) => (
294+
<img
295+
key={kf.id}
296+
src={`data:image/jpeg;base64,${kf.imageBase64}`}
297+
alt={`关键帧 ${new Date(kf.timestamp).toLocaleTimeString()}`}
298+
title={new Date(kf.timestamp).toLocaleTimeString()}
299+
className="w-16 h-9 rounded-kb-xs object-cover border border-border/30 flex-shrink-0"
300+
/>
301+
))}
302+
</div>
303+
)}
304+
284305
{/* 底部统计栏 */}
285306
<div className={cn(
286307
'px-3 py-2 border-t border-border/20 flex items-center justify-between text-[10px]',

‎client/src/features/classroom/hooks/useClassroomCapture.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ export function useClassroomCapture() {
146146
language: config.language, aiDetectEnabled, setCourseMeta,
147147
onNotify: notify,
148148
streamingAsrActive,
149+
windowTitle: selectedWindow?.title,
149150
});
150151

151152
// 转写/提取段最新值 ref 桥:停止收尾(useSessionControl)在事件回调
@@ -337,6 +338,8 @@ export function useClassroomCapture() {
337338
analysisError: analysis.analysisError,
338339
partialCount: events.partialCount,
339340
transcribedCount: events.transcribedCount,
341+
// P1-6 内容类型(课程/软件技能/手法技巧/讲座)
342+
contentKind: events.contentKind,
340343
// 实时转录
341344
liveTranscripts: events.liveTranscripts,
342345
handleEditTranscript: events.handleEditTranscript,

‎client/src/features/classroom/hooks/useClassroomEvents.ts‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { persistKeyframeImage } from '../utils/keyframePersistence';
3535
import { transcribeWithRetry, toAsrLanguage, useAsrSemaphore, isLocalAsrReady, setOnAsrFallback } from '../utils/asrTranscriber';
3636
import { applySessionReplaces, getSessionHotwordsString, addDynamicBoosts } from '../utils/hotwordRuntime';
3737
import { cleanAsrResult, dedupeAcrossFinals, estimateAsrConfidence } from '@/lib/capture/asrFilters';
38+
import { classifyContent, hasCommandCue, type ContentKind } from '@/lib/capture/contentClassifier';
3839

3940
/** 触发一次增量分析所需的关键帧数 */
4041
const INCREMENTAL_BATCH_SIZE = 5;
@@ -64,11 +65,20 @@ interface UseClassroomEventsOptions {
6465
onNotify: (type: 'error', message: string) => void;
6566
/** 真流式 ASR 是否激活(激活时跳过按段转写,改用流式 partial/final) */
6667
streamingAsrActive: boolean;
68+
/** P1-6:目标窗口标题(内容类型分类的标题信号) */
69+
windowTitle?: string;
6770
}
6871

72+
/** P1-6 分类尝试上限(转写样本仍无信号则停止,unknown 不再重试) */
73+
const MAX_CLASSIFY_ATTEMPTS = 5;
74+
/** P1-7 指令补帧节流:最小间隔(ms),防连续指令句触发补帧风暴 */
75+
const FORCE_CAPTURE_MIN_INTERVAL_MS = 1000;
76+
/** P1-6 分类转写样本上限(字符,FIFO 追加防无界) */
77+
const MAX_CLASSIFY_SAMPLE_CHARS = 500;
78+
6979
export function useClassroomEvents({
7080
captureManager, status, capturePath, language, aiDetectEnabled, setCourseMeta, onNotify,
71-
streamingAsrActive,
81+
streamingAsrActive, windowTitle,
7282
}: UseClassroomEventsOptions) {
7383
const [segments, setSegments] = useState<ExtractedSegment[]>([]);
7484
const [stats, setStats] = useState({ frames: 0, extracted: 0 });
@@ -82,11 +92,19 @@ export function useClassroomEvents({
8292
const [videoFilePath, setVideoFilePath] = useState<string | null>(null);
8393
const [partialCount, setPartialCount] = useState(0);
8494
const [transcribedCount, setTranscribedCount] = useState(0);
95+
/** P1-6 内容类型(课程/软件技能/手法技巧/讲座;驱动采样参数与产物形态) */
96+
const [contentKind, setContentKind] = useState<ContentKind>('unknown');
8597

8698
const partialNotesRef = useRef<string[]>([]);
8799
const pendingKeyframesRef = useRef<KeyFrame[]>([]);
88100
const isPartialAnalyzingRef = useRef(false);
89101
const courseDetectedRef = useRef(false);
102+
/** P1-6 分类尝试计数(超上限后停止,unknown 不再重试) */
103+
const classifyAttemptsRef = useRef(0);
104+
/** P1-6 转写样本累积(FIFO 截断) */
105+
const transcriptSampleRef = useRef('');
106+
/** P1-7 指令补帧节流时间戳 */
107+
const lastForceCaptureAtRef = useRef(0);
90108
/** @ai-context 会话时间基准(epoch ms):记录首帧 timestamp,供 analyzePartial 换算相对秒数 */
91109
const sessionStartMsRef = useRef<number | null>(null);
92110
/** 采集会话 ID(smart:keyframe 事件携带),供笔记持久化关联与关键帧图片清理 */
@@ -110,6 +128,36 @@ export function useClassroomEvents({
110128
},
111129
});
112130

131+
/**
132+
* P1-6/P1-7 转写文本场景消费(ref 桥接,事件回调经此调用避免 effect 重订阅):
133+
* ① 累积分类样本并触发内容分类(最多 5 次尝试,标题信号 > 转写证据)
134+
* ② 指令句检测命中 → 请求强制补帧(1s 节流,防补帧风暴)
135+
*/
136+
const sceneProcessorRef = useRef<(text: string) => void>(() => {});
137+
sceneProcessorRef.current = (text: string) => {
138+
if (!text) return;
139+
// 样本累积(FIFO 截断,防长课无界增长)
140+
transcriptSampleRef.current = (transcriptSampleRef.current + text).slice(-MAX_CLASSIFY_SAMPLE_CHARS);
141+
// 内容分类:未确定时尝试
142+
if (contentKind === 'unknown' && classifyAttemptsRef.current < MAX_CLASSIFY_ATTEMPTS) {
143+
classifyAttemptsRef.current++;
144+
const result = classifyContent(windowTitle ?? '', transcriptSampleRef.current);
145+
if (result.kind !== 'unknown') {
146+
setContentKind(result.kind);
147+
captureManager.applyContentKind(result.kind);
148+
console.info(`[useClassroomCapture] 内容类型识别: ${result.kind}(${result.source})`);
149+
}
150+
}
151+
// 指令句补帧(技能场景操作瞬间捕捉)
152+
if (hasCommandCue(text)) {
153+
const now = Date.now();
154+
if (now - lastForceCaptureAtRef.current >= FORCE_CAPTURE_MIN_INTERVAL_MS) {
155+
lastForceCaptureAtRef.current = now;
156+
captureManager.requestForceCapture();
157+
}
158+
}
159+
};
160+
113161
// ASR 本地→云端降级可见性:注册 toast 回调(会话级节流在 asrTranscriber 内部)
114162
useEffect(() => {
115163
setOnAsrFallback(() => toast({ type: 'warning', silent: true, message: '本地 ASR 不可用,已降级为云端转写' }));
@@ -284,6 +332,8 @@ export function useClassroomEvents({
284332
}));
285333
if (cleaned) {
286334
setTranscribedCount((c) => c + 1);
335+
// P1-6/P1-7:转写文本场景消费(内容分类 + 指令句补帧)
336+
sceneProcessorRef.current(cleaned);
287337
// 实时转录上屏(FIFO 上限控制);展示替换后文本(P1-3 替换词后处理),
288338
// 原始清洗后转写经 audioTextRaw 可回溯
289339
setLiveTranscripts((prev) => {
@@ -351,6 +401,8 @@ export function useClassroomEvents({
351401
: estimateAsrConfidence(data.text ?? '', cleaned);
352402
const id = crypto.randomUUID();
353403
const timestamp = data.timestamp || Date.now();
404+
// P1-6/P1-7:转写文本场景消费(内容分类 + 指令句补帧)
405+
sceneProcessorRef.current(text);
354406
// 实时转录上屏(FIFO 上限控制);展示替换后文本(P1-3)
355407
setLiveTranscripts((prev) => {
356408
const next = [...prev, { id, text: applySessionReplaces(text), timestamp, confidence }];
@@ -483,6 +535,7 @@ export function useClassroomEvents({
483535
vadStats, recordingStatus, setRecordingStatus,
484536
videoFilePath, setVideoFilePath,
485537
partialCount, setPartialCount, transcribedCount,
538+
contentKind,
486539
partialNotesRef, pendingKeyframesRef, isPartialAnalyzingRef,
487540
captureSessionIdRef,
488541
};

‎client/src/lib/capture/captureManager.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,20 @@ export class CaptureManager {
172172
audioPipeline.pushAudioChunk(this.rt, audioData);
173173
}
174174

175+
/**
176+
* P1-6 应用内容分类结果(smart 路径采样参数联动;非 smart 路径忽略)
177+
*/
178+
applyContentKind(kind: import('./contentClassifier').ContentKind): void {
179+
this.rt.smartController.applyContentKind(kind);
180+
}
181+
182+
/**
183+
* P1-7 请求一次强制补帧(指令句命中后调用;smart 路径生效)
184+
*/
185+
requestForceCapture(): void {
186+
this.rt.smartController.requestForceCapture();
187+
}
188+
175189
/**
176190
* 暂停采集(停止接收新数据,但保持会话状态)
177191
*/
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/**
2+
* 内容类型分类器测试(P1-6 规则版 + P1-7 指令句检测)
3+
*
4+
* @ai-context: 锁定分类契约:标题关键词优先(软件/手法/讲座/课程)、转写
5+
* 证据兜底(命中数显著领先才判定)、无信号 unknown;指令句检测覆盖操作
6+
* 动词与参数类词汇。
7+
*/
8+
import { describe, it, expect } from 'vitest';
9+
import {
10+
classifyByTitle,
11+
classifyByTranscript,
12+
classifyContent,
13+
hasCommandCue,
14+
SKILL_SAMPLER_CONFIG,
15+
COURSE_SAMPLER_CONFIG,
16+
} from './contentClassifier';
17+
18+
describe('classifyByTitle — 窗口标题规则', () => {
19+
it('软件工具名 → software_skill', () => {
20+
expect(classifyByTitle('Photoshop 2026 从入门到精通')).toBe('software_skill');
21+
expect(classifyByTitle('剪映专业版教程')).toBe('software_skill');
22+
expect(classifyByTitle('Python 数据分析实战')).toBe('software_skill');
23+
});
24+
25+
it('手法技巧类 → craft_skill', () => {
26+
expect(classifyByTitle('新手化妆教程:日常通勤妆')).toBe('craft_skill');
27+
expect(classifyByTitle('手机拍照技巧 10 分钟速成')).toBe('craft_skill');
28+
expect(classifyByTitle('瑜伽入门教学')).toBe('craft_skill');
29+
});
30+
31+
it('讲座/会议类 → lecture', () => {
32+
expect(classifyByTitle('2026 AI 开发者大会主题演讲')).toBe('lecture');
33+
expect(classifyByTitle('深度访谈:物理学家的世界观')).toBe('lecture');
34+
});
35+
36+
it('通用教程标题 → course', () => {
37+
expect(classifyByTitle('高等数学全程课程')).toBe('course');
38+
});
39+
40+
it('无信号标题 → unknown', () => {
41+
expect(classifyByTitle('')).toBe('unknown');
42+
expect(classifyByTitle('随便看看')).toBe('unknown');
43+
});
44+
});
45+
46+
describe('classifyByTranscript — 转写证据投票', () => {
47+
it('软件操作指令词显著 → software_skill', () => {
48+
expect(classifyByTranscript('我们点击图层面板,把这个参数调整一下,快捷键是 Ctrl+J')).toBe('software_skill');
49+
});
50+
51+
it('手法类词汇显著 → craft_skill', () => {
52+
expect(classifyByTranscript('注意手腕的力度和角度,光线从侧面打过来,构图要留白')).toBe('craft_skill');
53+
});
54+
55+
it('授课类词汇显著 → course', () => {
56+
expect(classifyByTranscript('这个定理的证明过程我们推导一遍,记住这个公式,考试会考')).toBe('course');
57+
});
58+
59+
it('证据不足/无信号 → unknown', () => {
60+
expect(classifyByTranscript('')).toBe('unknown');
61+
expect(classifyByTranscript('大家好,今天我们继续上次的内容')).toBe('unknown');
62+
});
63+
});
64+
65+
describe('classifyContent — 综合分类', () => {
66+
it('标题信号优先于转写证据', () => {
67+
const r = classifyContent('PS 教程', '这个定理的证明我们推导一遍');
68+
expect(r.kind).toBe('software_skill');
69+
expect(r.source).toBe('title');
70+
});
71+
72+
it('标题无信号时用转写证据', () => {
73+
const r = classifyContent('未命名窗口', '点击图层把参数调到百分之五十');
74+
expect(r.kind).toBe('software_skill');
75+
expect(r.source).toBe('transcript');
76+
});
77+
78+
it('双无信号 → unknown/none', () => {
79+
const r = classifyContent('', '');
80+
expect(r.kind).toBe('unknown');
81+
expect(r.source).toBe('none');
82+
});
83+
});
84+
85+
describe('采样参数(分类驱动)', () => {
86+
it('技能类参数收紧(阈值 0.05 / 间隔 5s)', () => {
87+
expect(SKILL_SAMPLER_CONFIG.changeThreshold).toBe(0.05);
88+
expect(SKILL_SAMPLER_CONFIG.periodicIntervalMs).toBe(5000);
89+
});
90+
91+
it('授课类参数维持默认(阈值 0.12 / 间隔 15s)', () => {
92+
expect(COURSE_SAMPLER_CONFIG.changeThreshold).toBe(0.12);
93+
expect(COURSE_SAMPLER_CONFIG.periodicIntervalMs).toBe(15000);
94+
});
95+
});
96+
97+
describe('hasCommandCue — 指令句检测(P1-7)', () => {
98+
it('操作动词命中', () => {
99+
expect(hasCommandCue('我们点击这个按钮')).toBe(true);
100+
expect(hasCommandCue('把图层拖到面板里')).toBe(true);
101+
expect(hasCommandCue('选择套索工具')).toBe(true);
102+
});
103+
104+
it('参数类词汇命中', () => {
105+
expect(hasCommandCue('把不透明度设置为百分之五十')).toBe(true);
106+
expect(hasCommandCue('这个参数调到三十')).toBe(true);
107+
});
108+
109+
it('普通讲解不命中', () => {
110+
expect(hasCommandCue('这个定理的证明思路是这样的')).toBe(false);
111+
expect(hasCommandCue('')).toBe(false);
112+
});
113+
});

0 commit comments

Comments
 (0)