Skip to content

Commit 6ea6f0e

Browse files
committed
feat(classroom): P1-1 两遍重打分(SenseVoice 整句复核)
config/modelManager: 新增 rescore 模型定义(SenseVoice int8 ~230MB),旧模型清理移除 SenseVoice 目录,双引擎下载/删除/状态 sensevoiceRescore: 非流式识别器懒加载 + pickRescored 一致性择优(Jaccard>=0.35 采用重打分,差异过大保留流式原结果) 接入:按段转写与流式 final 均句末重打分(句音频累积含静音尾);重打分通过置信度提升至 >=0.85 设置页:开关收紧为流式主模型就绪(rescore 不能单独转写);模型列表自动展示双模型
1 parent 9a52e84 commit 6ea6f0e

6 files changed

Lines changed: 331 additions & 37 deletions

File tree

‎client/electron/ai/local-asr/SherpaAsrService.ts‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ import {
2525
cleanAsrResult,
2626
estimateAsrConfidence,
2727
} from '../../../src/lib/capture/asrFilters.js';
28+
import {
29+
rescoreWithSenseVoice,
30+
pickRescored,
31+
resetRescoreCache,
32+
} from './sensevoiceRescore.js';
2833
import {
2934
getLocalAsrConfig,
3035
getModelDir,
@@ -195,9 +200,10 @@ export function isStreamingAsrAvailable(): boolean {
195200
return isModelReady();
196201
}
197202

198-
/** 重置可用性缓存(模型下载完成后调用) */
203+
/** 重置可用性缓存(模型下载/删除后调用;联动清理重打分缓存) */
199204
export function resetAvailabilityCache(): void {
200205
_onlineRecognizer = null;
206+
resetRescoreCache();
201207
}
202208

203209
/**
@@ -240,15 +246,27 @@ export async function transcribeStreaming(
240246
}
241247

242248
const rawText = recognizer.getResult(stream).text ?? '';
249+
// P1-1 两遍重打分:SenseVoice 整句复核(模型未下载/失败时静默跳过,
250+
// 保留流式原结果;一致性校验通过才替换,见 pickRescored)
251+
let finalRaw = rawText;
252+
let rescored = false;
253+
const rescoreResult = rescoreWithSenseVoice(pcmData);
254+
if (rescoreResult?.text) {
255+
const picked = pickRescored(rawText, rescoreResult.text);
256+
if (picked.rescored) {
257+
finalRaw = picked.text;
258+
rescored = true;
259+
}
260+
}
243261
// 输出后处理:相邻重复压缩 + 幻觉过滤
244-
// P1-1 两遍重打分接入点:此处为本地按段转写最终文本出口,
245-
// SenseVoice 重打分将在此处对 text 做句末复核(高置信度者胜出)
246-
const text = cleanAsrResult(rawText);
247-
// P0-3:置信度估算(清洗前后长度比代理信号,供 UI 低置信度标记)
248-
const confidence = estimateAsrConfidence(rawText, text);
262+
const text = cleanAsrResult(finalRaw);
263+
// P0-3:置信度估算(清洗前后长度比代理信号;重打分通过视为高置信)
264+
const confidence = rescored
265+
? Math.max(estimateAsrConfidence(finalRaw, text), 0.85)
266+
: estimateAsrConfidence(finalRaw, text);
249267
const durationMs = Date.now() - startTime;
250268

251-
logger.debug(`[LocalASR] Zipformer transcribe: ${text.length} chars, ${durationMs}ms`);
269+
logger.debug(`[LocalASR] Zipformer transcribe: ${text.length} chars, ${durationMs}ms${rescored ? ' (rescored)' : ''}`);
252270
return { text, confidence, engine: 'zipformer', durationMs };
253271
} finally {
254272
stream.free?.();

‎client/electron/ai/local-asr/config.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ export const MODEL_FILES = {
6464
tokens: 'tokens.txt',
6565
} as const;
6666

67+
/** P1-1 SenseVoice 重打分模型关键文件名(int8 量化包) */
68+
export const RESCORE_MODEL_FILES = {
69+
model: 'model.int8.onnx',
70+
tokens: 'tokens.txt',
71+
} as const;
72+
6773
/**
6874
* 模型定义(供设置页展示 + modelManager 下载)
6975
*
@@ -91,6 +97,21 @@ export const ASR_MODELS = {
9197
/** hf-mirror.com 国内镜像(逐文件下载,无需解压) */
9298
mirrorBaseUrl: 'https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/resolve/main',
9399
},
100+
/**
101+
* P1-1 两遍重打分模型:SenseVoice-Small(非流式,整句上下文识别)。
102+
* 流式 Zipformer 负责实时出字,句末由 SenseVoice 对整句音频重打分
103+
* (整句注意力上下文精度优于流式解码),两结果一致性校验后择优。
104+
*/
105+
rescore: {
106+
id: 'rescore-sensevoice',
107+
label: 'SenseVoice 重打分(句末整句复核,提升中文准确率)',
108+
description: '句末对整句音频二次识别,与流式结果一致性校验后择优,适合追求准确率的课堂场景',
109+
size: '~230MB',
110+
dirName: 'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17',
111+
files: [RESCORE_MODEL_FILES.model, RESCORE_MODEL_FILES.tokens],
112+
downloadUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2',
113+
mirrorBaseUrl: 'https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17/resolve/main',
114+
},
94115
} as const;
95116

96117
// ================================================================
@@ -119,6 +140,11 @@ export function getModelDir(): string {
119140
return path.join(getModelsDir(), modelDef.dirName);
120141
}
121142

143+
/** P1-1:获取重打分模型目录完整路径 */
144+
export function getRescoreModelDir(): string {
145+
return path.join(getModelsDir(), ASR_MODELS.rescore.dirName);
146+
}
147+
122148
/**
123149
* 获取当前本地 ASR 配置
124150
*/
@@ -192,3 +218,11 @@ export function isModelReady(): boolean {
192218
// 检查关键文件是否存在
193219
return modelDef.files.every(f => existsSync(path.join(modelDir, f)));
194220
}
221+
222+
/**
223+
* P1-1:检查重打分模型是否已下载就绪
224+
*/
225+
export function isRescoreModelReady(): boolean {
226+
const modelDir = getRescoreModelDir();
227+
return ASR_MODELS.rescore.files.every(f => existsSync(path.join(modelDir, f)));
228+
}

‎client/electron/ai/local-asr/modelManager.ts‎

Lines changed: 60 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,26 @@ import { Transform } from 'stream';
1717
import * as path from 'path';
1818
import { BrowserWindow } from 'electron';
1919
import { logger } from '../../logger.js';
20-
import { getModelsDir, getModelDir, isModelReady, ASR_MODELS } from './config.js';
20+
import {
21+
getModelsDir,
22+
getModelDir,
23+
getRescoreModelDir,
24+
isModelReady,
25+
isRescoreModelReady,
26+
ASR_MODELS,
27+
} from './config.js';
2128
import { resetAvailabilityCache } from './SherpaAsrService.js';
29+
import { resetRescoreCache } from './sensevoiceRescore.js';
2230

2331
/** 所有出站 HTTP 请求统一携带 UA,避免 CDN 返回 403 */
2432
const HTTP_HEADERS = { 'User-Agent': 'EntropyDecrease-Desktop/1.0 (Electron)' };
2533

26-
/** 已废弃的旧模型目录名(Paraformer、SenseVoice),不再被使用,应清理 */
34+
/**
35+
* 已废弃的旧模型目录名(仅 Paraformer 双语模型——SenseVoice 已于
36+
* P1-1 重新启用为重打分模型,不再清理;误清会导致重打分不可用)
37+
*/
2738
const OBSOLETE_MODEL_DIRS = [
2839
'sherpa-onnx-streaming-paraformer-bilingual-zh-en',
29-
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17',
3040
];
3141

3242
// ================================================================
@@ -47,14 +57,25 @@ export function getDownloadStatus(): { downloading: string | null; progress: num
4757

4858
/** 获取模型状态列表 */
4959
export function getModelsStatus(): Array<{ engine: string; id: string; label: string; description: string; size: string; ready: boolean }> {
50-
return [{
51-
engine: 'streaming',
52-
id: ASR_MODELS.streaming.id,
53-
label: ASR_MODELS.streaming.label,
54-
description: ASR_MODELS.streaming.description,
55-
size: ASR_MODELS.streaming.size,
56-
ready: isModelReady(),
57-
}];
60+
return [
61+
{
62+
engine: 'streaming',
63+
id: ASR_MODELS.streaming.id,
64+
label: ASR_MODELS.streaming.label,
65+
description: ASR_MODELS.streaming.description,
66+
size: ASR_MODELS.streaming.size,
67+
ready: isModelReady(),
68+
},
69+
// P1-1:SenseVoice 重打分模型(可选增强,未下载不影响流式转写)
70+
{
71+
engine: 'rescore',
72+
id: ASR_MODELS.rescore.id,
73+
label: ASR_MODELS.rescore.label,
74+
description: ASR_MODELS.rescore.description,
75+
size: ASR_MODELS.rescore.size,
76+
ready: isRescoreModelReady(),
77+
},
78+
];
5879
}
5980

6081
/**
@@ -96,14 +117,15 @@ export async function downloadModel(engine: string): Promise<string> {
96117
throw new Error(`已有模型正在下载中(${_downloading}),请等待完成`);
97118
}
98119

99-
// 当前仅支持 streaming 引擎,参数验证确保前端误传时能给出明确错误
100-
if (engine !== 'streaming') {
101-
throw new Error(`不支持的引擎: ${engine}(当前仅支持 streaming)`);
120+
// P1-1:支持 streaming(流式主引擎)与 rescore(SenseVoice 重打分)
121+
if (engine !== 'streaming' && engine !== 'rescore') {
122+
throw new Error(`不支持的引擎: ${engine}(支持 streaming / rescore)`);
102123
}
103124

104-
const modelDef = ASR_MODELS.streaming;
125+
const modelDef = ASR_MODELS[engine];
105126
const modelsDir = getModelsDir();
106-
const targetDir = getModelDir();
127+
const targetDir = engine === 'streaming' ? getModelDir() : getRescoreModelDir();
128+
const modelReady = engine === 'streaming' ? isModelReady : isRescoreModelReady;
107129

108130
await mkdir(modelsDir, { recursive: true });
109131

@@ -132,15 +154,15 @@ export async function downloadModel(engine: string): Promise<string> {
132154

133155
// ── 策略 2 + 3:逐文件下载(hf-mirror → huggingface 直连) ──
134156
if (ghError) {
135-
if (!isModelReady()) {
157+
if (!modelReady()) {
136158
const bases = [
137159
modelDef.mirrorBaseUrl, // hf-mirror.com
138160
modelDef.mirrorBaseUrl.replace('hf-mirror.com', 'huggingface.co'), // 直连兜底
139161
];
140162
let lastErr: unknown = ghError;
141163
for (const base of bases) {
142164
try {
143-
await downloadFromMirror(engine, base, targetDir);
165+
await downloadFromMirror(engine, base, targetDir, modelDef);
144166
lastErr = null;
145167
break;
146168
} catch (mirrorErr) {
@@ -154,11 +176,12 @@ export async function downloadModel(engine: string): Promise<string> {
154176

155177
// 校验关键文件
156178
broadcastProgress(engine, 95);
157-
if (!isModelReady()) {
179+
if (!modelReady()) {
158180
throw new Error(`模型下载后校验失败:缺少关键文件(${modelDef.files.join(', ')})`);
159181
}
160182

161183
resetAvailabilityCache();
184+
resetRescoreCache();
162185
logger.info(`[LocalASR] ${engine} model ready at: ${targetDir}`);
163186
broadcastProgress(engine, 100);
164187
// 新模型下载完成后清理旧模型
@@ -173,11 +196,22 @@ export async function downloadModel(engine: string): Promise<string> {
173196
}
174197
}
175198

199+
/** 模型定义最小结构(streaming / rescore 通用) */
200+
interface ModelDef {
201+
files: readonly string[];
202+
downloadUrl: string;
203+
mirrorBaseUrl: string;
204+
}
205+
176206
/**
177207
* 从指定 baseUrl 逐文件下载模型(无需解压)
178208
*/
179-
async function downloadFromMirror(engine: string, baseUrl: string, targetDir: string): Promise<void> {
180-
const modelDef = ASR_MODELS.streaming;
209+
async function downloadFromMirror(
210+
engine: string,
211+
baseUrl: string,
212+
targetDir: string,
213+
modelDef: ModelDef,
214+
): Promise<void> {
181215
await mkdir(targetDir, { recursive: true });
182216

183217
logger.info(`[LocalASR] Downloading ${modelDef.files.length} files from: ${baseUrl}`);
@@ -197,14 +231,15 @@ async function downloadFromMirror(engine: string, baseUrl: string, targetDir: st
197231
}
198232

199233
/**
200-
* 删除模型
234+
* 删除模型(P1-1:支持 streaming / rescore 双引擎)
201235
*/
202-
export async function deleteModel(_engine: string): Promise<void> {
203-
const modelDir = getModelDir();
236+
export async function deleteModel(engine: string): Promise<void> {
237+
const modelDir = engine === 'rescore' ? getRescoreModelDir() : getModelDir();
204238
if (existsSync(modelDir)) {
205239
rmSync(modelDir, { recursive: true, force: true });
206240
resetAvailabilityCache();
207-
logger.info(`[LocalASR] streaming model deleted: ${modelDir}`);
241+
resetRescoreCache();
242+
logger.info(`[LocalASR] ${engine} model deleted: ${modelDir}`);
208243
}
209244
}
210245

0 commit comments

Comments
 (0)