Skip to content

Commit 3522567

Browse files
committed
fix(ai-asr): 重构 ASR 引擎为 zipformer-transducer 单引擎
- 移除离线 SenseVoice 引擎,统一为 zipformer-transducer 中英双语流式模型 - 流式与按段转写共用同一引擎,简化维护 - 下载流程增加 Transform 流背压控制 + 10 分钟总超时保护 - 流式启动支持热词增强透传(createStream hotwords) - 双版本 API 兼容(工厂函数 <=1.12 / 类导出 1.13+)
1 parent ed5c55a commit 3522567

5 files changed

Lines changed: 234 additions & 261 deletions

File tree

‎client/electron/ai/local-asr/SherpaAsrService.ts‎

Lines changed: 53 additions & 153 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,20 @@
22
* 本地 ASR — sherpa-onnx 语音识别服务
33
*
44
* @ai-context: 基于 sherpa-onnx-node 原生 addon 实现本地 ASR。
5-
* 双引擎架构:
6-
* - offline(SenseVoice):非流式,50ms/段极速推理,中文准确率最高,课后精修
7-
* - streaming(Paraformer):流式,边说边出 < 200ms 延迟,实时字幕
5+
* 单一引擎:zipformer-transducer 中英双语流式模型,同时承担实时流式
6+
* 转写(streamingAsr.ts 真流式路径)与按段转写(transcribeLocal 路径)。
7+
* @ai-context: zipformer-transducer 支持 createStream(hotwords) 热词增强,
8+
* 热词字符串由调用方经 IPC payload 透传(详见 local_asr_transcribe 与
9+
* local_asr_stream_start 接口)。
810
* @ai-context: sherpa-onnx-node 为可选依赖(optionalDependencies),
911
* 加载失败时 isAvailable() 返回 false,上层自动降级到云端 ASR。
1012
* 符合项目"可选增强"设计原则。
1113
* @ai-context: 音频输入要求:16kHz 单声道 Float32 PCM。
1214
* 渲染进程发来的音频块已满足此格式(process-audio native addon 输出)。
1315
* @ai-context: sherpa-onnx-node API 兼容:旧版(<=1.12)导出工厂函数
14-
* createOfflineRecognizer/createOnlineRecognizer;新版(1.13+,如 1.13.4)
15-
* 改为导出类 OfflineRecognizer/OnlineRecognizer(new 构造,config 形状不变)。
16-
* 下方工厂包装双路径兼容,否则升级依赖后 TypeError: ... is not a function
17-
* (2026-08 真流式启动失败根因,离线路径同样受影响)。
16+
* createOnlineRecognizer;新版(1.13+,如 1.13.4)改为导出类
17+
* OnlineRecognizer(new 构造,config 形状不变)。
18+
* 下方工厂包装双路径兼容,否则升级依赖后 TypeError: ... is not a function。
1819
*/
1920

2021
import * as path from 'path';
@@ -25,7 +26,7 @@ import {
2526
getLocalAsrConfig,
2627
getModelDir,
2728
isModelReady,
28-
type AsrEngine,
29+
MODEL_FILES,
2930
} from './config.js';
3031

3132
// ================================================================
@@ -37,30 +38,13 @@ type RecognizerCtor<T> = new (config: Record<string, unknown>) => T;
3738

3839
interface SherpaOnnx {
3940
/** 旧版 API(<=1.12):工厂函数 */
40-
createOfflineRecognizer?: (config: Record<string, unknown>) => OfflineRecognizer;
4141
createOnlineRecognizer?: (config: Record<string, unknown>) => OnlineRecognizer;
4242
/** 新版 API(1.13+):类导出 */
43-
OfflineRecognizer?: RecognizerCtor<OfflineRecognizer>;
4443
OnlineRecognizer?: RecognizerCtor<OnlineRecognizer>;
4544
}
4645

47-
interface OfflineRecognizer {
48-
createStream(): OfflineStream;
49-
decode(stream: OfflineStream): void;
50-
getResult(stream: OfflineStream): { text: string };
51-
}
52-
53-
interface OfflineStream {
54-
/** 旧版:(sampleRate, samples);新版 1.13+:({ samples, sampleRate }),统一走 feedWaveform */
55-
acceptWaveform(sampleRate: number, samples: Float32Array): void;
56-
acceptWaveform(waveform: { samples: Float32Array; sampleRate: number }): void;
57-
inputFinished(): void;
58-
/** 新版 OnlineStream/OfflineStream 未提供 free(句柄由 GC 回收),故为可选 */
59-
free?(): void;
60-
}
61-
6246
export interface OnlineRecognizer {
63-
createStream(): OnlineStream;
47+
createStream(hotwords?: string): OnlineStream;
6448
decode(stream: OnlineStream): void;
6549
isReady(stream: OnlineStream): boolean;
6650
getResult(stream: OnlineStream): { text: string };
@@ -73,6 +57,7 @@ export interface OnlineStream {
7357
acceptWaveform(sampleRate: number, samples: Float32Array): void;
7458
acceptWaveform(waveform: { samples: Float32Array; sampleRate: number }): void;
7559
inputFinished(): void;
60+
/** 新版 OnlineStream 未提供 free(句柄由 GC 回收),故为可选 */
7661
free?(): void;
7762
}
7863

@@ -83,7 +68,7 @@ export interface OnlineStream {
8368
* 按函数形参数(length)判别,调用方无需感知版本。
8469
*/
8570
export function feedWaveform(
86-
stream: OnlineStream | OfflineStream,
71+
stream: OnlineStream,
8772
sampleRate: number,
8873
samples: Float32Array,
8974
): void {
@@ -97,17 +82,6 @@ export function feedWaveform(
9782
let _sherpa: SherpaOnnx | null = null;
9883
let _loadAttempted = false;
9984

100-
/** 创建离线识别器:优先旧版工厂函数,缺失时用新版类构造 */
101-
function instantiateOffline(sherpa: SherpaOnnx, config: Record<string, unknown>): OfflineRecognizer {
102-
if (typeof sherpa.createOfflineRecognizer === 'function') {
103-
return sherpa.createOfflineRecognizer(config);
104-
}
105-
if (typeof sherpa.OfflineRecognizer === 'function') {
106-
return new sherpa.OfflineRecognizer(config);
107-
}
108-
throw new Error('sherpa-onnx-node 既无 createOfflineRecognizer 工厂也无 OfflineRecognizer 类,请核对依赖版本');
109-
}
110-
11185
/** 创建在线识别器:优先旧版工厂函数,缺失时用新版类构造 */
11286
function instantiateOnline(sherpa: SherpaOnnx, config: Record<string, unknown>): OnlineRecognizer {
11387
if (typeof sherpa.createOnlineRecognizer === 'function') {
@@ -139,67 +113,42 @@ function loadSherpa(): SherpaOnnx | null {
139113
// Recognizer 单例缓存
140114
// ================================================================
141115

142-
let _offlineRecognizer: OfflineRecognizer | null = null;
143116
let _onlineRecognizer: OnlineRecognizer | null = null;
144117

145-
/** 获取/创建 SenseVoice 非流式识别器(单例) */
146-
function getOfflineRecognizer(): OfflineRecognizer | null {
147-
if (_offlineRecognizer) return _offlineRecognizer;
148-
149-
const sherpa = loadSherpa();
150-
if (!sherpa) return null;
151-
if (!isModelReady('offline')) return null;
152-
153-
const modelDir = getModelDir('offline');
154-
const config = getLocalAsrConfig();
155-
const threads = config.threads > 0 ? config.threads : Math.max(1, os.cpus().length - 1);
156-
157-
try {
158-
_offlineRecognizer = instantiateOffline(sherpa, {
159-
featConfig: { sampleRate: 16000, featureDim: 80 },
160-
modelConfig: {
161-
senseVoice: {
162-
model: path.join(modelDir, 'model.int8.onnx'),
163-
language: config.language === 'auto' ? 'auto' : config.language,
164-
useItn: true,
165-
},
166-
tokens: path.join(modelDir, 'tokens.txt'),
167-
numThreads: threads,
168-
provider: 'cpu',
169-
},
170-
});
171-
logger.info(`[LocalASR] SenseVoice offline recognizer created (threads=${threads})`);
172-
return _offlineRecognizer;
173-
} catch (err) {
174-
logger.error(`[LocalASR] Failed to create offline recognizer: ${err}`);
175-
return null;
176-
}
177-
}
178-
179-
/** 获取/创建 Paraformer 流式识别器(单例) */
118+
/** 获取/创建 Zipformer 流式识别器(单例) */
180119
export function getOnlineRecognizer(): OnlineRecognizer | null {
181120
if (_onlineRecognizer) return _onlineRecognizer;
182121

183122
const sherpa = loadSherpa();
184123
if (!sherpa) return null;
185-
if (!isModelReady('streaming')) return null;
124+
if (!isModelReady()) return null;
186125

187-
const modelDir = getModelDir('streaming');
126+
const modelDir = getModelDir();
188127
const config = getLocalAsrConfig();
189-
const threads = config.threads > 0 ? config.threads : Math.max(1, os.cpus().length - 1);
128+
const cpuCount = Math.max(1, os.cpus().length);
129+
const rawThreads = config.threads > 0 ? config.threads : Math.max(1, cpuCount - 1);
130+
// 线程数上限:不超过 CPU 核心数(防止用户配置过大值导致 CPU 过载)
131+
const threads = Math.min(rawThreads, cpuCount);
190132

191133
try {
192134
_onlineRecognizer = instantiateOnline(sherpa, {
193135
featConfig: { sampleRate: 16000, featureDim: 80 },
194136
modelConfig: {
195-
paraformer: {
196-
encoder: path.join(modelDir, 'encoder.onnx'),
197-
decoder: path.join(modelDir, 'decoder.onnx'),
137+
// zipformer2 属于 transducer 类模型:sherpa-onnx-node 的 C API 字段为
138+
// `transducer`(encoder/decoder/joiner 三件套),没有 `zipformer2` 字段;
139+
// 曾误用 zipformer2 导致字段被忽略、encoder 为空而创建失败
140+
transducer: {
141+
encoder: path.join(modelDir, MODEL_FILES.encoder),
142+
decoder: path.join(modelDir, MODEL_FILES.decoder),
143+
joiner: path.join(modelDir, MODEL_FILES.joiner),
198144
},
199-
tokens: path.join(modelDir, 'tokens.txt'),
145+
tokens: path.join(modelDir, MODEL_FILES.tokens),
200146
numThreads: threads,
201147
provider: 'cpu',
202148
},
149+
// enableEndpoint 显式开启:sherpa-onnx-node 1.13+ 官方 API 为扁平字段,
150+
// 同时保留 endpointConfig 嵌套形状以兼容 rule2.minUtteranceLength=8
151+
enableEndpoint: true,
203152
endpointConfig: {
204153
rule1: { minTrailingSilence: 2.4 },
205154
// rule2 必须配 minUtteranceLength:仅给 minTrailingSilence 时默认 0,
@@ -209,7 +158,7 @@ export function getOnlineRecognizer(): OnlineRecognizer | null {
209158
rule3: { minUtteranceLength: 20 },
210159
},
211160
});
212-
logger.info(`[LocalASR] Paraformer streaming recognizer created (threads=${threads})`);
161+
logger.info(`[LocalASR] Zipformer streaming recognizer created (threads=${threads})`);
213162
return _onlineRecognizer;
214163
} catch (err) {
215164
logger.error(`[LocalASR] Failed to create streaming recognizer: ${err}`);
@@ -221,100 +170,53 @@ export function getOnlineRecognizer(): OnlineRecognizer | null {
221170
// 公共 API
222171
// ================================================================
223172

224-
/**
225-
* TODO(P1-3 热词增强): 经查 sherpa-onnx-node 类型声明,hotwords 仅 transducer
226-
* 系模型支持(OfflineStream createStream(hotwords),见 non-streaming-asr.js 注释
227-
* "Hotwords are supported only by transducer models");本项目使用的
228-
* SenseVoice(offline)与 Paraformer(streaming)均不支持。故本期不透传热词,
229-
* 渲染进程词表 boost 词条已由 hotwordRuntime.getSessionBoostWords() 预留,
230-
* 未来若换用 zipformer-transducer 模型,可给 local_asr_transcribe /
231-
* local_asr_stream_start 的 IPC payload 增加可选 hotwords 字段(旧载荷须兼容)
232-
* 并在 createStream 时传入。云端网关热词透传同为遗留项(不改 transcribe.py)。
233-
*/
234-
235173
/**
236174
* 检测本地 ASR 是否可用(sherpa-onnx 已加载 + 模型已下载)
237175
*/
238176
export async function checkLocalAsrAvailable(): Promise<boolean> {
239177
const sherpa = loadSherpa();
240178
if (!sherpa) return false;
241-
242-
const config = getLocalAsrConfig();
243-
return isModelReady(config.engine);
179+
return isModelReady();
244180
}
245181

246182
/**
247-
* 真流式 ASR 是否可用(同步):sherpa 已加载 + 本地 ASR 启用 + streaming(Paraformer)模型就绪。
183+
* 真流式 ASR 是否可用(同步):sherpa 已加载 + 本地 ASR 启用 + 模型就绪。
248184
* 供课堂智能采集决定是否走真流式链路(否则回退按段转写)。
249185
*/
250186
export function isStreamingAsrAvailable(): boolean {
251187
const sherpa = loadSherpa();
252188
if (!sherpa) return false;
253189
if (!getLocalAsrConfig().enabled) return false;
254-
return isModelReady('streaming');
190+
return isModelReady();
255191
}
256192

257193
/** 重置可用性缓存(模型下载完成后调用) */
258194
export function resetAvailabilityCache(): void {
259-
_offlineRecognizer = null;
260195
_onlineRecognizer = null;
261196
}
262197

263198
/**
264-
* 非流式转写(SenseVoice)— 适合课后全量分析
265-
*
266-
* @param pcmData - Float32 PCM 音频数据(16kHz 单声道)
267-
* @param options - 转写选项
268-
* @returns 转写结果
269-
*/
270-
export async function transcribeOffline(
271-
pcmData: Float32Array,
272-
options?: { language?: string },
273-
): Promise<{ text: string; engine: 'offline'; durationMs: number }> {
274-
const startTime = Date.now();
275-
276-
const recognizer = getOfflineRecognizer();
277-
if (!recognizer) {
278-
throw new Error('SenseVoice 识别器不可用(模型未下载或 sherpa-onnx 未安装)');
279-
}
280-
281-
const stream = recognizer.createStream();
282-
try {
283-
feedWaveform(stream, 16000, pcmData);
284-
stream.inputFinished();
285-
recognizer.decode(stream);
286-
const result = recognizer.getResult(stream);
287-
// 输出后处理:相邻重复压缩 + 幻觉过滤(与流式路径一致,见 streamingAsr.ts)
288-
const text = cleanAsrResult(result.text ?? '');
289-
const durationMs = Date.now() - startTime;
290-
291-
logger.debug(`[LocalASR] Offline transcribe: ${text.length} chars, ${durationMs}ms`);
292-
return { text, engine: 'offline', durationMs };
293-
} finally {
294-
stream.free?.();
295-
}
296-
}
297-
298-
/**
299-
* 流式转写(Paraformer)— 适合实时字幕
199+
* 完整音频段转写(Zipformer 在线识别器,模拟流式喂入)
300200
*
301201
* 将完整音频段一次性喂入流式识别器,逐帧解码后返回最终文本。
302-
* 真正的"边录边出"需要渲染进程持续推送音频块(后续版本支持)。
202+
* 用于按段转写路径(课堂非真流式 / 课后分析)。
303203
*
304204
* @param pcmData - Float32 PCM 音频数据(16kHz 单声道)
205+
* @param hotwords - 可选热词增强字符串(zipformer-transducer 支持)
305206
* @returns 转写结果
306207
*/
307208
export async function transcribeStreaming(
308209
pcmData: Float32Array,
309-
): Promise<{ text: string; engine: 'streaming'; durationMs: number }> {
210+
hotwords?: string,
211+
): Promise<{ text: string; engine: 'zipformer'; durationMs: number }> {
310212
const startTime = Date.now();
311213

312214
const recognizer = getOnlineRecognizer();
313215
if (!recognizer) {
314-
throw new Error('Paraformer 识别器不可用(模型未下载或 sherpa-onnx 未安装)');
216+
throw new Error('Zipformer 识别器不可用(模型未下载或 sherpa-onnx 未安装)');
315217
}
316218

317-
const stream = recognizer.createStream();
219+
const stream = recognizer.createStream(hotwords);
318220
try {
319221
// 分块喂入(模拟流式,每块 1600 样本 = 100ms)
320222
const chunkSize = 1600;
@@ -333,41 +235,39 @@ export async function transcribeStreaming(
333235
}
334236

335237
const result = recognizer.getResult(stream);
336-
// 输出后处理:相邻重复压缩 + 幻觉过滤(与流式路径一致,见 streamingAsr.ts)
238+
// 输出后处理:相邻重复压缩 + 幻觉过滤
337239
const text = cleanAsrResult(result.text ?? '');
338240
const durationMs = Date.now() - startTime;
339241

340-
logger.debug(`[LocalASR] Streaming transcribe: ${text.length} chars, ${durationMs}ms`);
341-
return { text, engine: 'streaming', durationMs };
242+
logger.debug(`[LocalASR] Zipformer transcribe: ${text.length} chars, ${durationMs}ms`);
243+
return { text, engine: 'zipformer', durationMs };
342244
} finally {
343245
stream.free?.();
344246
}
345247
}
346248

347249
/**
348-
* 统一转写入口(根据配置选择引擎)
250+
* 统一转写入口
349251
*
350252
* @param audioBase64 - base64 编码的 Float32 PCM 音频(16kHz 单声道)
351253
* @param options - 转写选项
352254
*/
353255
export async function transcribeLocal(
354256
audioBase64: string,
355-
options?: { language?: string; sampleRate?: number; channels?: number; engine?: AsrEngine },
257+
options?: { language?: string; sampleRate?: number; channels?: number; hotwords?: string },
356258
): Promise<{ text: string; language: string; durationMs: number }> {
357259
const config = getLocalAsrConfig();
358-
const engine = options?.engine ?? config.engine;
359260
const language = options?.language ?? config.language;
360261

262+
// 防御性校验:非 16kHz 采样率会严重降低识别质量
263+
if (options?.sampleRate && options.sampleRate !== 16000) {
264+
logger.warn(`[LocalASR] 非预期采样率: ${options.sampleRate}Hz,本地 ASR 要求 16kHz 单声道 Float32 PCM`);
265+
}
266+
361267
// base64 → Float32Array
362268
const rawBytes = Buffer.from(audioBase64, 'base64');
363269
const pcmData = new Float32Array(rawBytes.buffer, rawBytes.byteOffset, rawBytes.byteLength / 4);
364270

365-
if (engine === 'streaming' && isModelReady('streaming')) {
366-
const result = await transcribeStreaming(pcmData);
367-
return { text: result.text, language, durationMs: result.durationMs };
368-
}
369-
370-
// 默认走 offline(SenseVoice)
371-
const result = await transcribeOffline(pcmData, { language });
271+
const result = await transcribeStreaming(pcmData, options?.hotwords);
372272
return { text: result.text, language, durationMs: result.durationMs };
373-
}
273+
}

0 commit comments

Comments
 (0)