Skip to content

Commit 2250310

Browse files
committed
feat(asr): asr_eval harness M2(批量双路 A/B/画像/回归门)
1 parent 11b3bb8 commit 2250310

2 files changed

Lines changed: 322 additions & 0 deletions

File tree

‎app/src-tauri/Cargo.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ name = "cer_bench"
4545
path = "src/bin/cer_bench.rs"
4646
test = false
4747

48+
# v0.20.0(REQ-263):asr_eval 自验证 harness(批量/双路 A/B/基线回归门)
49+
[[bin]]
50+
name = "asr_eval"
51+
path = "src/bin/asr_eval.rs"
52+
test = false
53+
4854
[features]
4955
# ADR-009(v0.4.0 M1):OCR CUDA EP 编译开关(默认开)——ORT 1.28 官方 gpu_cuda12
5056
# 运行时存在(download-ort-cuda.ps1 分发);feature 关闭时 CUDA 请求显式失败并

‎app/src-tauri/src/bin/asr_eval.rs‎

Lines changed: 316 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,316 @@
1+
//! asr_eval 自验证 harness(v0.20.0 / REQ-263,M2)。
2+
//!
3+
//! @ai-context: 目的——"无人工语料也能测 ASR"(2026-09-03 用户裁决①):
4+
//! 批量样本(wav + 同名 .srt 外挂字幕 = 参考信道 L1 ~100% 无损)
5+
//! → SenseVoice 离线转写(预处理开/关两路,A/B)→ CER + 混淆画像
6+
//! → 汇总/A/B 对比/画像 top-N → 相对基线回归门(退出码契约)。
7+
//! @ai-context: 复用现成件——app_lib::cer(CER/推荐)、app_lib::eval_*(画像/
8+
//! SRT/统计/回归门)、app_lib::audio_preprocess(A/B 预处理路);
9+
//! 引擎装载与转写沿用 bin/cer_bench.rs 同款 API(独立 bin 范式)。
10+
//! @ai-context: 局限(诚实登记)——①无参考样本只列表不产 CER;②SRT 仅 UTF-8/
11+
//! BOM(GBK 需 encoding 依赖,留 P1);③漂移分布需带时间轴的
12+
//! 会话信道(--db 会话模式,M2b 接线);④结论只做相对对比与回归,
13+
//! 不宣称绝对指标。
14+
//! @ai-context: 用法:cargo run --bin asr_eval -- <样本目录> [--model <dir>]
15+
//! [--preproc off|on|both] [--baseline <json>] [--update-baseline]
16+
//! [--out <目录>]
17+
//! @ai-context: 退出码:0=通过(含无参考样本的诚实告警);1=基线回归退化
18+
//! (均值 CER 超容差)或模型/运行失败;2=用法错误。
19+
20+
use std::fs;
21+
use std::path::{Path, PathBuf};
22+
use std::process::ExitCode;
23+
24+
use app_lib::audio_preprocess::{AudioPreprocessConfig, AudioPreprocessor};
25+
use app_lib::cer;
26+
use app_lib::eval_confusion::{profile, ConfusionAggregator};
27+
use app_lib::eval_report::{ab_verdict, cer_stats, is_regression, CerStats};
28+
use app_lib::eval_samples::{is_media_file, parse_srt, reference_text, srt_path_for};
29+
use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig, OfflineSenseVoiceModelConfig};
30+
31+
/// 默认 SenseVoice 模型目录(与 cer_bench/download 脚本约定一致)。
32+
const DEFAULT_MODEL_DIR: &str = "models/asr/sensevoice";
33+
/// 基线回归容差(v0.20.0 契约:相对基线均值 CER 退化 >2% 即失败)。
34+
const REGRESSION_TOLERANCE: f32 = 0.02;
35+
36+
/// A/B 路选择。
37+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38+
enum Route {
39+
Off,
40+
On,
41+
Both,
42+
}
43+
44+
impl Route {
45+
fn names(&self) -> Vec<&'static str> {
46+
match self {
47+
Route::Off => vec!["preproc_off"],
48+
Route::On => vec!["preproc_on"],
49+
Route::Both => vec!["preproc_off", "preproc_on"],
50+
}
51+
}
52+
}
53+
54+
#[derive(Debug)]
55+
struct Options {
56+
samples_dir: PathBuf,
57+
model_dir: PathBuf,
58+
out_dir: PathBuf,
59+
baseline: Option<PathBuf>,
60+
update_baseline: bool,
61+
preproc: Route,
62+
}
63+
64+
/// 单样本(文件对信道)。
65+
struct Sample {
66+
name: String,
67+
wav: PathBuf,
68+
reference: String,
69+
}
70+
71+
fn main() -> ExitCode {
72+
let opts = match parse_args() {
73+
Some(o) => o,
74+
None => return ExitCode::from(2),
75+
};
76+
let _ = fs::create_dir_all(&opts.out_dir);
77+
78+
let (samples, no_ref_skips) = collect_samples(&opts);
79+
if samples.is_empty() {
80+
println!("无可用样本(wav+srt 配对为空);跳过列表:{no_ref_skips:?}");
81+
return ExitCode::SUCCESS;
82+
}
83+
84+
let recognizer = load_recognizer(&opts.model_dir);
85+
let mut out = String::new();
86+
let mut rows: Vec<(String, String, Option<f32>)> = Vec::new();
87+
let mut cers: std::collections::BTreeMap<String, Vec<f32>> = Default::default();
88+
let mut portraits: std::collections::BTreeMap<String, ConfusionAggregator> = Default::default();
89+
90+
for s in &samples {
91+
let Some(wave) = sherpa_onnx::Wave::read(&s.wav.to_string_lossy()) else {
92+
println!("[跳过] 音频解析失败: {}", s.wav.display());
93+
continue;
94+
};
95+
let (sr, samples_f32) = (wave.sample_rate(), wave.samples());
96+
for route in opts.preproc.names() {
97+
let preproc_on = route == "preproc_on";
98+
let text = transcribe(&recognizer, samples_f32, sr, preproc_on);
99+
let c = cer::cer(&s.reference, &text);
100+
rows.push((s.name.clone(), route.to_string(), c));
101+
if let Some(v) = c {
102+
cers.entry(route.to_string()).or_default().push(v);
103+
}
104+
portraits
105+
.entry(route.to_string())
106+
.or_default()
107+
.add(&profile(&s.reference, &text));
108+
}
109+
print!("{}. ", s.name);
110+
}
111+
112+
// ── 汇总渲染 ──
113+
out.push_str(&render_rows(&rows));
114+
out.push_str("\n== 各样本行 ==\n");
115+
for (name, route, c) in &rows {
116+
match c {
117+
Some(v) => out.push_str(&format!("{name}\t{route}\t{v:.4}\n")),
118+
None => out.push_str(&format!("{name}\t{route}\t-\n")),
119+
}
120+
}
121+
for route in opts.preproc.names() {
122+
let stats = cer_stats(cers.get(route).map(|v| v.as_slice()).unwrap_or(&[]));
123+
out.push_str(&format!(
124+
"\n[{}] 样本={} CER均值={} 范围={}~{}",
125+
route,
126+
stats.map(|s: CerStats| s.n.to_string()).unwrap_or_else(|| "0".into()),
127+
stats.map(|s| format!("{:.4}", s.mean)).unwrap_or_else(|| "不可比".into()),
128+
stats.map(|s| format!("{:.4}", s.min)).unwrap_or_else(|| "-".into()),
129+
stats.map(|s| format!("{:.4}", s.max)).unwrap_or_else(|| "-".into()),
130+
));
131+
let agg = portraits.get(route).cloned().unwrap_or_default();
132+
let top = agg.top_n(10);
133+
if top.is_empty() {
134+
out.push_str(&format!("\n[{}] 画像:无(样本不足或全对)\n", route));
135+
} else {
136+
out.push_str(&format!("\n[{}] 画像 top-10:", route));
137+
for ((a, b), n) in top {
138+
out.push_str(&format!(" {a}→{b}({n})"));
139+
}
140+
out.push('\n');
141+
}
142+
}
143+
// A/B 结论(仅 both 时)
144+
if opts.preproc == Route::Both {
145+
let a = cer_stats(cers.get("preproc_on").map(|v| v.as_slice()).unwrap_or(&[]));
146+
let b = cer_stats(cers.get("preproc_off").map(|v| v.as_slice()).unwrap_or(&[]));
147+
out.push_str(&format!("\n{}\n", ab_verdict("A/B(preproc 开 vs 关)", a, b)));
148+
}
149+
if !no_ref_skips.is_empty() {
150+
out.push_str(&format!("\n无参考样本(不产 CER):{no_ref_skips:?}\n"));
151+
}
152+
153+
println!("{out}");
154+
let _ = fs::write(opts.out_dir.join("summary.txt"), &out);
155+
156+
// ── 基线回归门 ──
157+
let primary = cers.get("preproc_off").map(|v| cer_stats(v)).flatten();
158+
if let Some(bp) = &opts.baseline {
159+
let cur = primary.map(|s| s.mean);
160+
let base_mean = read_baseline_mean(bp);
161+
match is_regression(cur, base_mean, REGRESSION_TOLERANCE) {
162+
Some(true) => {
163+
println!("[回归门] 失败:均值 CER {:.4} 相对基线 {:.4} 退化超容差", cur.unwrap(), base_mean.unwrap());
164+
return ExitCode::from(1);
165+
}
166+
Some(false) => println!("[回归门] 通过(均值 CER {:.4} vs 基线 {:.4})", cur.unwrap(), base_mean.unwrap()),
167+
None => println!("[回归门] 不可比(无参考/无基线数据),跳过"),
168+
}
169+
}
170+
if opts.update_baseline {
171+
let mean = primary.map(|s| format!("{:.4}", s.mean)).unwrap_or_else(|| "null".into());
172+
let json = format!("{{\"route\":\"preproc_off\",\"mean_cer\":{mean},\"samples\":{}}}\n", primary.map(|s| s.n).unwrap_or(0));
173+
let _ = fs::write(opts.out_dir.join("baseline.json"), json);
174+
println!("[基线] 已写 {}/baseline.json", opts.out_dir.display());
175+
}
176+
ExitCode::SUCCESS
177+
}
178+
179+
/// 汇总表渲染(CSV 行序:样本 × 路)。
180+
fn render_rows(rows: &[(String, String, Option<f32>)]) -> String {
181+
let mut out = String::from("样本,路,CER\n");
182+
for (name, route, c) in rows {
183+
match c {
184+
Some(v) => out.push_str(&format!("{name},{route},{v:.4}\n")),
185+
None => out.push_str(&format!("{name},{route},-\n")),
186+
}
187+
}
188+
out
189+
}
190+
191+
/// 读取基线 JSON 的 mean_cer(缺失/坏文件 → None,诚实不可比)。
192+
fn read_baseline_mean(path: &Path) -> Option<f32> {
193+
let raw = fs::read_to_string(path).ok()?;
194+
let v: serde_json::Value = serde_json::from_str(&raw).ok()?;
195+
v.get("mean_cer")?.as_f64().map(|x| x as f32)
196+
}
197+
198+
/// 收集样本:wav 文件 + 同名 srt(存在且 UTF-8 可解)→ Sample;
199+
/// 无参考/不可解 → no_ref 列表(只列名不产 CER)。
200+
fn collect_samples(opts: &Options) -> (Vec<Sample>, Vec<String>) {
201+
let mut samples = Vec::new();
202+
let mut skips = Vec::new();
203+
let Ok(entries) = fs::read_dir(&opts.samples_dir) else {
204+
eprintln!("样本目录不可读: {}", opts.samples_dir.display());
205+
return (samples, skips);
206+
};
207+
let mut names: Vec<String> = entries
208+
.filter_map(|e| e.ok())
209+
.map(|e| e.file_name().to_string_lossy().to_string())
210+
.filter(|n| is_media_file(n) && n.to_ascii_lowercase().ends_with(".wav"))
211+
.collect();
212+
names.sort();
213+
for name in names {
214+
let wav = opts.samples_dir.join(&name);
215+
let Some(srt_name) = srt_path_for(&name) else { continue };
216+
let srt_path = opts.samples_dir.join(srt_name);
217+
let raw = match fs::read(&srt_path) {
218+
Ok(b) => b,
219+
Err(_) => {
220+
skips.push(format!("{name}(无同名 srt)"));
221+
continue;
222+
}
223+
};
224+
let srt_text = match String::from_utf8(raw) {
225+
Ok(t) => t,
226+
Err(_) => {
227+
skips.push(format!("{name}(srt 非 UTF-8,GBK 留 P1)"));
228+
continue;
229+
}
230+
};
231+
let reference = reference_text(&parse_srt(&srt_text));
232+
if reference.trim().is_empty() {
233+
skips.push(format!("{name}(srt 解析为空参考)"));
234+
continue;
235+
}
236+
samples.push(Sample { name, wav, reference });
237+
}
238+
(samples, skips)
239+
}
240+
241+
/// 参数解析:位置参数=样本目录;--model/--preproc/--baseline/--update-baseline/--out。
242+
fn parse_args() -> Option<Options> {
243+
let args: Vec<String> = std::env::args().skip(1).collect();
244+
let mut samples_dir = None;
245+
let mut model_dir = DEFAULT_MODEL_DIR.to_string();
246+
let mut out_dir = "asr_eval_out".to_string();
247+
let mut baseline = None;
248+
let mut update_baseline = false;
249+
let mut preproc = Route::Both;
250+
let mut it = args.iter();
251+
while let Some(a) = it.next() {
252+
match a.as_str() {
253+
"--model" => model_dir = it.next()?.clone(),
254+
"--out" => out_dir = it.next()?.clone(),
255+
"--baseline" => baseline = Some(PathBuf::from(it.next()?.clone())),
256+
"--update-baseline" => update_baseline = true,
257+
"--preproc" => {
258+
preproc = match it.next()?.as_str() {
259+
"off" => Route::Off,
260+
"on" => Route::On,
261+
"both" => Route::Both,
262+
_ => return None,
263+
};
264+
}
265+
_ if samples_dir.is_none() => samples_dir = Some(a.clone()),
266+
_ => {
267+
eprintln!("用法: asr_eval <样本目录> [--model <dir>] [--preproc off|on|both] [--baseline <json>] [--update-baseline] [--out <dir>]");
268+
return None;
269+
}
270+
}
271+
}
272+
Some(Options {
273+
samples_dir: PathBuf::from(samples_dir?),
274+
model_dir: PathBuf::from(model_dir),
275+
out_dir: PathBuf::from(out_dir),
276+
baseline,
277+
update_baseline,
278+
preproc,
279+
})
280+
}
281+
282+
/// 加载 SenseVoice 离线识别器(与 cer_bench 同约定:model.int8.onnx + tokens.txt)。
283+
fn load_recognizer(model_dir: &Path) -> OfflineRecognizer {
284+
let mut config = OfflineRecognizerConfig::default();
285+
config.model_config.sense_voice = OfflineSenseVoiceModelConfig {
286+
model: Some(model_dir.join("model.int8.onnx").to_string_lossy().to_string()),
287+
language: Some("auto".into()),
288+
use_itn: true,
289+
};
290+
config.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().to_string());
291+
OfflineRecognizer::create(&config).expect("SenseVoice 模型加载失败(检查 --model 目录)")
292+
}
293+
294+
/// 单路转写(preproc=true 走业务链同参预处理:AGC 目标 RMS 0.125)。
295+
fn transcribe(
296+
recognizer: &OfflineRecognizer,
297+
samples: &[f32],
298+
sample_rate: i32,
299+
preproc_on: bool,
300+
) -> String {
301+
let audio: Vec<f32> = if preproc_on {
302+
let mut p = AudioPreprocessor::new(AudioPreprocessConfig { enabled: true, target_rms: 0.125 });
303+
p.process(samples, 0.005).samples
304+
} else {
305+
samples.to_vec()
306+
};
307+
let stream = recognizer.create_stream();
308+
stream.accept_waveform(sample_rate, &audio);
309+
recognizer.decode(&stream);
310+
stream
311+
.get_result()
312+
.map(|r| r.text)
313+
.unwrap_or_default()
314+
.trim()
315+
.to_string()
316+
}

0 commit comments

Comments
 (0)