|
| 1 | +//! asr_eval 自验证 harness(v0.20.0 / REQ-263,M2)。 |
| 2 | +//! |
| 3 | +//! @ai-context: 目的——"无人工语料也能测 ASR"(2026-09-03 用户裁决①): |
| 4 | +//! 批量样本(wav + 同名 .srt 外挂字幕 = 参考信道 L1 ~100% 无损) |
| 5 | +//! → SenseVoice 离线转写(预处理开/关两路,A/B)→ CER + 混淆画像 |
| 6 | +//! → 汇总/A/B 对比/画像 top-N → 相对基线回归门(退出码契约)。 |
| 7 | +//! @ai-context: 复用现成件——app_lib::cer(CER/推荐)、app_lib::eval_*(画像/ |
| 8 | +//! SRT/统计/回归门)、app_lib::audio_preprocess(A/B 预处理路); |
| 9 | +//! 引擎装载与转写沿用 bin/cer_bench.rs 同款 API(独立 bin 范式)。 |
| 10 | +//! @ai-context: 局限(诚实登记)——①无参考样本只列表不产 CER;②SRT 仅 UTF-8/ |
| 11 | +//! BOM(GBK 需 encoding 依赖,留 P1);③漂移分布需带时间轴的 |
| 12 | +//! 会话信道(--db 会话模式,M2b 接线);④结论只做相对对比与回归, |
| 13 | +//! 不宣称绝对指标。 |
| 14 | +//! @ai-context: 用法:cargo run --bin asr_eval -- <样本目录> [--model <dir>] |
| 15 | +//! [--preproc off|on|both] [--baseline <json>] [--update-baseline] |
| 16 | +//! [--out <目录>] |
| 17 | +//! @ai-context: 退出码:0=通过(含无参考样本的诚实告警);1=基线回归退化 |
| 18 | +//! (均值 CER 超容差)或模型/运行失败;2=用法错误。 |
| 19 | +
|
| 20 | +use std::fs; |
| 21 | +use std::path::{Path, PathBuf}; |
| 22 | +use std::process::ExitCode; |
| 23 | + |
| 24 | +use app_lib::audio_preprocess::{AudioPreprocessConfig, AudioPreprocessor}; |
| 25 | +use app_lib::cer; |
| 26 | +use app_lib::eval_confusion::{profile, ConfusionAggregator}; |
| 27 | +use app_lib::eval_report::{ab_verdict, cer_stats, is_regression, CerStats}; |
| 28 | +use app_lib::eval_samples::{is_media_file, parse_srt, reference_text, srt_path_for}; |
| 29 | +use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig, OfflineSenseVoiceModelConfig}; |
| 30 | + |
| 31 | +/// 默认 SenseVoice 模型目录(与 cer_bench/download 脚本约定一致)。 |
| 32 | +const DEFAULT_MODEL_DIR: &str = "models/asr/sensevoice"; |
| 33 | +/// 基线回归容差(v0.20.0 契约:相对基线均值 CER 退化 >2% 即失败)。 |
| 34 | +const REGRESSION_TOLERANCE: f32 = 0.02; |
| 35 | + |
| 36 | +/// A/B 路选择。 |
| 37 | +#[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 38 | +enum Route { |
| 39 | + Off, |
| 40 | + On, |
| 41 | + Both, |
| 42 | +} |
| 43 | + |
| 44 | +impl Route { |
| 45 | + fn names(&self) -> Vec<&'static str> { |
| 46 | + match self { |
| 47 | + Route::Off => vec!["preproc_off"], |
| 48 | + Route::On => vec!["preproc_on"], |
| 49 | + Route::Both => vec!["preproc_off", "preproc_on"], |
| 50 | + } |
| 51 | + } |
| 52 | +} |
| 53 | + |
| 54 | +#[derive(Debug)] |
| 55 | +struct Options { |
| 56 | + samples_dir: PathBuf, |
| 57 | + model_dir: PathBuf, |
| 58 | + out_dir: PathBuf, |
| 59 | + baseline: Option<PathBuf>, |
| 60 | + update_baseline: bool, |
| 61 | + preproc: Route, |
| 62 | +} |
| 63 | + |
| 64 | +/// 单样本(文件对信道)。 |
| 65 | +struct Sample { |
| 66 | + name: String, |
| 67 | + wav: PathBuf, |
| 68 | + reference: String, |
| 69 | +} |
| 70 | + |
| 71 | +fn main() -> ExitCode { |
| 72 | + let opts = match parse_args() { |
| 73 | + Some(o) => o, |
| 74 | + None => return ExitCode::from(2), |
| 75 | + }; |
| 76 | + let _ = fs::create_dir_all(&opts.out_dir); |
| 77 | + |
| 78 | + let (samples, no_ref_skips) = collect_samples(&opts); |
| 79 | + if samples.is_empty() { |
| 80 | + println!("无可用样本(wav+srt 配对为空);跳过列表:{no_ref_skips:?}"); |
| 81 | + return ExitCode::SUCCESS; |
| 82 | + } |
| 83 | + |
| 84 | + let recognizer = load_recognizer(&opts.model_dir); |
| 85 | + let mut out = String::new(); |
| 86 | + let mut rows: Vec<(String, String, Option<f32>)> = Vec::new(); |
| 87 | + let mut cers: std::collections::BTreeMap<String, Vec<f32>> = Default::default(); |
| 88 | + let mut portraits: std::collections::BTreeMap<String, ConfusionAggregator> = Default::default(); |
| 89 | + |
| 90 | + for s in &samples { |
| 91 | + let Some(wave) = sherpa_onnx::Wave::read(&s.wav.to_string_lossy()) else { |
| 92 | + println!("[跳过] 音频解析失败: {}", s.wav.display()); |
| 93 | + continue; |
| 94 | + }; |
| 95 | + let (sr, samples_f32) = (wave.sample_rate(), wave.samples()); |
| 96 | + for route in opts.preproc.names() { |
| 97 | + let preproc_on = route == "preproc_on"; |
| 98 | + let text = transcribe(&recognizer, samples_f32, sr, preproc_on); |
| 99 | + let c = cer::cer(&s.reference, &text); |
| 100 | + rows.push((s.name.clone(), route.to_string(), c)); |
| 101 | + if let Some(v) = c { |
| 102 | + cers.entry(route.to_string()).or_default().push(v); |
| 103 | + } |
| 104 | + portraits |
| 105 | + .entry(route.to_string()) |
| 106 | + .or_default() |
| 107 | + .add(&profile(&s.reference, &text)); |
| 108 | + } |
| 109 | + print!("{}. ", s.name); |
| 110 | + } |
| 111 | + |
| 112 | + // ── 汇总渲染 ── |
| 113 | + out.push_str(&render_rows(&rows)); |
| 114 | + out.push_str("\n== 各样本行 ==\n"); |
| 115 | + for (name, route, c) in &rows { |
| 116 | + match c { |
| 117 | + Some(v) => out.push_str(&format!("{name}\t{route}\t{v:.4}\n")), |
| 118 | + None => out.push_str(&format!("{name}\t{route}\t-\n")), |
| 119 | + } |
| 120 | + } |
| 121 | + for route in opts.preproc.names() { |
| 122 | + let stats = cer_stats(cers.get(route).map(|v| v.as_slice()).unwrap_or(&[])); |
| 123 | + out.push_str(&format!( |
| 124 | + "\n[{}] 样本={} CER均值={} 范围={}~{}", |
| 125 | + route, |
| 126 | + stats.map(|s: CerStats| s.n.to_string()).unwrap_or_else(|| "0".into()), |
| 127 | + stats.map(|s| format!("{:.4}", s.mean)).unwrap_or_else(|| "不可比".into()), |
| 128 | + stats.map(|s| format!("{:.4}", s.min)).unwrap_or_else(|| "-".into()), |
| 129 | + stats.map(|s| format!("{:.4}", s.max)).unwrap_or_else(|| "-".into()), |
| 130 | + )); |
| 131 | + let agg = portraits.get(route).cloned().unwrap_or_default(); |
| 132 | + let top = agg.top_n(10); |
| 133 | + if top.is_empty() { |
| 134 | + out.push_str(&format!("\n[{}] 画像:无(样本不足或全对)\n", route)); |
| 135 | + } else { |
| 136 | + out.push_str(&format!("\n[{}] 画像 top-10:", route)); |
| 137 | + for ((a, b), n) in top { |
| 138 | + out.push_str(&format!(" {a}→{b}({n})")); |
| 139 | + } |
| 140 | + out.push('\n'); |
| 141 | + } |
| 142 | + } |
| 143 | + // A/B 结论(仅 both 时) |
| 144 | + if opts.preproc == Route::Both { |
| 145 | + let a = cer_stats(cers.get("preproc_on").map(|v| v.as_slice()).unwrap_or(&[])); |
| 146 | + let b = cer_stats(cers.get("preproc_off").map(|v| v.as_slice()).unwrap_or(&[])); |
| 147 | + out.push_str(&format!("\n{}\n", ab_verdict("A/B(preproc 开 vs 关)", a, b))); |
| 148 | + } |
| 149 | + if !no_ref_skips.is_empty() { |
| 150 | + out.push_str(&format!("\n无参考样本(不产 CER):{no_ref_skips:?}\n")); |
| 151 | + } |
| 152 | + |
| 153 | + println!("{out}"); |
| 154 | + let _ = fs::write(opts.out_dir.join("summary.txt"), &out); |
| 155 | + |
| 156 | + // ── 基线回归门 ── |
| 157 | + let primary = cers.get("preproc_off").map(|v| cer_stats(v)).flatten(); |
| 158 | + if let Some(bp) = &opts.baseline { |
| 159 | + let cur = primary.map(|s| s.mean); |
| 160 | + let base_mean = read_baseline_mean(bp); |
| 161 | + match is_regression(cur, base_mean, REGRESSION_TOLERANCE) { |
| 162 | + Some(true) => { |
| 163 | + println!("[回归门] 失败:均值 CER {:.4} 相对基线 {:.4} 退化超容差", cur.unwrap(), base_mean.unwrap()); |
| 164 | + return ExitCode::from(1); |
| 165 | + } |
| 166 | + Some(false) => println!("[回归门] 通过(均值 CER {:.4} vs 基线 {:.4})", cur.unwrap(), base_mean.unwrap()), |
| 167 | + None => println!("[回归门] 不可比(无参考/无基线数据),跳过"), |
| 168 | + } |
| 169 | + } |
| 170 | + if opts.update_baseline { |
| 171 | + let mean = primary.map(|s| format!("{:.4}", s.mean)).unwrap_or_else(|| "null".into()); |
| 172 | + let json = format!("{{\"route\":\"preproc_off\",\"mean_cer\":{mean},\"samples\":{}}}\n", primary.map(|s| s.n).unwrap_or(0)); |
| 173 | + let _ = fs::write(opts.out_dir.join("baseline.json"), json); |
| 174 | + println!("[基线] 已写 {}/baseline.json", opts.out_dir.display()); |
| 175 | + } |
| 176 | + ExitCode::SUCCESS |
| 177 | +} |
| 178 | + |
| 179 | +/// 汇总表渲染(CSV 行序:样本 × 路)。 |
| 180 | +fn render_rows(rows: &[(String, String, Option<f32>)]) -> String { |
| 181 | + let mut out = String::from("样本,路,CER\n"); |
| 182 | + for (name, route, c) in rows { |
| 183 | + match c { |
| 184 | + Some(v) => out.push_str(&format!("{name},{route},{v:.4}\n")), |
| 185 | + None => out.push_str(&format!("{name},{route},-\n")), |
| 186 | + } |
| 187 | + } |
| 188 | + out |
| 189 | +} |
| 190 | + |
| 191 | +/// 读取基线 JSON 的 mean_cer(缺失/坏文件 → None,诚实不可比)。 |
| 192 | +fn read_baseline_mean(path: &Path) -> Option<f32> { |
| 193 | + let raw = fs::read_to_string(path).ok()?; |
| 194 | + let v: serde_json::Value = serde_json::from_str(&raw).ok()?; |
| 195 | + v.get("mean_cer")?.as_f64().map(|x| x as f32) |
| 196 | +} |
| 197 | + |
| 198 | +/// 收集样本:wav 文件 + 同名 srt(存在且 UTF-8 可解)→ Sample; |
| 199 | +/// 无参考/不可解 → no_ref 列表(只列名不产 CER)。 |
| 200 | +fn collect_samples(opts: &Options) -> (Vec<Sample>, Vec<String>) { |
| 201 | + let mut samples = Vec::new(); |
| 202 | + let mut skips = Vec::new(); |
| 203 | + let Ok(entries) = fs::read_dir(&opts.samples_dir) else { |
| 204 | + eprintln!("样本目录不可读: {}", opts.samples_dir.display()); |
| 205 | + return (samples, skips); |
| 206 | + }; |
| 207 | + let mut names: Vec<String> = entries |
| 208 | + .filter_map(|e| e.ok()) |
| 209 | + .map(|e| e.file_name().to_string_lossy().to_string()) |
| 210 | + .filter(|n| is_media_file(n) && n.to_ascii_lowercase().ends_with(".wav")) |
| 211 | + .collect(); |
| 212 | + names.sort(); |
| 213 | + for name in names { |
| 214 | + let wav = opts.samples_dir.join(&name); |
| 215 | + let Some(srt_name) = srt_path_for(&name) else { continue }; |
| 216 | + let srt_path = opts.samples_dir.join(srt_name); |
| 217 | + let raw = match fs::read(&srt_path) { |
| 218 | + Ok(b) => b, |
| 219 | + Err(_) => { |
| 220 | + skips.push(format!("{name}(无同名 srt)")); |
| 221 | + continue; |
| 222 | + } |
| 223 | + }; |
| 224 | + let srt_text = match String::from_utf8(raw) { |
| 225 | + Ok(t) => t, |
| 226 | + Err(_) => { |
| 227 | + skips.push(format!("{name}(srt 非 UTF-8,GBK 留 P1)")); |
| 228 | + continue; |
| 229 | + } |
| 230 | + }; |
| 231 | + let reference = reference_text(&parse_srt(&srt_text)); |
| 232 | + if reference.trim().is_empty() { |
| 233 | + skips.push(format!("{name}(srt 解析为空参考)")); |
| 234 | + continue; |
| 235 | + } |
| 236 | + samples.push(Sample { name, wav, reference }); |
| 237 | + } |
| 238 | + (samples, skips) |
| 239 | +} |
| 240 | + |
| 241 | +/// 参数解析:位置参数=样本目录;--model/--preproc/--baseline/--update-baseline/--out。 |
| 242 | +fn parse_args() -> Option<Options> { |
| 243 | + let args: Vec<String> = std::env::args().skip(1).collect(); |
| 244 | + let mut samples_dir = None; |
| 245 | + let mut model_dir = DEFAULT_MODEL_DIR.to_string(); |
| 246 | + let mut out_dir = "asr_eval_out".to_string(); |
| 247 | + let mut baseline = None; |
| 248 | + let mut update_baseline = false; |
| 249 | + let mut preproc = Route::Both; |
| 250 | + let mut it = args.iter(); |
| 251 | + while let Some(a) = it.next() { |
| 252 | + match a.as_str() { |
| 253 | + "--model" => model_dir = it.next()?.clone(), |
| 254 | + "--out" => out_dir = it.next()?.clone(), |
| 255 | + "--baseline" => baseline = Some(PathBuf::from(it.next()?.clone())), |
| 256 | + "--update-baseline" => update_baseline = true, |
| 257 | + "--preproc" => { |
| 258 | + preproc = match it.next()?.as_str() { |
| 259 | + "off" => Route::Off, |
| 260 | + "on" => Route::On, |
| 261 | + "both" => Route::Both, |
| 262 | + _ => return None, |
| 263 | + }; |
| 264 | + } |
| 265 | + _ if samples_dir.is_none() => samples_dir = Some(a.clone()), |
| 266 | + _ => { |
| 267 | + eprintln!("用法: asr_eval <样本目录> [--model <dir>] [--preproc off|on|both] [--baseline <json>] [--update-baseline] [--out <dir>]"); |
| 268 | + return None; |
| 269 | + } |
| 270 | + } |
| 271 | + } |
| 272 | + Some(Options { |
| 273 | + samples_dir: PathBuf::from(samples_dir?), |
| 274 | + model_dir: PathBuf::from(model_dir), |
| 275 | + out_dir: PathBuf::from(out_dir), |
| 276 | + baseline, |
| 277 | + update_baseline, |
| 278 | + preproc, |
| 279 | + }) |
| 280 | +} |
| 281 | + |
| 282 | +/// 加载 SenseVoice 离线识别器(与 cer_bench 同约定:model.int8.onnx + tokens.txt)。 |
| 283 | +fn load_recognizer(model_dir: &Path) -> OfflineRecognizer { |
| 284 | + let mut config = OfflineRecognizerConfig::default(); |
| 285 | + config.model_config.sense_voice = OfflineSenseVoiceModelConfig { |
| 286 | + model: Some(model_dir.join("model.int8.onnx").to_string_lossy().to_string()), |
| 287 | + language: Some("auto".into()), |
| 288 | + use_itn: true, |
| 289 | + }; |
| 290 | + config.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().to_string()); |
| 291 | + OfflineRecognizer::create(&config).expect("SenseVoice 模型加载失败(检查 --model 目录)") |
| 292 | +} |
| 293 | + |
| 294 | +/// 单路转写(preproc=true 走业务链同参预处理:AGC 目标 RMS 0.125)。 |
| 295 | +fn transcribe( |
| 296 | + recognizer: &OfflineRecognizer, |
| 297 | + samples: &[f32], |
| 298 | + sample_rate: i32, |
| 299 | + preproc_on: bool, |
| 300 | +) -> String { |
| 301 | + let audio: Vec<f32> = if preproc_on { |
| 302 | + let mut p = AudioPreprocessor::new(AudioPreprocessConfig { enabled: true, target_rms: 0.125 }); |
| 303 | + p.process(samples, 0.005).samples |
| 304 | + } else { |
| 305 | + samples.to_vec() |
| 306 | + }; |
| 307 | + let stream = recognizer.create_stream(); |
| 308 | + stream.accept_waveform(sample_rate, &audio); |
| 309 | + recognizer.decode(&stream); |
| 310 | + stream |
| 311 | + .get_result() |
| 312 | + .map(|r| r.text) |
| 313 | + .unwrap_or_default() |
| 314 | + .trim() |
| 315 | + .to_string() |
| 316 | +} |
0 commit comments