|
| 1 | +//! ASR 同音混淆画像闭环(v0.20.2 / REQ-269)。 |
| 2 | +//! |
| 3 | +//! @ai-context: 画像源——用户「采纳」的离线精修草稿(REQ-268 裁决流): |
| 4 | +//! 旧文本(被换掉)与精修文本(用户选中的更优面)做字符级对齐, |
| 5 | +//! 连续错位段聚成词级对 (wrong, right)——采纳=有标注的弱参考, |
| 6 | +//! 画像即真实产出,无人工标注成本。 |
| 7 | +//! @ai-context: 闭环三步——① 聚合画像(词级对 + 次数,JSON 可校准);② 候选 |
| 8 | +//! (次数 ≥ 门槛、未确认未忽略)经确认制进规则表(OCR ocr_confusion |
| 9 | +//! 哲学迁移:共现才替换——from 只在「正确词 to 已出现在同文语料」时 |
| 10 | +//! 才替换,无共现不猜,防错纠);③ 反哺 hotwords 由命令层完成 |
| 11 | +//! (确认即把 to 加入热词注入流式 ASR,下次识别自带正确词先验)。 |
| 12 | +//! @ai-context: 只读消费面=产物文本组装(commands_session_note 转换/预览), |
| 13 | +//! 原料 session_segments 永不变(与 REQ-268 同可逆契约)。 |
| 14 | +//! @ai-context: 纯逻辑 + JSON IO(路径注入,测试用内存/temp);无网络依赖。 |
| 15 | +
|
| 16 | +use serde::{Deserialize, Serialize}; |
| 17 | + |
| 18 | +/// 词级混淆对统计(wrong=被换掉的旧文本词段;right=用户采纳的新词段)。 |
| 19 | +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] |
| 20 | +pub struct AsrPair { |
| 21 | + pub wrong: String, |
| 22 | + pub right: String, |
| 23 | + /// 出现次数(采纳流聚合) |
| 24 | + pub count: u64, |
| 25 | +} |
| 26 | + |
| 27 | +/// 已确认纠错规则(from → to;from 为待替换词形,to 为正确词)。 |
| 28 | +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 29 | +pub struct AsrRule { |
| 30 | + pub from: String, |
| 31 | + pub to: String, |
| 32 | +} |
| 33 | + |
| 34 | +/// 画像 + 规则 + 忽略单(单文件 JSON 校准,data_dir/asr_confusion.json)。 |
| 35 | +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] |
| 36 | +pub struct AsrConfusionStore { |
| 37 | + /// 词级画像(wrong→right 计数) |
| 38 | + #[serde(default)] |
| 39 | + pub pairs: Vec<AsrPair>, |
| 40 | + /// 已确认规则(用户裁决——机器不自动生效) |
| 41 | + #[serde(default)] |
| 42 | + pub rules: Vec<AsrRule>, |
| 43 | + /// 已忽略候选键("wrong|right"——不再推荐) |
| 44 | + #[serde(default)] |
| 45 | + pub dismissed: Vec<String>, |
| 46 | +} |
| 47 | + |
| 48 | +/// 候选次数门槛(防单次噪声;与 OCR 画像 min_count 先例同哲学)。 |
| 49 | +pub const MIN_CANDIDATE_COUNT: u64 = 2; |
| 50 | +/// 词级对最大长度(超长=对齐错位噪声,不提名——宁漏勿错)。 |
| 51 | +pub const MAX_RUN_CHARS: usize = 8; |
| 52 | + |
| 53 | +impl AsrConfusionStore { |
| 54 | + /// 从磁盘加载(缺失/损坏 → 空画像与规则,不阻断启动——校准文件可删)。 |
| 55 | + pub fn load(path: &std::path::Path) -> Self { |
| 56 | + match std::fs::read_to_string(path) { |
| 57 | + Ok(raw) => serde_json::from_str(&raw).unwrap_or_default(), |
| 58 | + Err(_) => Self::default(), |
| 59 | + } |
| 60 | + } |
| 61 | + |
| 62 | + /// 原子写(先 tmp 后 rename——防写一半损坏;vocab.json 同款)。 |
| 63 | + pub fn save(&self, path: &std::path::Path) -> crate::error::Result<()> { |
| 64 | + let raw = serde_json::to_string_pretty(self) |
| 65 | + .map_err(|e| crate::error::AppError::Io(format!("序列化 ASR 混淆表失败: {}", e)))?; |
| 66 | + if let Some(parent) = path.parent() { |
| 67 | + std::fs::create_dir_all(parent)?; |
| 68 | + } |
| 69 | + let tmp = path.with_extension("json.tmp"); |
| 70 | + std::fs::write(&tmp, raw)?; |
| 71 | + std::fs::rename(&tmp, path)?; |
| 72 | + Ok(()) |
| 73 | + } |
| 74 | + |
| 75 | + /// 记录一次采纳差异(old=被换旧文,new=采纳精修文——new 侧为弱参考): |
| 76 | + /// 聚合词级对计数;返回新增对数。 |
| 77 | + pub fn record_adoption(&mut self, old: &str, new: &str) -> usize { |
| 78 | + let mut added = 0; |
| 79 | + for (wrong, right) in word_pairs(old, new) { |
| 80 | + match self.pairs.iter_mut().find(|p| p.wrong == wrong && p.right == right) { |
| 81 | + Some(p) => p.count += 1, |
| 82 | + None => { |
| 83 | + self.pairs.push(AsrPair { wrong, right, count: 1 }); |
| 84 | + added += 1; |
| 85 | + } |
| 86 | + } |
| 87 | + } |
| 88 | + added |
| 89 | + } |
| 90 | + |
| 91 | + /// 候选(次数达标、未被确认/忽略;按次数降序,最多 limit 条)。 |
| 92 | + pub fn candidates(&self, limit: usize) -> Vec<&AsrPair> { |
| 93 | + let mut list: Vec<&AsrPair> = self |
| 94 | + .pairs |
| 95 | + .iter() |
| 96 | + .filter(|p| { |
| 97 | + p.count >= MIN_CANDIDATE_COUNT |
| 98 | + && !self.rules.iter().any(|r| r.from == p.wrong) |
| 99 | + && !self.dismissed.iter().any(|k| k == &pair_key(&p.wrong, &p.right)) |
| 100 | + }) |
| 101 | + .collect(); |
| 102 | + list.sort_by_key(|p| std::cmp::Reverse(p.count)); |
| 103 | + list.truncate(limit); |
| 104 | + list |
| 105 | + } |
| 106 | + |
| 107 | + /// 确认候选 → 规则表(from=wrong,to=right;重复 from 覆盖——现场校准优先); |
| 108 | + /// 返回 true=生效。 |
| 109 | + pub fn confirm(&mut self, wrong: &str, right: &str) -> bool { |
| 110 | + if wrong.trim().is_empty() || right.trim().is_empty() || wrong == right { |
| 111 | + return false; |
| 112 | + } |
| 113 | + let from = wrong.trim(); |
| 114 | + let to = right.trim(); |
| 115 | + if let Some(r) = self.rules.iter_mut().find(|r| r.from == from) { |
| 116 | + r.to = to.to_string(); |
| 117 | + } else { |
| 118 | + self.rules.push(AsrRule { from: from.to_string(), to: to.to_string() }); |
| 119 | + } |
| 120 | + // 规则已生效的画像对不再推荐(候选过滤器同样拦截) |
| 121 | + self.pairs.retain(|p| p.wrong != from || p.right != to); |
| 122 | + true |
| 123 | + } |
| 124 | + |
| 125 | + /// 忽略候选(仅记录,不删历史画像——未来可再评估)。 |
| 126 | + pub fn dismiss(&mut self, wrong: &str, right: &str) { |
| 127 | + let key = pair_key(wrong, right); |
| 128 | + if !self.dismissed.iter().any(|k| *k == key) { |
| 129 | + self.dismissed.push(key.clone()); |
| 130 | + } |
| 131 | + self.pairs.retain(|p| pair_key(&p.wrong, &p.right) != key); |
| 132 | + } |
| 133 | + |
| 134 | + /// 删除规则(恢复画像原样——只回规则表,历史计数已在 confirm 时移除)。 |
| 135 | + pub fn remove_rule(&mut self, from: &str) -> bool { |
| 136 | + let before = self.rules.len(); |
| 137 | + self.rules.retain(|r| r.from != from); |
| 138 | + self.rules.len() != before |
| 139 | + } |
| 140 | +} |
| 141 | + |
| 142 | +fn pair_key(wrong: &str, right: &str) -> String { |
| 143 | + format!("{}|{}", wrong, right) |
| 144 | +} |
| 145 | + |
| 146 | +/// 纠错(纯函数):长词优先顺序替换;**共现才替换**——正确词 to 未出现在 |
| 147 | +/// 语料(同会话全文)中不猜(OCR ocr_correction 哲学迁移:互证缺失=孤立词形, |
| 148 | +/// 可能是特有词而非错词)。 |
| 149 | +pub fn apply_rules(text: &str, corpus: &str, rules: &[AsrRule]) -> String { |
| 150 | + if rules.is_empty() || text.is_empty() || corpus.is_empty() { |
| 151 | + return text.to_string(); |
| 152 | + } |
| 153 | + let mut sorted = rules.to_vec(); |
| 154 | + sorted.sort_by_key(|r| std::cmp::Reverse(r.from.chars().count())); |
| 155 | + let mut out = text.to_string(); |
| 156 | + for r in sorted { |
| 157 | + if !r.from.is_empty() && out.contains(&r.from) && corpus.contains(&r.to) { |
| 158 | + out = out.replace(&r.from, &r.to); |
| 159 | + } |
| 160 | + } |
| 161 | + out |
| 162 | +} |
| 163 | + |
| 164 | +/// 词级对提取(纯函数):两段采纳差异文本 → 连续错位段词对 (old 侧, new 侧)。 |
| 165 | +/// |
| 166 | +/// @ai-context: 复用 eval_confusion::diff_ops(字符级 DP 回溯,去标点口径同 |
| 167 | +/// CER/画像)——连续非匹配操作聚成段;**含单字对**(同音词常见 |
| 168 | +/// 仅首字不同:概念/该念)——单字替换风险由「确认制 + 共现才 |
| 169 | +/// 替换 + JSON 可删」三重把关(OCR 0/O 先例同哲学);超长段 |
| 170 | +/// (> MAX_RUN_CHARS,插删错位)丢弃——宁漏勿错,候选只做提名人。 |
| 171 | +fn word_pairs(old: &str, new: &str) -> Vec<(String, String)> { |
| 172 | + use crate::asr_rescore::strip_punct; |
| 173 | + use crate::eval_confusion::{diff_ops, EditOp}; |
| 174 | + let old_chars = strip_punct(old); |
| 175 | + let new_chars = strip_punct(new); |
| 176 | + if old_chars.len() > 2000 || new_chars.len() > 2000 { |
| 177 | + return Vec::new(); // 超长护栏(同 eval_confusion::MAX_DIFF_CHARS 精神) |
| 178 | + } |
| 179 | + let ops = diff_ops(&old_chars, &new_chars); |
| 180 | + let mut out = Vec::new(); |
| 181 | + let mut run_old: Vec<char> = Vec::new(); |
| 182 | + let mut run_new: Vec<char> = Vec::new(); |
| 183 | + let mut flush = |run_old: &mut Vec<char>, run_new: &mut Vec<char>, out: &mut Vec<(String, String)>| { |
| 184 | + if !run_old.is_empty() |
| 185 | + && !run_new.is_empty() |
| 186 | + && run_old.len() == run_new.len() |
| 187 | + && run_old.len() <= MAX_RUN_CHARS |
| 188 | + { |
| 189 | + out.push((run_old.iter().collect::<String>(), run_new.iter().collect::<String>())); |
| 190 | + } |
| 191 | + run_old.clear(); |
| 192 | + run_new.clear(); |
| 193 | + }; |
| 194 | + for op in ops { |
| 195 | + match op { |
| 196 | + EditOp::Match(_) => flush(&mut run_old, &mut run_new, &mut out), |
| 197 | + EditOp::Sub(a, b) => { |
| 198 | + run_old.push(a); |
| 199 | + run_new.push(b); |
| 200 | + } |
| 201 | + // 插/删不配对——仅计入两侧串影响长度不等,flush 时按规则丢弃 |
| 202 | + EditOp::Ins(c) => run_new.push(c), |
| 203 | + EditOp::Del(c) => run_old.push(c), |
| 204 | + } |
| 205 | + } |
| 206 | + flush(&mut run_old, &mut run_new, &mut out); |
| 207 | + out |
| 208 | +} |
| 209 | + |
| 210 | +#[cfg(test)] |
| 211 | +#[path = "asr_confusion_tests.rs"] |
| 212 | +mod tests; |
0 commit comments