|
| 1 | +//! NDJSON 流式喂入缓冲(观察 2026-09-05-2,v0.19.7 审查收口)。 |
| 2 | +//! |
| 3 | +//! @ai-context: 精修逐节流式的行缓冲逻辑从 adapter 闭包抽为纯函数(可单测): |
| 4 | +//! chunk 可能切在行中/行间/CRLF 边界;`feed_ndjson` 只解析完整行, |
| 5 | +//! 残留留 pending;`flush_ndjson` 处理末行无换行与尾随垃圾。 |
| 6 | +
|
| 7 | +use crate::ai_refine_protocol::{AiRefineSection, parse_section_ndjson_line}; |
| 8 | + |
| 9 | +/// 喂入增量文本:解析全部完整行并入 sink(未换行残留在 pending)。 |
| 10 | +pub fn feed_ndjson(pending: &mut String, chunk: &str, sink: &mut Vec<AiRefineSection>) { |
| 11 | + pending.push_str(chunk); |
| 12 | + while let Some(pos) = pending.find('\n') { |
| 13 | + let line: String = pending.drain(..=pos).collect(); |
| 14 | + if let Some(sec) = parse_section_ndjson_line(&line) { |
| 15 | + sink.push(sec); |
| 16 | + } |
| 17 | + } |
| 18 | +} |
| 19 | + |
| 20 | +/// 流结束 flush:解析剩余(末行无换行/模型尾部垃圾行忽略)。 |
| 21 | +pub fn flush_ndjson(pending: &mut String, sink: &mut Vec<AiRefineSection>) { |
| 22 | + if pending.is_empty() { |
| 23 | + return; |
| 24 | + } |
| 25 | + let tail = std::mem::take(pending); |
| 26 | + if let Some(sec) = parse_section_ndjson_line(&tail) { |
| 27 | + sink.push(sec); |
| 28 | + } |
| 29 | +} |
| 30 | + |
| 31 | +#[cfg(test)] |
| 32 | +mod tests { |
| 33 | + use super::*; |
| 34 | + |
| 35 | + fn sec(heading: &str) -> AiRefineSection { |
| 36 | + // 解析侧只关心 heading/blocks 形状——空 blocks 元素同构 |
| 37 | + let line = format!(r#"{{"heading":"{}","blocks":[]}}"#, heading); |
| 38 | + parse_section_ndjson_line(&line).expect("构造节失败") |
| 39 | + } |
| 40 | + |
| 41 | + #[test] |
| 42 | + fn chunk_split_across_lines_accumulates() { |
| 43 | + let mut pending = String::new(); |
| 44 | + let mut sink = Vec::new(); |
| 45 | + // 第 1 个对象被切成两半 + 第 2 个对象完整 |
| 46 | + feed_ndjson(&mut pending, r#"{"heading":"节A","blocks":[]}"#, &mut sink); |
| 47 | + feed_ndjson(&mut pending, "\n", &mut sink); |
| 48 | + feed_ndjson(&mut pending, r#"{"heading":"节B","blocks":[]}"#, &mut sink); |
| 49 | + assert_eq!(sink.len(), 0, "首行未换行前不解析"); |
| 50 | + feed_ndjson(&mut pending, "\n", &mut sink); |
| 51 | + assert_eq!(sink.len(), 2); |
| 52 | + assert_eq!(sink[0], sec("节A")); |
| 53 | + assert_eq!(sink[1], sec("节B")); |
| 54 | + } |
| 55 | + |
| 56 | + #[test] |
| 57 | + fn crlf_and_tail_garbage_handled() { |
| 58 | + let mut pending = String::new(); |
| 59 | + let mut sink = Vec::new(); |
| 60 | + feed_ndjson(&mut pending, "{\"heading\":\"A\",\"blocks\":[]}\r\n{\"heading\":\"B\",\"blocks\":[]}\n", &mut sink); |
| 61 | + assert_eq!(sink.len(), 2, "CRLF 尾部 \\r 应被 trim 忽略"); |
| 62 | + // 末行无换行 + 尾随解释文本 → flush 只取可解析对象 |
| 63 | + feed_ndjson(&mut pending, "{\"heading\":\"C\",\"blocks\":[]}", &mut sink); |
| 64 | + feed_ndjson(&mut pending, "(以上为整理结果)", &mut sink); |
| 65 | + flush_ndjson(&mut pending, &mut sink); |
| 66 | + assert_eq!(sink.len(), 3); |
| 67 | + assert_eq!(sink[2], sec("C")); |
| 68 | + assert!(pending.is_empty()); |
| 69 | + } |
| 70 | + |
| 71 | + #[test] |
| 72 | + fn compact_array_lines_rejected_per_line() { |
| 73 | + // 模型输出完整数组(违反逐节约定)→ 行级全部拒绝;整包回退由 |
| 74 | + // adapter 层按全文解析(见 refine_stream_ndjson) |
| 75 | + let mut pending = String::new(); |
| 76 | + let mut sink = Vec::new(); |
| 77 | + feed_ndjson(&mut pending, "[{\"heading\":\"A\",\"blocks\":[]}]", &mut sink); |
| 78 | + flush_ndjson(&mut pending, &mut sink); |
| 79 | + assert!(sink.is_empty()); |
| 80 | + } |
| 81 | +} |
0 commit comments