Skip to content

Commit df8d3be

Browse files
committed
feat(ocr): 结构识别接线——屏卡渲染精修产物+停止后自动精修
1 parent d84ad14 commit df8d3be

4 files changed

Lines changed: 144 additions & 2 deletions

File tree

‎app/src-tauri/src/commands_live.rs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,39 @@ pub async fn stop_live_session(state: State<'_, AppState>) -> Result<Option<i64>
9393
.map_err(|e| format!("停止任务失败: {}", e))?
9494
.map_err(|e| e.to_string())?;
9595
stop_clipboard_monitor(&state);
96+
// v0.7.3(REQ-159):停止后自动结构精修(表格/公式区域;模型未就绪/
97+
// 无结构区域 → 内部降级跳过——run_refine 已有完整降级链)
98+
if let Some(sid) = stopped {
99+
trigger_auto_refine(&state, sid);
100+
}
96101
Ok(stopped)
97102
}
98103

104+
/// 停止后自动结构精修触发(v0.7.3 REQ-159)。
105+
///
106+
/// @ai-context: 前置检查:会话存在 region_kind=table/formula 的 OCR 块才触发
107+
/// (无结构区域不白跑);模型就绪检查交给 run_refine 内部
108+
/// decide_refine(未下载 → session:refine-skipped 事件,诚实降级)。
109+
fn trigger_auto_refine(state: &AppState, session_id: i64) {
110+
let has_structure = state
111+
.db
112+
.list_ocr_blocks(session_id)
113+
.map(|blocks| {
114+
blocks.iter().any(|b| {
115+
matches!(b.region_kind.as_deref(), Some("table" | "formula"))
116+
})
117+
})
118+
.unwrap_or(false);
119+
if !has_structure {
120+
return;
121+
}
122+
eprintln!("[Refine] 会话 {} 停止:检测到表格/公式区域,自动触发结构精修", session_id);
123+
let state: AppState = (*state).clone();
124+
tauri::async_runtime::spawn_blocking(move || {
125+
let _ = crate::commands_refine_inner::run_refine(&state, session_id);
126+
});
127+
}
128+
99129
/// 查询实时会话状态(活动会话 id + 预热就绪标记)。
100130
#[tauri::command]
101131
pub fn live_session_status(state: State<'_, AppState>) -> LiveSessionStatus {

‎app/src-tauri/src/commands_session.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,11 @@ pub async fn get_session_detail(state: State<'_, AppState>, id: i64) -> Result<S
8383
// v0.7.3(REQ-160,ADR-015):画面要点屏卡(新数据按 screen_id 分组、
8484
// 旧数据聚类兜底;图匹配失败/目录缺失不阻断——前端无缩略图降级)
8585
let images_dir = state.data_dir.join("session-images").join(id.to_string());
86-
let screens = crate::screens::build_screens(&ocr_blocks, Some(&images_dir));
86+
let mut screens = crate::screens::build_screens(&ocr_blocks, Some(&images_dir));
87+
// v0.7.3(REQ-159):屏内结构块渲染课后精修产物(表格 Markdown/公式 LaTeX;
88+
// 未精修 → 保留原始 OCR 文本,徽标降级)
89+
let artifact = state.db.get_artifact(id).map_err(|e| e.to_string())?;
90+
crate::screens::refine_screen_structures(&mut screens, artifact.as_ref());
8791
Ok(SessionDetail { session, segments, ocr_blocks, events, screens })
8892
}
8993

‎app/src-tauri/src/screens.rs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,50 @@ fn screen_from_cluster(
174174
}
175175
}
176176

177+
/// 屏结构产物匹配(纯函数):课后精修产物 → 屏 structure.rendered 填充。
178+
///
179+
/// @ai-context: 消费 artifact_blocks(kind=Table/Formula,REQ-049/050 精修产物)——
180+
/// 屏卡内结构块渲染 Markdown 表格/LaTeX;匹配 = 产物 refs.frame_ms
181+
/// 落在屏区间内(区域裁剪时刻即屏成员时刻);未命中 → 保留原始
182+
/// OCR 文本(徽标降级);产物缺失 → 无操作(不阻断,M5 接线)。
183+
pub fn refine_screen_structures(
184+
screens: &mut [SessionScreen],
185+
artifact: Option<&crate::artifact::SessionArtifact>,
186+
) {
187+
let Some(art) = artifact else { return };
188+
for s in screens.iter_mut() {
189+
for st in s.structure.iter_mut() {
190+
if st.rendered.is_some() {
191+
continue;
192+
}
193+
let want = match st.kind.as_str() {
194+
"table" => crate::artifact::ArtifactKind::Table,
195+
"formula" => crate::artifact::ArtifactKind::Formula,
196+
_ => continue, // code 区域无精修产物(原生文本展示)
197+
};
198+
let rendered = art
199+
.blocks
200+
.iter()
201+
.find(|b| {
202+
b.kind == want
203+
&& b.refs.frame_ms.is_some_and(|f| {
204+
f >= s.first_seen_ms && f <= s.last_seen_ms
205+
})
206+
})
207+
.and_then(|b| match &b.payload {
208+
crate::artifact::BlockPayload::Table(t) => Some(t.markdown.clone()),
209+
crate::artifact::BlockPayload::Formula(f) => {
210+
Some(format!("$${}$$", f.latex))
211+
}
212+
_ => None,
213+
});
214+
if let Some(r) = rendered {
215+
st.rendered = Some(r);
216+
}
217+
}
218+
}
219+
}
220+
177221
/// 匹配归档 full 图(纯 IO):取时间戳 ≤ 屏首见时刻的最近图;无则取最早图。
178222
///
179223
/// @ai-context: 归档图按时间戳命名(full/{ts}.webp,image_store 约定);屏首帧

‎app/src-tauri/src/screens_tests.rs‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
55
use std::fs;
66

7-
use crate::screens::build_screens;
7+
use crate::screens::{build_screens, refine_screen_structures};
88
use crate::types::{SessionOcrBlock, TextBox};
99

1010
/// 测试辅助:构造会话 OCR 块(region=full,可带 bbox/screen_id/region_kind)。
@@ -145,6 +145,70 @@ fn build_screens_empty_and_no_full_blocks() {
145145
assert!(build_screens(&subtitle_only, Some(&dir)).is_empty());
146146
}
147147

148+
#[test]
149+
fn refine_structures_from_artifact() {
150+
// Arrange:屏内 table/formula 结构块 + 课后精修产物(frame_ms 在屏区间内)
151+
use crate::artifact::{ArtifactBlock, ArtifactKind, BlockPayload, BlockRefs, BlockSource, SessionArtifact};
152+
use crate::table_reconstruct::TableBlock;
153+
let dir = tmp_images_dir("refine", &[]);
154+
let blocks = vec![
155+
blk(1, 2_000, "表格标题", Some(1), Some((100.0, 100.0, 300.0, 40.0)), None),
156+
blk(2, 2_000, "| A | B |", Some(1), Some((100.0, 200.0, 300.0, 60.0)), Some("table")),
157+
blk(3, 2_000, "x^2+1", Some(1), Some((100.0, 300.0, 200.0, 40.0)), Some("formula")),
158+
];
159+
let mut screens = build_screens(&blocks, Some(&dir));
160+
let artifact = SessionArtifact {
161+
session_id: 1,
162+
profile: String::new(),
163+
blocks: vec![
164+
ArtifactBlock {
165+
id: 1,
166+
kind: ArtifactKind::Table,
167+
refs: BlockRefs { segment_id: None, ocr_block_id: None, frame_ms: Some(2_000) },
168+
payload: BlockPayload::Table(TableBlock {
169+
markdown: "| A | B |\n|---|---|\n| 1 | 2 |".to_string(),
170+
structure_confidence: 0.9,
171+
cell_refs: Vec::new(),
172+
}),
173+
order: 0,
174+
source: BlockSource::Local,
175+
},
176+
ArtifactBlock {
177+
id: 2,
178+
kind: ArtifactKind::Formula,
179+
refs: BlockRefs { segment_id: None, ocr_block_id: None, frame_ms: Some(2_000) },
180+
payload: BlockPayload::Formula(crate::formula_reconstruct::FormulaBlock {
181+
latex: "x^2+1".to_string(),
182+
source_text: "x2+1".to_string(),
183+
confidence: 0.8,
184+
}),
185+
order: 1,
186+
source: BlockSource::Local,
187+
},
188+
],
189+
};
190+
// Act
191+
refine_screen_structures(&mut screens, Some(&artifact));
192+
// Assert:表格/公式 rendered 填充
193+
let s = &screens[0];
194+
let table = s.structure.iter().find(|st| st.kind == "table").unwrap();
195+
assert_eq!(table.rendered.as_deref(), Some("| A | B |\n|---|---|\n| 1 | 2 |"));
196+
let formula = s.structure.iter().find(|st| st.kind == "formula").unwrap();
197+
assert_eq!(formula.rendered.as_deref(), Some("$$x^2+1$$"));
198+
}
199+
200+
#[test]
201+
fn refine_structures_none_without_artifact() {
202+
// Arrange:无产物 + 屏结构块
203+
let dir = tmp_images_dir("refine-none", &[]);
204+
let blocks = vec![blk(1, 2_000, "| A |", Some(1), Some((100.0, 200.0, 300.0, 60.0)), Some("table"))];
205+
let mut screens = build_screens(&blocks, Some(&dir));
206+
// Act:None 产物 → 无操作(rendered 保持 None)
207+
refine_screen_structures(&mut screens, None);
208+
// Assert
209+
assert_eq!(screens[0].structure[0].rendered, None);
210+
}
211+
148212
#[test]
149213
fn image_ref_matches_nearest_at_or_before_first_seen() {
150214
// Arrange:归档图 2000/10000/30000;屏 first_seen=36404 → 最近 ≤ 为 30000

0 commit comments

Comments
 (0)