Skip to content

Commit 0ca4fbe

Browse files
committed
feat(ocr): 结构图批量捕获管线——屏内选优+版面过滤+裁剪入库(REQ-182)
1 parent 5bb9c3a commit 0ca4fbe

9 files changed

Lines changed: 548 additions & 26 deletions

File tree

‎app/src-tauri/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ mod structure_models;
156156
mod structure_note;
157157
// v0.7.7(REQ-182):结构图检测纯函数——diagram_likeness/pick_sharpest/区域过滤
158158
mod structure_detect;
159+
// v0.7.7(REQ-182):结构图批量捕获管线——屏→选优帧→版面→过滤→裁剪→入库
160+
mod structure_capture;
159161
// v0.7.7(REQ-183):结构图存储——struct/ 命名空间 + 独立预算 + 去重
160162
mod structure_store;
161163
mod structure_tier;

‎app/src-tauri/src/screens.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,9 @@ pub fn attach_images(screens: &mut [SessionScreen], images_dir: &Path) {
203203
}
204204

205205
/// 归档 full 图时间戳列表(纯 IO,一次扫描)。
206-
fn list_full_image_timestamps(images_dir: &Path) -> Vec<u64> {
206+
/// @ai-context: pub(crate)(v0.7.7 REQ-182):structure_capture 批量捕获复用
207+
/// (屏时间窗帧候选——同一目录扫描约定,避免重复实现)。
208+
pub(crate) fn list_full_image_timestamps(images_dir: &Path) -> Vec<u64> {
207209
let entries = std::fs::read_dir(images_dir.join("full")).ok();
208210
let mut ts: Vec<u64> = entries
209211
.into_iter()
Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
//! 结构图批量捕获管线(REQ-182 / v0.7.7):屏 → 选优帧 → 版面分析 → 过滤 → 裁剪 → 入库。
2+
//!
3+
//! @ai-context: 非线性结构(表格/公式/代码/流程图等)"图像即产物"兜底(ADR-010):
4+
//! 会话停止后批量执行(live 停止链路触发)+ 图库「重新捕获」可重跑
5+
//! (same_image 去重保证幂等——同图不重复入库)。
6+
//! @ai-context: 方案 A+(2026-08-20 裁决):不动实时链路——对每屏时间窗内的
7+
//! 归档 full 帧采样(≤8 帧 bound 解码成本)→ pick_sharpest 选优
8+
//! (边缘能量,动效结束后的稳定清晰帧)→ analyze_layout 复用 →
9+
//! table/formula/code 直收 + Image 过 diagram_likeness → 白边裁剪
10+
//! (image_crop 机制)→ struct/ + 记录入库。
11+
//! @ai-context: 预算耗尽(auto 桶 80/会话)→ 停止本会话后续捕获(summary 标记,
12+
//! 命令层提示);旧会话无屏/无图 → 自然跳过(降级链)。
13+
14+
use std::path::Path;
15+
16+
use crate::db::Db;
17+
use crate::error::Result;
18+
use crate::layout_analyzer::{analyze_layout, FrameGrid, LayoutRegion};
19+
use crate::structure_detect::{filter_structure_regions, pick_sharpest};
20+
21+
/// 每屏采样上限(帧解码成本 bound——屏窗口内归档帧通常 1-2 张,长屏最多 8)。
22+
const MAX_FRAMES_PER_SCREEN: usize = 8;
23+
24+
/// 捕获结果摘要(命令层事件/前端提示数据源)。
25+
#[derive(Debug, Clone, Copy, Default, serde::Serialize)]
26+
#[serde(rename_all = "camelCase")]
27+
pub struct CaptureSummary {
28+
/// 有图屏数(参与扫描的屏)
29+
pub screens_scanned: usize,
30+
/// 实际入库结构图数
31+
pub captured: usize,
32+
/// 自动桶预算是否耗尽(提前终止)
33+
pub budget_exhausted: bool,
34+
}
35+
36+
/// 批量捕获主入口(编排 + IO;纯逻辑均在 structure_detect/layout_analyzer)。
37+
///
38+
/// @ai-context: now_ms 注入(文件名时间基;测试可控);每屏独立失败不阻断
39+
/// (解码失败帧跳过、无候选屏跳过——诚实降级)。
40+
pub fn capture_session_structures(
41+
db: &Db,
42+
data_dir: &Path,
43+
session_id: i64,
44+
now_ms: u64,
45+
) -> Result<CaptureSummary> {
46+
let images_dir = data_dir.join("session-images").join(session_id.to_string());
47+
let blocks = db.list_ocr_blocks(session_id)?;
48+
// ① 屏构建(复用屏卡体系;只保留有归档图的屏——无图无法裁剪)
49+
let mut screens = crate::screens::build_screens(&blocks, Some(&images_dir));
50+
screens.retain(|s| s.image_ref.is_some());
51+
let mut store = crate::structure_store::StructureImageStore::new(images_dir.clone())?;
52+
let mut summary = CaptureSummary { screens_scanned: screens.len(), ..Default::default() };
53+
let mut seq = 0u64;
54+
for screen in &screens {
55+
// ② 屏时间窗内归档帧采样(≤8 均匀)
56+
let candidates = frame_candidates(
57+
&images_dir,
58+
screen.first_seen_ms,
59+
screen.last_seen_ms,
60+
MAX_FRAMES_PER_SCREEN,
61+
);
62+
// ③ 解码(失败帧跳过)+ 网格
63+
let decoded = decode_candidates(&images_dir, &candidates);
64+
if decoded.is_empty() {
65+
continue;
66+
}
67+
let grids: Vec<(u64, FrameGrid)> =
68+
decoded.iter().map(|(ts, img)| (*ts, grid_from_rgb(img))).collect();
69+
// ④ 选优帧(边缘能量最高;全零能量 → 该屏跳过)
70+
let Some(idx) = pick_sharpest(&grids) else { continue };
71+
let grid = &grids[idx].1;
72+
// ⑤ 版面分析 + 区域过滤(结构三类直收 + Text/Image 门控)
73+
let kept = filter_structure_regions(&analyze_layout(grid), grid);
74+
if kept.is_empty() {
75+
continue;
76+
}
77+
let frame = &decoded[idx].1;
78+
let frame_regions = crate::frame_features::regions_to_frame(
79+
&kept,
80+
grid.cols,
81+
grid.rows,
82+
frame.width(),
83+
frame.height(),
84+
);
85+
// ⑥ 逐区域裁剪 → 存储 → 记录
86+
for r in frame_regions {
87+
let Some(crop) = crop_region(frame, &r) else { continue };
88+
let bgra = rgb_to_bgra(&crop);
89+
seq += 1;
90+
match store.save_auto(now_ms + seq, &bgra, crop.width(), crop.height()) {
91+
Ok(outcome) => {
92+
// 去重命中(跨屏同图/重跑):不重复插记录——同图只留一份
93+
if !outcome.is_new {
94+
continue;
95+
}
96+
crate::db_structures::insert_structure_image(
97+
db,
98+
&crate::db_structures::StructureImageRecord {
99+
id: 0,
100+
session_id,
101+
screen_id: screen.screen_id,
102+
kind: kind_name(r.kind),
103+
bbox: bbox_json(&r),
104+
source_ts_ms: grids[idx].0,
105+
crop_path: outcome.rel,
106+
source: "auto".to_string(),
107+
created_at: now_ms + seq,
108+
},
109+
)?;
110+
summary.captured += 1;
111+
}
112+
// 预算耗尽:终止本会话后续捕获(已捕获保留)
113+
Err(_) => {
114+
summary.budget_exhausted = true;
115+
return Ok(summary);
116+
}
117+
}
118+
}
119+
}
120+
Ok(summary)
121+
}
122+
123+
/// 屏时间窗内归档帧候选(纯函数):时间戳 ∈ [first, last];超出采样上限
124+
/// 均匀抽样(bound 解码成本);目录缺失/空 → 空列表。
125+
fn frame_candidates(
126+
images_dir: &Path,
127+
first_ms: u64,
128+
last_ms: u64,
129+
max: usize,
130+
) -> Vec<u64> {
131+
let in_window: Vec<u64> = crate::screens::list_full_image_timestamps(images_dir)
132+
.into_iter()
133+
.filter(|t| *t >= first_ms && *t <= last_ms)
134+
.collect();
135+
if in_window.len() <= max {
136+
return in_window;
137+
}
138+
// 均匀抽样(首尾必含——首帧=屏开始画面、尾帧=最近画面,都值得参与选优)
139+
let last = in_window.len() - 1;
140+
(0..max)
141+
.map(|i| in_window[((i as f32 * last as f32 / (max - 1) as f32).round() as usize)])
142+
.collect()
143+
}
144+
145+
/// 解码候选帧(纯 IO):webp → RGB;解码失败帧跳过(不阻断)。
146+
fn decode_candidates(images_dir: &Path, timestamps: &[u64]) -> Vec<(u64, image::RgbImage)> {
147+
timestamps
148+
.iter()
149+
.filter_map(|t| {
150+
let img = image::open(images_dir.join("full").join(format!("{t}.webp")))
151+
.ok()?
152+
.to_rgb8();
153+
Some((*t, img))
154+
})
155+
.collect()
156+
}
157+
158+
/// RGB 帧 → 版面网格(纯函数;与 frame_features::grid_from_bgra 同口径:
159+
/// 目标 ~32×18 格、中心像素采样、Rec.601 亮度)。
160+
fn grid_from_rgb(rgb: &image::RgbImage) -> FrameGrid {
161+
let (width, height) = (rgb.width(), rgb.height());
162+
let cols = 32u32.min(width).max(1);
163+
let rows = 18u32.min(height).max(1);
164+
let mut cells = Vec::with_capacity((cols * rows) as usize);
165+
for gy in 0..rows {
166+
let y = (gy * height) / rows;
167+
for gx in 0..cols {
168+
let x = (gx * width) / cols;
169+
let p = rgb.get_pixel(x, y);
170+
let luma = (p[0] as u32 * 299 + p[1] as u32 * 587 + p[2] as u32 * 114) / 1000;
171+
cells.push(luma.clamp(0, 255) as u8);
172+
}
173+
}
174+
FrameGrid { cols, rows, cells }
175+
}
176+
177+
/// 区域裁剪(纯函数):帧像素坐标 → 子图;越界钳制(不 panic),空尺寸 → None。
178+
fn crop_region(frame: &image::RgbImage, r: &LayoutRegion) -> Option<image::RgbImage> {
179+
let x = r.x.min(frame.width());
180+
let y = r.y.min(frame.height());
181+
let w = r.w.min(frame.width() - x);
182+
let h = r.h.min(frame.height() - y);
183+
if w == 0 || h == 0 {
184+
return None;
185+
}
186+
Some(image::imageops::crop_imm(frame, x, y, w, h).to_image())
187+
}
188+
189+
/// RGB → BGRA(结构图存储契约;纯函数)。
190+
fn rgb_to_bgra(rgb: &image::RgbImage) -> Vec<u8> {
191+
let mut out = Vec::with_capacity((rgb.width() * rgb.height() * 4) as usize);
192+
for p in rgb.pixels() {
193+
out.extend_from_slice(&[p[2], p[1], p[0], 255]);
194+
}
195+
out
196+
}
197+
198+
/// 区域类型 → 记录 kind(与 DB 字符串枚举一致)。
199+
fn kind_name(kind: crate::layout_analyzer::RegionKind) -> String {
200+
match kind {
201+
crate::layout_analyzer::RegionKind::Table => "table",
202+
crate::layout_analyzer::RegionKind::Formula => "formula",
203+
crate::layout_analyzer::RegionKind::Code => "code",
204+
crate::layout_analyzer::RegionKind::Image => "image",
205+
_ => "unknown",
206+
}
207+
.to_string()
208+
}
209+
210+
/// bbox JSON(帧坐标;记录契约)。
211+
fn bbox_json(r: &LayoutRegion) -> String {
212+
format!(r#"{{"x":{},"y":{},"w":{},"h":{}}}"#, r.x, r.y, r.w, r.h)
213+
}
214+
215+
#[cfg(test)]
216+
#[path = "structure_capture_tests.rs"]
217+
mod tests;

0 commit comments

Comments
 (0)