Skip to content

Commit 0f3cda7

Browse files
committed
feat(photo-session): 图文截图保存+OCR 编排模块(存图/去重/落库/降级)
1 parent 4975343 commit 0f3cda7

3 files changed

Lines changed: 227 additions & 0 deletions

File tree

‎app/src-tauri/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,8 @@ mod outline;
228228
mod playback_region;
229229
// v0.7.0 M2(REQ-125):播放器行为信号(暂停检测 + M17 倍速缩放采样)
230230
mod player_behavior;
231+
// v0.11.7(图文会话,ADR-020):图文截图保存 + OCR 编排(业务模块,命令层在 commands_photo)
232+
mod photo_capture;
231233
mod practice_detect;
232234
mod quality_report;
233235
mod purify_config;

‎app/src-tauri/src/photo_capture.rs‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
//! 图文会话截图保存 + OCR 编排(v0.11.7,ADR-020)。
2+
//!
3+
//! @ai-context: 用户框选截屏 → base64(PNG 无损,前端 canvas 裁剪)→ 解码验证 →
4+
//! 存图(full+thumb,双指纹去重)→ 全图 OCR(960px 口径,bbox 反算)
5+
//! → OCR 块落库(region=full,screen_id=None 走视图层聚类)。
6+
//! @ai-context: 与实时/导入链路 OCR 口径统一(960px 缩放 + score≥0.5);与自动
7+
//! 采帧不同:用户显式框选的内容不过 UI 垃圾黑名单(意图即内容)。
8+
//! @ai-context: OCR 失败不阻断截图保存(图已落盘,block_count=0 诚实返回)。
9+
10+
use base64::Engine;
11+
use image::GenericImageView;
12+
use serde::Serialize;
13+
14+
use crate::db::Db;
15+
use crate::engine::EnginePool;
16+
use crate::image_store::SessionImageStore;
17+
use crate::types::{NewSessionOcrBlock, TextBox};
18+
19+
/// OCR 输入最大宽度(与实时/导入链路同口径——推理成本近平方下降)。
20+
const OCR_MAX_WIDTH: u32 = 960;
21+
/// OCR 块最低置信度(与实时/导入链路同口径)。
22+
const MIN_SCORE: f32 = 0.5;
23+
/// 识别预览条数上限(前端即时反馈)。
24+
const PREVIEW_MAX: usize = 3;
25+
/// 单图 base64 解码后字节上限(16MB——解码前拒绝,省 CPU)。
26+
const MAX_PHOTO_BYTES: usize = 16 * 1024 * 1024;
27+
28+
/// 截图保存结果(前端反馈契约;camelCase 与前端 PhotoResult 对齐)。
29+
#[derive(Debug, Clone, PartialEq, Serialize)]
30+
#[serde(rename_all = "camelCase")]
31+
pub struct PhotoCaptureResult {
32+
/// 双指纹命中(与最近保存图相同,未重复存图/未重复落块)
33+
pub duplicated: bool,
34+
/// 新落库 OCR 块数(OCR 失败/超时可能为 0)
35+
pub block_count: usize,
36+
/// 识别文本预览(≤3 条;空 = 未识别出文字)
37+
pub preview: Vec<String>,
38+
/// 图片相对路径(full/<ts>.webp)
39+
pub image_ref: String,
40+
}
41+
42+
/// 保存截图并 OCR(主入口;注入 store/DB/引擎池——端到端可测)。
43+
///
44+
/// @ai-context: store 由调用方跨命令持有(命令层互斥槽内复用)——双指纹去重
45+
/// FIFO 是内存态,每次 new 会清空导致去重失效(实时链路即长驻
46+
/// store);budget 计数同样跨截图累计(50 张预算会话级生效)。
47+
/// @ai-context: OCR 有界等待 ≤20s——调用方在锁内调用时锁时长与此相当;
48+
/// 图文采集为用户串行动线(同刻仅一个截图保存),可接受。
49+
pub fn save_photo_capture(
50+
store: &mut SessionImageStore,
51+
db: &Db,
52+
engines: &EnginePool,
53+
session_id: i64,
54+
image_b64: &str,
55+
timestamp_ms: u64,
56+
) -> Result<PhotoCaptureResult, String> {
57+
let img = decode_image(image_b64)?;
58+
let (w, h) = img.dimensions();
59+
// ① 存图:RGB → BGRA8(save_frame 输入口径)→ full+thumb + 双指纹去重
60+
let bgra: Vec<u8> = img.pixels().flat_map(|p| [p[2], p[1], p[0], 255u8]).collect();
61+
let rel = store
62+
.save_frame(timestamp_ms, &bgra, w, h)
63+
.map_err(|e| format!("保存截图失败: {}", e))?;
64+
// 重复判定:save_frame 对重复帧返回已有路径(旧时间戳)——文件名 ≠ 本次 ts 即重复
65+
let duplicated = rel != format!("full/{}.webp", timestamp_ms);
66+
if duplicated {
67+
return Ok(PhotoCaptureResult { duplicated: true, block_count: 0, preview: Vec::new(), image_ref: rel });
68+
}
69+
// ② 全图 OCR(960px 口径;失败/超时降级——图已保存)
70+
let (blocks, scale) = ocr_blocks(&img, engines);
71+
// ③ 落库(不过 UI 垃圾黑名单——用户显式框选的内容即意图)
72+
let mut count = 0usize;
73+
let mut preview = Vec::new();
74+
for b in blocks {
75+
if b.score < MIN_SCORE || b.text.trim().is_empty() {
76+
continue;
77+
}
78+
count += 1;
79+
if preview.len() < PREVIEW_MAX {
80+
preview.push(b.text.clone());
81+
}
82+
if let Err(e) = db.add_ocr_block(&NewSessionOcrBlock {
83+
session_id,
84+
timestamp_ms,
85+
text: b.text,
86+
score: b.score,
87+
region: "full".to_string(),
88+
region_kind: None,
89+
bbox: b.bbox.map(|bb| TextBox {
90+
x: bb.x * scale,
91+
y: bb.y * scale,
92+
w: bb.w * scale,
93+
h: bb.h * scale,
94+
}),
95+
screen_id: None,
96+
}) {
97+
eprintln!("[Photo] OCR 块落库失败: {}", e);
98+
}
99+
}
100+
Ok(PhotoCaptureResult { duplicated: false, block_count: count, preview, image_ref: rel })
101+
}
102+
103+
/// base64 解码 + 解码器验证(白名单由 image 解码器把关——capture_fragment 同模式)。
104+
fn decode_image(image_b64: &str) -> Result<image::RgbImage, String> {
105+
let bytes = base64::engine::general_purpose::STANDARD
106+
.decode(image_b64.trim())
107+
.map_err(|e| format!("图片 base64 解码失败: {}", e))?;
108+
if bytes.is_empty() {
109+
return Err("图片数据为空".to_string());
110+
}
111+
if bytes.len() > MAX_PHOTO_BYTES {
112+
return Err(format!("图片超限(>{}MB)", MAX_PHOTO_BYTES / 1024 / 1024));
113+
}
114+
let img = image::load_from_memory(&bytes)
115+
.map_err(|e| format!("图片解码失败(仅支持 PNG/JPEG 等常见格式): {}", e))?;
116+
let (w, h) = img.dimensions();
117+
if w == 0 || h == 0 {
118+
return Err("图片尺寸非法".to_string());
119+
}
120+
Ok(img.to_rgb8())
121+
}
122+
123+
/// 全图 OCR:缩至 960px 后识别,返回 (块, bbox 缩放比)。
124+
/// @ai-context: 缩放比用于 bbox 反算回原图坐标系(TD-046 同思路——OCR 输入
125+
/// 缩小后 bbox 处于缩小坐标系)。
126+
fn ocr_blocks(img: &image::RgbImage, engines: &EnginePool) -> (Vec<crate::types::OcrBlock>, f32) {
127+
let (w, h) = img.dimensions();
128+
let scale = if w > OCR_MAX_WIDTH { w as f32 / OCR_MAX_WIDTH as f32 } else { 1.0 };
129+
let input = if w > OCR_MAX_WIDTH {
130+
let new_h = ((h as u64 * OCR_MAX_WIDTH as u64 / w as u64)).max(1) as u32;
131+
image::imageops::resize(img, OCR_MAX_WIDTH, new_h, image::imageops::FilterType::Triangle)
132+
} else {
133+
img.clone()
134+
};
135+
match engines.recognize_image_timeout(input, crate::engine::OCR_REQUEST_TIMEOUT) {
136+
Ok(blocks) => (blocks, scale),
137+
Err(e) => {
138+
// OCR 失败/超时:图已保存,块数 0 诚实返回(下张截图重试语义)
139+
eprintln!("[Photo] OCR 失败(截图已保存,块数 0): {}", e);
140+
(Vec::new(), scale)
141+
}
142+
}
143+
}
144+
145+
/// 单测独立文件(保持本文件 ≤300 行,AGENTS.md §3)。
146+
#[cfg(test)]
147+
#[path = "photo_capture_tests.rs"]
148+
mod tests;
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
//! photo_capture 业务模块测试(内存库 + 临时目录,环境隔离)。
2+
3+
use base64::Engine;
4+
5+
use crate::db::Db;
6+
use crate::engine::EnginePool;
7+
use crate::image_store::SessionImageStore;
8+
use crate::photo_capture::{decode_image, save_photo_capture};
9+
use crate::types::NewSession;
10+
11+
/// 构造一个纯色 PNG 的 base64(测试夹具)。
12+
fn png_b64(w: u32, h: u32, gray: u8) -> String {
13+
let img = image::RgbImage::from_pixel(w, h, image::Rgb([gray, gray, gray]));
14+
let mut bytes = Vec::new();
15+
img.write_to(&mut std::io::Cursor::new(&mut bytes), image::ImageFormat::Png)
16+
.expect("PNG 编码成功");
17+
base64::engine::general_purpose::STANDARD.encode(&bytes)
18+
}
19+
20+
#[test]
21+
fn save_photo_end_to_end_engine_failure_still_saves() {
22+
// Arrange:图文会话 + 临时图片目录 + 长驻 store + dummy 引擎(OCR 立即失败)
23+
let db = Db::open(":memory:").expect("内存库");
24+
let session = db
25+
.create_session(&NewSession {
26+
title: "图文".into(),
27+
source_window: None,
28+
profile: None,
29+
kind: Some("photo".into()),
30+
})
31+
.expect("建会话");
32+
let dir = tempfile::tempdir().expect("临时目录");
33+
let mut store = SessionImageStore::new(dir.path().to_path_buf()).expect("store");
34+
// Act:保存 200×100 纯色图(ts=1000)
35+
let result = save_photo_capture(&mut store, &db, &EnginePool::dummy(), session.id, &png_b64(200, 100, 120), 1000)
36+
.expect("保存成功");
37+
// Assert:图已落盘(full+thumb)、块数诚实 0(dummy 引擎失败不阻断)、非重复
38+
assert!(!result.duplicated);
39+
assert_eq!(result.block_count, 0);
40+
assert_eq!(result.image_ref, "full/1000.webp");
41+
assert!(dir.path().join("full/1000.webp").is_file(), "full 图应存在");
42+
assert!(dir.path().join("thumb/1000.webp").is_file(), "缩略图应存在");
43+
}
44+
45+
#[test]
46+
fn duplicate_photo_returns_duplicated_flag() {
47+
// Arrange:同内容连续两次保存(同一长驻 store——去重 FIFO 跨调用生效)
48+
let db = Db::open(":memory:").unwrap();
49+
let session = db
50+
.create_session(&NewSession {
51+
title: "图文".into(),
52+
source_window: None,
53+
profile: None,
54+
kind: Some("photo".into()),
55+
})
56+
.unwrap();
57+
let dir = tempfile::tempdir().unwrap();
58+
let mut store = SessionImageStore::new(dir.path().to_path_buf()).unwrap();
59+
let b64 = png_b64(300, 200, 80);
60+
let first = save_photo_capture(&mut store, &db, &EnginePool::dummy(), session.id, &b64, 1000).unwrap();
61+
// Act:第二次保存(新 ts=2000,内容相同)
62+
let second = save_photo_capture(&mut store, &db, &EnginePool::dummy(), session.id, &b64, 2000).unwrap();
63+
// Assert:第二次命中双指纹 → duplicated=true、image_ref 仍为首次路径、不新增文件
64+
assert_eq!(first.image_ref, "full/1000.webp");
65+
assert!(second.duplicated);
66+
assert_eq!(second.image_ref, "full/1000.webp");
67+
assert!(!dir.path().join("full/2000.webp").exists(), "重复内容不得新增文件");
68+
}
69+
70+
#[test]
71+
fn decode_rejects_garbage_and_empty() {
72+
// Act & Assert:非法 base64 / 空串 / 非图片字节 → 可诊断错误
73+
assert!(decode_image("not-base64!!").is_err());
74+
assert!(decode_image("").is_err());
75+
let junk = base64::engine::general_purpose::STANDARD.encode(b"not an image");
76+
assert!(decode_image(&junk).is_err());
77+
}

0 commit comments

Comments
 (0)