Skip to content

Commit 790509e

Browse files
committed
feat(photo): 系统级覆盖层截图——1:1 原始像素框选替代 letterbox(v0.12.0 M3)
- commands_overlay.rs(新):open_capture_overlay(gdi 截屏 → data_dir/overlay-tmp /snapshot.jpg → 全屏透明 capture-overlay 窗口 index.html?overlay=1); overlay_get_image(覆盖层取图路径,convertFileSrc 显示);overlay_submit_capture (后端裁剪 PNG base64 → overlay:captured 回传主窗口 → 关窗;裁剪收敛到 image crate, 前端 canvas 无 taint 问题);overlay_cancel(Esc → overlay:cancelled → 关窗,无副作用) - App.tsx:?overlay=1 → CaptureOverlayPanel(不渲染主导航壳) - CaptureOverlayPanel.tsx(新):1:1 显示(scale 换算——DPI/多屏准确)、拖拽框选、 确认(Enter)/取消(Esc);窗口不持久,截完即销毁 - PhotoCapturePanel:截屏按钮改 open_capture_overlay(覆盖层创建失败回退 letterbox); 监听 overlay:captured(PNG base64 同口径)→ save_photo_capture / cancelled 无副作用 - capabilities:windows 加 capture-overlay 真机验收:图文采集一步触发覆盖层框选;4K 屏不再缩略;框选精度提升。
1 parent 1ec1cc0 commit 790509e

8 files changed

Lines changed: 335 additions & 6 deletions

File tree

‎app/src-tauri/capabilities/default.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"$schema": "../gen/schemas/desktop-schema.json",
33
"identifier": "default",
44
"description": "Capability for the main window",
5-
"windows": ["main", "capture-float"],
5+
"windows": ["main", "capture-float", "capture-overlay"],
66
"permissions": [
77
"core:default",
88
"dialog:default"

‎app/src-tauri/src/app_setup.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,8 @@ pub fn setup_app_state(app: &mut tauri::App) -> Result<(), String> {
274274
// v0.11.7(图文会话,ADR-020):图文采集互斥槽(会话 id + 长驻图片库 store)
275275
photo_session: std::sync::Arc::new(std::sync::Mutex::new(None)),
276276
photo_store: std::sync::Arc::new(std::sync::Mutex::new(None)),
277+
// v0.12.0 M3:系统级覆盖层截图临时文件路径(open_capture_overlay 写入)
278+
overlay_image_path: std::sync::Arc::new(std::sync::Mutex::new(None)),
277279
});
278280
Ok(())
279281
}

‎app/src-tauri/src/commands.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,9 @@ pub struct AppState {
170170
/// v0.11.7(图文会话,ADR-020):图文图片库 store(跨截图常驻——保持
171171
/// 双指纹去重 FIFO 与预算计数;与 photo_session 同生命周期)
172172
pub photo_store: std::sync::Arc<std::sync::Mutex<Option<crate::image_store::SessionImageStore>>>,
173+
/// v0.12.0 M3:系统级覆盖层截图临时文件路径(open_capture_overlay 写入;
174+
/// 覆盖层窗口与其后主窗口裁剪共用——窗口不持久,截完即销毁)
175+
pub overlay_image_path: std::sync::Arc<std::sync::Mutex<Option<std::path::PathBuf>>>,
173176
}
174177

175178
/// 枚举可捕获的窗口/进程(课堂助手目标窗口选择,含推荐评分)。
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
//! 系统级覆盖层截图命令(v0.12.0 M3,ADR-022 关联——交互债)。
2+
//!
3+
//! @ai-context: 替代应用内 letterbox 框选(4K 屏在 1200px 窗口内缩到 73%,
4+
//! 文字细节丢失 + 三步操作):Tauri 独立全屏透明窗口 1:1 原始像素
5+
//! 显示当前帧 → 拖拽框选 → 确认 → 后端裁剪 PNG base64 回传主窗口
6+
//! (overlay:captured)→ 主窗口 save_photo_capture;Esc → 取消。
7+
//! @ai-context: 覆盖层窗口不持久(截完即销毁);跨窗口通信走 IPC event 直传,
8+
//! 不走 DB/文件(YAGNI);当前 letterbox 路径保留为降级
9+
//! (覆盖层窗口创建失败时主窗口回退应用内框选)。
10+
//! @ai-context: 截图临时文件(data_dir/overlay-tmp/snapshot.jpg)经 asset 协议
11+
//! 供覆盖层窗口显示;主窗口裁剪与 OCR 均不依赖前端 canvas(前端
12+
//! 无 taint 问题——crop 收敛到后端 image crate)。
13+
14+
use base64::Engine;
15+
use tauri::{Emitter, Manager, State, WebviewUrl, WebviewWindowBuilder};
16+
17+
use crate::commands::AppState;
18+
19+
/// 框选矩形(图像像素坐标——覆盖层 1:1 显示,鼠标坐标即图像坐标)。
20+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
21+
#[serde(rename_all = "camelCase")]
22+
pub struct OverlayRect {
23+
pub x: u32,
24+
pub y: u32,
25+
pub w: u32,
26+
pub h: u32,
27+
}
28+
29+
/// 覆盖层确认载荷(回传主窗口:裁剪 PNG base64——与 save_photo_capture 输入同源)。
30+
#[derive(Debug, Clone, serde::Serialize)]
31+
#[serde(rename_all = "camelCase")]
32+
pub struct OverlayCapturedPayload {
33+
pub image_b64: String,
34+
}
35+
36+
/// 覆盖层窗口标签。
37+
pub const CAPTURE_OVERLAY_LABEL: &str = "capture-overlay";
38+
39+
/// 打开覆盖层截图:截屏 → 存临时文件 → 创建全屏透明覆盖层窗口(幂等)。
40+
#[tauri::command]
41+
pub fn open_capture_overlay(state: State<'_, AppState>, app: tauri::AppHandle) -> Result<(), String> {
42+
let now_ms = std::time::SystemTime::now()
43+
.duration_since(std::time::UNIX_EPOCH)
44+
.map(|d| d.as_millis() as u64)
45+
.unwrap_or(0);
46+
// 1) 全屏快照(与 capture_screen_snapshot 同源——gdi_capture;一次性调用无状态)
47+
let frame = crate::capture::gdi_capture::gdi_capture(None, now_ms)
48+
.map_err(|e| format!("截屏失败: {}", e))?;
49+
let rgb = crate::image_store::bgra_to_rgb_public(&frame.bgraw, frame.width, frame.height)
50+
.ok_or_else(|| "截屏数据无效".to_string())?;
51+
// 2) 存 JPEG 临时文件(覆盖层窗口经 asset 协议显示;主窗口裁剪同源)
52+
let dir = state.data_dir.join("overlay-tmp");
53+
std::fs::create_dir_all(&dir).map_err(|e| format!("创建覆盖层临时目录失败: {}", e))?;
54+
let path = dir.join("snapshot.jpg");
55+
rgb.save(&path).map_err(|e| format!("保存覆盖层截图失败: {}", e))?;
56+
*state
57+
.overlay_image_path
58+
.lock()
59+
.map_err(|e| format!("覆盖层截图路径锁中毒: {}", e))? = Some(path);
60+
// 3) 创建覆盖层窗口(已存在则复用——幂等)
61+
if app.get_webview_window(CAPTURE_OVERLAY_LABEL).is_none() {
62+
WebviewWindowBuilder::new(&app, CAPTURE_OVERLAY_LABEL, WebviewUrl::App("index.html?overlay=1".into()))
63+
.title("熵减截图")
64+
.decorations(false)
65+
.transparent(true)
66+
.fullscreen(true)
67+
.skip_taskbar(true)
68+
.always_on_top(true)
69+
.build()
70+
.map_err(|e| format!("创建覆盖层窗口失败: {}", e))?;
71+
}
72+
Ok(())
73+
}
74+
75+
/// 覆盖层窗口取待选截图路径(convertFileSrc 显示用)。
76+
#[tauri::command]
77+
pub fn overlay_get_image(state: State<'_, AppState>) -> Result<Option<String>, String> {
78+
let p = state
79+
.overlay_image_path
80+
.lock()
81+
.map_err(|e| format!("覆盖层截图路径锁中毒: {}", e))?;
82+
Ok(p.clone().map(|x| x.to_string_lossy().to_string()))
83+
}
84+
85+
/// 覆盖层确认框选 → 后端裁剪 PNG base64 → overlay:captured 回传主窗口 → 关窗。
86+
///
87+
/// @ai-context: 裁剪在 Rust 侧完成(前端 canvas 可能被 asset 协议 taint——
88+
/// 裁剪收敛到 image crate,回传 base64 与旧 letterbox 路径同语义)。
89+
#[tauri::command]
90+
pub fn overlay_submit_capture(
91+
state: State<'_, AppState>,
92+
app: tauri::AppHandle,
93+
rect: OverlayRect,
94+
) -> Result<(), String> {
95+
let path = {
96+
state
97+
.overlay_image_path
98+
.lock()
99+
.map_err(|e| format!("覆盖层截图路径锁中毒: {}", e))?
100+
.clone()
101+
}
102+
.ok_or_else(|| "无待选截图(先打开覆盖层)".to_string())?;
103+
let img = image::open(&path).map_err(|e| format!("读取覆盖层截图失败: {}", e))?;
104+
let (iw, ih) = (img.width(), img.height());
105+
let (x, y) = (rect.x.min(iw.saturating_sub(1)), rect.y.min(ih.saturating_sub(1)));
106+
let (w, h) = (rect.w.max(1).min(iw - x), rect.h.max(1).min(ih - y));
107+
let crop = image::imageops::crop_imm(&img, x, y, w, h).to_image();
108+
let mut buf = std::io::Cursor::new(Vec::new());
109+
let encoder = image::codecs::png::PngEncoder::new(&mut buf);
110+
crop.write_with_encoder(encoder)
111+
.map_err(|e| format!("裁剪 PNG 编码失败: {}", e))?;
112+
let b64 = base64::engine::general_purpose::STANDARD.encode(buf.into_inner());
113+
let _ = app.emit("overlay:captured", OverlayCapturedPayload { image_b64: b64 });
114+
close_overlay(&app);
115+
Ok(())
116+
}
117+
118+
/// 覆盖层取消(Esc)→ overlay:cancelled 通知主窗口 → 关窗,无副作用。
119+
#[tauri::command]
120+
pub fn overlay_cancel(app: tauri::AppHandle) -> Result<(), String> {
121+
let _ = app.emit("overlay:cancelled", ());
122+
close_overlay(&app);
123+
Ok(())
124+
}
125+
126+
/// 关闭覆盖层窗口(不存在视为已关闭——幂等)。
127+
fn close_overlay(app: &tauri::AppHandle) {
128+
if let Some(window) = app.get_webview_window(CAPTURE_OVERLAY_LABEL) {
129+
let _ = window.close();
130+
}
131+
}

‎app/src-tauri/src/lib.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ mod commands_vocab;
122122
mod commands_video;
123123
// v0.12.0 M6(采集体验债):采集浮窗窗口命令(open/close_capture_float)
124124
mod commands_window;
125+
// v0.12.0 M3(交互债):系统级覆盖层截图命令(open/close/submit/cancel)
126+
mod commands_overlay;
125127
mod concat;
126128
mod db;
127129
// H3 拆分(db.rs 原 678 行硬拆):schema 建表/列迁移 + notes 读写独立成块
@@ -500,6 +502,11 @@ pub fn run() {
500502
// v0.12.0 M6(采集体验债):采集浮窗打开/关闭
501503
commands_window::open_capture_float,
502504
commands_window::close_capture_float,
505+
// v0.12.0 M3(系统级覆盖层截图):打开/取图/确认裁剪/取消
506+
commands_overlay::open_capture_overlay,
507+
commands_overlay::overlay_get_image,
508+
commands_overlay::overlay_submit_capture,
509+
commands_overlay::overlay_cancel,
503510
// 会话结构化分析(REQ-044/045/046,v0.5.0 M2:章节/重点/术语/讲者)
504511
commands_analysis::analyze_session_command,
505512
// 说话人分离(REQ-153,v0.7.2:弱化版讲者切换离线分析——幂等懒加载)

‎app/src/App.tsx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import SessionsPage from "./pages/SessionsPage";
2020
import SettingsPage from "./pages/SettingsPage";
2121
import AppErrorBoundary from "./components/AppErrorBoundary";
2222
import CaptureFloatPanel from "./components/CaptureFloatPanel";
23+
import CaptureOverlayPanel from "./components/CaptureOverlayPanel";
2324
import type { AiTaskState } from "./types";
2425

2526
type Page = "classroom" | "sessions" | "notes" | "settings";
@@ -32,6 +33,12 @@ const NAV_ITEMS: { key: Page; label: string }[] = [
3233
];
3334

3435
function App() {
36+
// v0.12.0 M3:系统级覆盖层截图窗口入口——URL 带 ?overlay=1 时仅渲染
37+
// CaptureOverlayPanel(全屏透明 1:1 框选;不渲染主导航壳;独立窗口)。
38+
// eslint-disable-next-line react-hooks/rules-of-hooks
39+
if (new URLSearchParams(window.location.search).get("overlay") === "1") {
40+
return <CaptureOverlayPanel />;
41+
}
3542
// v0.12.0 M6:采集浮窗入口——URL 带 ?float=1 时仅渲染 CaptureFloatPanel
3643
//(不渲染主导航壳;浮窗独立窗口 alwaysOnTop,加载 index.html?float=1)。
3744
// 规则:float 标志 per-window 恒定(URL 不变),故此处 before-hooks 早返回安全;
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/**
2+
* CaptureOverlayPanel — 系统级覆盖层截图面板(v0.12.0 M3,交互债)。
3+
*
4+
* @ai-context: 应用内 letterbox 框选(4K 屏缩到 73% 丢细节 + 三步操作)替代——
5+
* 覆盖层窗口全屏透明显示当前帧(1:1 原始像素,鼠标坐标经 scale
6+
* 换算回图像坐标),拖拽框选 → 确认 → overlay_submit_capture(后端
7+
* 裁剪 PNG 回传主窗口);Esc → overlay_cancel。窗口不持久,截完即销毁。
8+
* @ai-context: 自动边缘检测候选(微信截图对标)YAGNI 不预排——仅拖拽框选。
9+
*/
10+
import { useEffect, useRef, useState } from "react";
11+
import { invoke } from "@tauri-apps/api/core";
12+
import { convertFileSrc } from "@tauri-apps/api/core";
13+
14+
interface Sel {
15+
x1: number;
16+
y1: number;
17+
x2: number;
18+
y2: number;
19+
}
20+
21+
export default function CaptureOverlayPanel() {
22+
const [src, setSrc] = useState<string | null>(null);
23+
const [natural, setNatural] = useState<{ w: number; h: number } | null>(null);
24+
const [sel, setSel] = useState<Sel | null>(null);
25+
const dragging = useRef(false);
26+
const imgRef = useRef<HTMLImageElement>(null);
27+
const scaleRef = useRef<{ x: number; y: number }>({ x: 1, y: 1 });
28+
29+
useEffect(() => {
30+
void invoke<string | null>("overlay_get_image")
31+
.then((p) => {
32+
if (p) setSrc(convertFileSrc(p));
33+
})
34+
.catch(() => undefined);
35+
const onKey = (e: KeyboardEvent) => {
36+
if (e.key === "Escape") void invoke("overlay_cancel").catch(() => undefined);
37+
if (e.key === "Enter" && sel) void submit();
38+
};
39+
window.addEventListener("keydown", onKey);
40+
return () => window.removeEventListener("keydown", onKey);
41+
// eslint-disable-next-line react-hooks/exhaustive-deps
42+
}, [sel]);
43+
44+
const onLoad = () => {
45+
const img = imgRef.current;
46+
if (!img) return;
47+
setNatural({ w: img.naturalWidth, h: img.naturalHeight });
48+
// 1:1 换算:图像坐标 = 客户端坐标 ×(自然尺寸/显示尺寸)——DPI/多屏下依旧准确
49+
scaleRef.current = {
50+
x: img.naturalWidth / Math.max(1, img.clientWidth),
51+
y: img.naturalHeight / Math.max(1, img.clientHeight),
52+
};
53+
};
54+
55+
const toImageRect = (s: Sel) => {
56+
const img = imgRef.current;
57+
if (!img || !natural) return null;
58+
const box = img.getBoundingClientRect();
59+
const sx = Math.min(s.x1, s.x2);
60+
const sy = Math.min(s.y1, s.y2);
61+
const ex = Math.max(s.x1, s.x2);
62+
const ey = Math.max(s.y1, s.y2);
63+
const x = Math.round((sx - box.left) * scaleRef.current.x);
64+
const y = Math.round((sy - box.top) * scaleRef.current.y);
65+
const w = Math.round((ex - sx) * scaleRef.current.x);
66+
const h = Math.round((ey - sy) * scaleRef.current.y);
67+
if (w < 4 || h < 4) return null;
68+
return { x: Math.max(0, x), y: Math.max(0, y), w: Math.min(w, natural.w - x), h: Math.min(h, natural.h - y) };
69+
};
70+
71+
const submit = () => {
72+
if (!sel) return;
73+
const r = toImageRect(sel);
74+
if (!r) return;
75+
void invoke("overlay_submit_capture", { rect: r }).catch(() => undefined);
76+
};
77+
78+
const onMouseDown = (e: React.MouseEvent) => {
79+
dragging.current = true;
80+
setSel({ x1: e.clientX, y1: e.clientY, x2: e.clientX, y2: e.clientY });
81+
};
82+
const onMouseMove = (e: React.MouseEvent) => {
83+
if (!dragging.current || !sel) return;
84+
setSel({ ...sel, x2: e.clientX, y2: e.clientY });
85+
};
86+
const onMouseUp = () => {
87+
dragging.current = false;
88+
};
89+
90+
const selBox =
91+
sel && Math.abs(sel.x2 - sel.x1) > 3 && Math.abs(sel.y2 - sel.y1) > 3
92+
? {
93+
left: Math.min(sel.x1, sel.x2),
94+
top: Math.min(sel.y1, sel.y2),
95+
width: Math.abs(sel.x2 - sel.x1),
96+
height: Math.abs(sel.y2 - sel.y1),
97+
}
98+
: null;
99+
100+
return (
101+
<div
102+
style={{ position: "fixed", inset: 0, background: "#000", cursor: "crosshair", overflow: "hidden", userSelect: "none" }}
103+
onMouseDown={onMouseDown}
104+
onMouseMove={onMouseMove}
105+
onMouseUp={onMouseUp}
106+
>
107+
{src && (
108+
<img
109+
ref={imgRef}
110+
src={src}
111+
onLoad={onLoad}
112+
alt="待框选截图"
113+
draggable={false}
114+
style={{ position: "absolute", left: 0, top: 0, width: "100vw", imageRendering: "pixelated" }}
115+
/>
116+
)}
117+
{/* 框选遮罩:选中区域高亮,其余压暗 */}
118+
{selBox && (
119+
<div
120+
style={{
121+
position: "absolute",
122+
left: selBox.left - 1,
123+
top: selBox.top - 1,
124+
width: selBox.width + 2,
125+
height: selBox.height + 2,
126+
border: "2px solid #14b8a6",
127+
background: "rgba(20,184,166,0.15)",
128+
boxShadow: "0 0 0 9999px rgba(0,0,0,0.45)",
129+
}}
130+
/>
131+
)}
132+
{/* 操作条 */}
133+
<div style={{ position: "fixed", bottom: 24, left: "50%", transform: "translateX(-50%)", display: "flex", gap: 8, zIndex: 10 }}>
134+
<button
135+
style={{ padding: "8px 18px", borderRadius: 6, border: "none", background: "#0d9488", color: "#fff", fontWeight: 600, cursor: "pointer" }}
136+
disabled={!selBox}
137+
onClick={submit}
138+
>
139+
确认截图(Enter)
140+
</button>
141+
<button
142+
style={{ padding: "8px 18px", borderRadius: 6, border: "1px solid #d1d5db", background: "#fff", cursor: "pointer" }}
143+
onClick={() => void invoke("overlay_cancel").catch(() => undefined)}
144+
>
145+
取消(Esc)
146+
</button>
147+
<span style={{ alignSelf: "center", fontSize: 12, color: "#e5e7eb" }}>
148+
{natural ? `${natural.w}×${natural.h} 像素 · 1:1` : "加载截图…"}
149+
</span>
150+
</div>
151+
</div>
152+
);
153+
}

0 commit comments

Comments
 (0)