diff --git a/extensions/assistive_harness/README.md b/extensions/assistive_harness/README.md index 904aedf..8be88a5 100644 --- a/extensions/assistive_harness/README.md +++ b/extensions/assistive_harness/README.md @@ -1,23 +1,7 @@ -# Assistive Voice Skill Harness (Phase A / Phase B Core) +# Assistive Voice Skill Harness (Phase A) -This directory is the transport-neutral control Core shared by the optional -MiniCPM-o browser sidecar and OpenGlass Phase B device adapters. It is disabled -by default and does not replace the native microphone/video path. - -## Clean-clone setup - -From the OpenGlass repository root, install the Core and explicitly download -the public ASR model: - -```powershell -python -m pip install -r extensions/assistive_harness/requirements.txt -python -m extensions.assistive_harness.download_modelscope_model -``` - -The second command prints the downloaded local directory and a complete start -command. Model weights are not stored in this Git repository. The runtime never -downloads a model implicitly: `--model-path` must point to an existing local -directory, otherwise startup fails clearly. +This directory is an optional, local sidecar for the MiniCPM-o browser Demo. It is +disabled by default and does not replace the native microphone/video path. Start the sidecar explicitly: @@ -26,10 +10,7 @@ python -m extensions.assistive_harness.server --enabled ` --model-path "C:\path\to\a\local\FunASR\model" ``` -Then opt the browser tab in with `?assistive_harness=1`. Browser integration -assets and their hook contract live in -[`../../integrations/minicpm_browser/`](../../integrations/minicpm_browser/). -Test-only transcript +Then opt the browser tab in with `?assistive_harness=1`. Test-only transcript injection additionally requires `--allow-test-injection`; it is never enabled by the normal command above. diff --git a/extensions/assistive_harness/__init__.py b/extensions/assistive_harness/__init__.py index f40e11f..339e7b9 100644 --- a/extensions/assistive_harness/__init__.py +++ b/extensions/assistive_harness/__init__.py @@ -11,3 +11,4 @@ "RuleIntentRouter", "SkillRegistry", ] + diff --git a/extensions/assistive_harness/asr/__init__.py b/extensions/assistive_harness/asr/__init__.py index c3bbd54..6d88083 100644 --- a/extensions/assistive_harness/asr/__init__.py +++ b/extensions/assistive_harness/asr/__init__.py @@ -3,3 +3,4 @@ from .funasr_engine import FunASREngine __all__ = ["ASREngine", "ASRResult", "EnergyVAD", "FunASREngine", "UtteranceAudio"] + diff --git a/extensions/assistive_harness/asr/base.py b/extensions/assistive_harness/asr/base.py index 511f465..a6fc73d 100644 --- a/extensions/assistive_harness/asr/base.py +++ b/extensions/assistive_harness/asr/base.py @@ -27,3 +27,4 @@ def transcribe(self, audio: np.ndarray, sample_rate: int) -> ASRResult: del audio, sample_rate text = self.transcripts.pop(0) if self.transcripts else "" return ASRResult(text=text, confidence=1.0, model="scripted", device="cpu") + diff --git a/extensions/assistive_harness/asr/energy_vad.py b/extensions/assistive_harness/asr/energy_vad.py index 2de6ef1..58cec2e 100644 --- a/extensions/assistive_harness/asr/energy_vad.py +++ b/extensions/assistive_harness/asr/energy_vad.py @@ -89,3 +89,4 @@ def reset(self) -> None: self._active = [] self._started_at_ms = None self._last_voice_ms = None + diff --git a/extensions/assistive_harness/config/skills.example.yaml b/extensions/assistive_harness/config/skills.example.yaml index f2b4c76..4e589ff 100644 --- a/extensions/assistive_harness/config/skills.example.yaml +++ b/extensions/assistive_harness/config/skills.example.yaml @@ -31,9 +31,9 @@ cv: type: noop yolo_onnx: type: yolo_onnx - # Resolved relative to this YAML. Weights stay in OpenGlass/models and - # are ignored by Git; the Core never downloads them implicitly. - model_path: ../../../models/yolo26n.onnx + # Resolved relative to this YAML. Keep weights outside the Core so the + # same plugin can be reused by the later OpenGlass Adapter. + model_path: ../../../../OmniHarness/mini_omni_harness/models/yolo26n.onnx device: cpu confidence: 0.25 image_size: 640 diff --git a/extensions/assistive_harness/cv/README.md b/extensions/assistive_harness/cv/README.md index 2165f26..6c986c2 100644 --- a/extensions/assistive_harness/cv/README.md +++ b/extensions/assistive_harness/cv/README.md @@ -69,8 +69,8 @@ skills: ``` Relative model paths are resolved against the YAML directory. Weights are not -owned by the Core. The shipped sample resolves to the Git-ignored local file -`OpenGlass/models/yolo26n.onnx`. +owned by the Core. The current local sample points to the previously validated +`OmniHarness/mini_omni_harness/models/yolo26n.onnx` file. ## Provider contract diff --git a/extensions/assistive_harness/package_phase_a.ps1 b/extensions/assistive_harness/package_phase_a.ps1 new file mode 100644 index 0000000..8264580 --- /dev/null +++ b/extensions/assistive_harness/package_phase_a.ps1 @@ -0,0 +1,69 @@ +param( + [string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path, + [string]$Date = '2026-08-11' +) + +$ErrorActionPreference = 'Stop' +$deliverables = Join-Path $RepoRoot 'deliverables' +$packageName = "VOICE_SKILL_HARNESS_PHASE_A_CODEX_RETURN_$Date" +$stage = Join-Path $deliverables $packageName +$zipPath = Join-Path $deliverables "$packageName.zip" + +$resolvedRepo = (Resolve-Path $RepoRoot).Path.TrimEnd('\') +New-Item -ItemType Directory -Force -Path $deliverables | Out-Null +$resolvedDeliverables = (Resolve-Path $deliverables).Path +if (-not $resolvedDeliverables.StartsWith($resolvedRepo, [StringComparison]::OrdinalIgnoreCase)) { + throw "Unsafe delivery target: $resolvedDeliverables" +} +if (Test-Path $stage) { Remove-Item -LiteralPath $stage -Recurse -Force } +if (Test-Path $zipPath) { Remove-Item -LiteralPath $zipPath -Force } +New-Item -ItemType Directory -Force -Path $stage | Out-Null + +function Copy-TreeFiltered([string]$Source, [string]$Destination) { + Get-ChildItem -LiteralPath $Source -Recurse -File | Where-Object { + $_.FullName -notmatch '[\\/]runs[\\/]' -and + $_.FullName -notmatch '[\\/]__pycache__[\\/]' -and + $_.FullName -notmatch '[\\/]test_artifacts[\\/]' -and + $_.Extension -ne '.pyc' + } | ForEach-Object { + $relative = $_.FullName.Substring($Source.TrimEnd('\').Length).TrimStart('\') + $target = Join-Path $Destination $relative + New-Item -ItemType Directory -Force -Path (Split-Path $target) | Out-Null + Copy-Item -LiteralPath $_.FullName -Destination $target + } +} + +Copy-TreeFiltered (Join-Path $RepoRoot 'extensions\assistive_harness') (Join-Path $stage 'extensions\assistive_harness') +Copy-TreeFiltered (Join-Path $RepoRoot 'static\assistive_harness') (Join-Path $stage 'static\assistive_harness') +New-Item -ItemType Directory -Force -Path (Join-Path $stage 'static\omni') | Out-Null +Copy-Item -LiteralPath (Join-Path $RepoRoot 'static\omni\omni-app.js') -Destination (Join-Path $stage 'static\omni\omni-app.js') + +$docNames = @( + 'VOICE_SKILL_HARNESS_AUDIT.md', + 'VOICE_SKILL_HARNESS_DESIGN.md', + 'VOICE_SKILL_HARNESS_EXECUTION_REPORT.md', + 'VOICE_SKILL_HARNESS_GO_NO_GO.md', + 'VOICE_SKILL_HARNESS_RERUN_COMMANDS.md', + 'VOICE_SKILL_HARNESS_CHANGED_FILES.md' +) +$docTarget = Join-Path $stage '_codex_context' +New-Item -ItemType Directory -Force -Path $docTarget | Out-Null +foreach ($name in $docNames) { + $text = Get-Content -Raw -Encoding UTF8 (Join-Path $RepoRoot "_codex_context\$name") + $text = $text.Replace($env:USERPROFILE, '%USERPROFILE%') + Set-Content -Encoding UTF8 -NoNewline -Path (Join-Path $docTarget $name) -Value $text +} + +$manifest = [ordered]@{ + project = 'Voice-Controlled Skill Harness' + phase = 'A_MINICPMO_BROWSER' + date = $Date + ready_for_phase_b = $false + feature_default = 'off' + excluded = @('model weights', 'raw audio/video', 'runs', 'credentials', 'absolute user paths') +} +$manifest | ConvertTo-Json -Depth 5 | Set-Content -Encoding UTF8 (Join-Path $stage 'MANIFEST.json') + +Compress-Archive -Path (Join-Path $stage '*') -DestinationPath $zipPath -CompressionLevel Optimal +$hash = (Get-FileHash -Algorithm SHA256 -LiteralPath $zipPath).Hash +[pscustomobject]@{Zip=$zipPath; SHA256=$hash} diff --git a/extensions/assistive_harness/phase_b/bridge_ui.py b/extensions/assistive_harness/phase_b/bridge_ui.py new file mode 100644 index 0000000..c3940f8 --- /dev/null +++ b/extensions/assistive_harness/phase_b/bridge_ui.py @@ -0,0 +1,337 @@ +"""bridge_ui.py — ESP32 duplex bridge 的 Web 观测/回放服务 + +URL: + / 实时观测(摄像头 + 字幕 + 状态) + /replay session 列表 + /replay/ session 回放(直接播 live_session.mp4,字幕跟随) + +依赖 recorder_live.py v5+ 写到 session_dir 的成品: + live_session.mp4 ← 播放主体(ffmpeg 已对齐) + live_session.m4a ← 没有视频帧时的纯音频回退 + live_user.wav ← 诊断下载 + live_ai.wav ← 诊断下载 + events.jsonl ← 字幕/事件源 + meta.json ← 会话元信息 +""" + +from __future__ import annotations +from typing import Optional, Set, Callable +import json +import logging +from pathlib import Path +from typing import Optional, Set + +from aiohttp import web + +LOGGER = logging.getLogger("bridge_ui") + +# 模板目录(和 bridge_ui.py 同级的 templates/) +TEMPLATES_DIR = Path(__file__).parent / "templates" + +# 开发期想"改完刷新浏览器就生效"就设 True;生产环境设 False 只读一次 +HOT_RELOAD_TEMPLATES = True + +_template_cache: dict[str, str] = {} + +def _load_template(name: str) -> str: + """读取 templates/。HOT_RELOAD_TEMPLATES=True 时每次都重读。""" + if not HOT_RELOAD_TEMPLATES and name in _template_cache: + return _template_cache[name] + path = TEMPLATES_DIR / name + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + LOGGER.error("[UI] template not found: %s", path) + return f"

Template not found: {name}

" + _template_cache[name] = text + return text + + +# ============================================================ +# Server +# ============================================================ + +class WebUIServer: + def __init__(self, port: int = 8080, host: str = "0.0.0.0", + sessions_root: Path = Path("./sessions"), + stop_callback: Optional[Callable[[], None]] = None, + mode_info: Optional[dict] = None): + self.port = port + self.host = host + self.sessions_root = Path(sessions_root) + self.live_clients: Set[web.WebSocketResponse] = set() + self._runner: Optional[web.AppRunner] = None + self._stop_callback = stop_callback # ← 新增 + self._stop_fired = False # ← 新增,防重入 + self.mode_info = mode_info or {"mode": "live"} + + async def start(self) -> None: + app = web.Application() + app.router.add_get('/', self._h_live) + app.router.add_get('/replay', self._h_replay_index) + app.router.add_get('/replay/{sid}', self._h_replay) + app.router.add_get('/live_ws', self._h_live_ws) + app.router.add_get('/api/sessions', self._h_list) + app.router.add_get('/api/session/{sid}/events', self._h_events) + app.router.add_get('/api/session/{sid}/meta', self._h_meta) + # —— v2: 直接吐成品文件,FileResponse 自带 HTTP Range 支持 —— + app.router.add_get('/api/session/{sid}/video', self._h_video) + app.router.add_get('/api/session/{sid}/audio', self._h_audio_only) + app.router.add_get('/api/session/{sid}/user_audio.wav', self._h_user_wav) + app.router.add_get('/api/session/{sid}/ai_audio.wav', self._h_ai_wav) + # 兼容老路由(可能旧版 events.jsonl 引用 images/xxx.jpg) + app.router.add_get('/api/session/{sid}/{path:images/.+}', self._h_image_legacy) + app.router.add_get('/api/session/{sid}/{path:live_images/.+}', self._h_image_live) + app.router.add_post('/api/stop', self._h_stop) + self._runner = web.AppRunner(app) + await self._runner.setup() + site = web.TCPSite(self._runner, self.host, self.port) + await site.start() + LOGGER.info("[UI] http://%s:%d (live + replay)", self.host, self.port) + + async def stop(self) -> None: + for ws in list(self.live_clients): + try: await ws.close() + except Exception: pass + if self._runner: + await self._runner.cleanup() + + async def emit(self, evt: dict) -> None: + if not self.live_clients: + return + msg = json.dumps(evt, ensure_ascii=False) + dead = [] + for ws in list(self.live_clients): + try: await ws.send_str(msg) + except Exception: dead.append(ws) + for ws in dead: + self.live_clients.discard(ws) + + # ---- handlers ---- + async def _h_live(self, request): + return web.Response(text=_load_template("live.html"), content_type='text/html') + + async def _h_stop(self, request): + if self._stop_fired: + return web.json_response({"ok": True, "already": True}) + self._stop_fired = True + LOGGER.info("[UI] /api/stop triggered by browser") + # 通知所有 live 客户端"要关了",前端可以把按钮改成"Saving..." + try: + await self.emit({"type": "stopping"}) + except Exception: + pass + if self._stop_callback is not None: + try: + self._stop_callback() + except Exception as e: + LOGGER.warning("[UI] stop_callback err: %s", e) + return web.json_response({"ok": True}) + + async def _h_replay_index(self, request): + return web.Response(text=_load_template("replay_index.html"), content_type='text/html') + + async def _h_replay(self, request): + return web.Response(text=_load_template("replay.html"), content_type='text/html') + + async def _h_live_ws(self, request): + ws = web.WebSocketResponse(heartbeat=30) + await ws.prepare(request) + self.live_clients.add(ws) + + LOGGER.info("[UI] live client +1 (total=%d)", len(self.live_clients)) + try: + await ws.send_str(json.dumps( + {"type": "mode", **self.mode_info}, ensure_ascii=False)) + except Exception: + pass + + try: + async for _ in ws: pass + finally: + self.live_clients.discard(ws) + LOGGER.info("[UI] live client -1 (total=%d)", len(self.live_clients)) + return ws + + async def _h_list(self, request): + sessions = [] + if self.sessions_root.exists(): + for d in sorted(self.sessions_root.iterdir(), reverse=True): + if not d.is_dir(): + continue + meta_p = d / "meta.json" + meta = {} + if meta_p.exists(): + try: + meta = json.loads(meta_p.read_text(encoding='utf-8')) + except Exception: + pass + if (d / "live_session.mp4").exists(): + media = "mp4" + elif (d / "live_session.m4a").exists(): + media = "m4a" + else: + media = None + sessions.append({ + "id": d.name, + "tag": meta.get("session_tag", d.name), + "start_time": meta.get("start_time"), + "end_time": meta.get("end_time"), + "stats": meta.get("stats", {}), + "media": media, + }) + return web.json_response({"sessions": sessions}) + + def _safe(self, sid: str) -> Optional[Path]: + if "/" in sid or ".." in sid: + return None + p = (self.sessions_root / sid).resolve() + try: + p.relative_to(self.sessions_root.resolve()) + except ValueError: + return None + if not p.is_dir(): + return None + return p + + async def _h_meta(self, request): + p = self._safe(request.match_info['sid']) + if not p: + return web.json_response({"error": "not found"}, status=404) + meta = p / "meta.json" + if not meta.exists(): + return web.json_response({"error": "no meta"}, status=404) + return web.json_response(json.loads(meta.read_text(encoding='utf-8'))) + + async def _h_events(self, request): + p = self._safe(request.match_info['sid']) + if not p: + return web.json_response({"error": "not found"}, status=404) + ev = p / "events.jsonl" + if not ev.exists(): + return web.json_response({"events": []}) + out = [] + for line in ev.read_text(encoding='utf-8').splitlines(): + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except Exception: + pass + return web.json_response({"events": out}) + + # ---- 媒体:直接 FileResponse(支持 Range,可拖动进度条)---- + + async def _h_video(self, request): + p = self._safe(request.match_info['sid']) + if not p: + return web.Response(status=404) + mp4 = p / "live_session.mp4" + if not mp4.exists(): + return web.Response(status=404, text="no live_session.mp4") + return web.FileResponse(mp4, headers={"Content-Type": "video/mp4"}) + + async def _h_audio_only(self, request): + p = self._safe(request.match_info['sid']) + if not p: + return web.Response(status=404) + m4a = p / "live_session.m4a" + if not m4a.exists(): + return web.Response(status=404, text="no live_session.m4a") + return web.FileResponse(m4a, headers={"Content-Type": "audio/mp4"}) + + async def _h_user_wav(self, request): + p = self._safe(request.match_info['sid']) + if not p: + return web.Response(status=404) + w = p / "live_user.wav" + if not w.exists(): + return web.Response(status=404) + return web.FileResponse(w, headers={"Content-Type": "audio/wav"}) + + async def _h_ai_wav(self, request): + p = self._safe(request.match_info['sid']) + if not p: + return web.Response(status=404) + w = p / "live_ai.wav" + if not w.exists(): + return web.Response(status=404) + return web.FileResponse(w, headers={"Content-Type": "audio/wav"}) + + async def _h_image_live(self, request): + return await self._serve_image(request, subdir="live_images") + + async def _h_image_legacy(self, request): + return await self._serve_image(request, subdir="images") + + async def _serve_image(self, request, subdir: str): + p = self._safe(request.match_info['sid']) + if not p: + return web.Response(status=404) + rel = request.match_info['path'] + # rel 形如 "live_images/img_00001.jpg" 或 "images/xxx.jpg" + # 这里只取末尾文件名,防穿越 + name = Path(rel).name + if "/" in name or ".." in name or not name: + return web.Response(status=400) + img = p / subdir / name + if not img.exists(): + return web.Response(status=404) + return web.FileResponse(img, headers={"Content-Type": "image/jpeg"}) + + +# ============================================================ +# Standalone entry +# ============================================================ + +def _main() -> None: + import argparse + import asyncio + + parser = argparse.ArgumentParser( + description="Bridge UI — standalone replay server (no ESP32 / no model)" + ) + parser.add_argument("--port", type=int, default=8080) + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--sessions", default="./sessions", + help="sessions 根目录(默认 ./sessions)") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + ) + + sessions_root = Path(args.sessions).resolve() + if not sessions_root.exists(): + LOGGER.warning("[UI] sessions dir 不存在: %s(之后录到这里就能看见)", + sessions_root) + + server = WebUIServer( + port=args.port, + host=args.host, + sessions_root=sessions_root + ) + + async def _run(): + await server.start() + LOGGER.info("[UI] standalone mode — replay only") + LOGGER.info("[UI] sessions root: %s", sessions_root) + LOGGER.info("[UI] open http://localhost:%d/replay", args.port) + try: + while True: + await asyncio.sleep(3600) + except (KeyboardInterrupt, asyncio.CancelledError): + pass + finally: + await server.stop() + + try: + asyncio.run(_run()) + except KeyboardInterrupt: + print("\n[UI] bye.") + + +if __name__ == "__main__": + _main() \ No newline at end of file diff --git a/extensions/assistive_harness/phase_b/cam_pipeline_v2.py b/extensions/assistive_harness/phase_b/cam_pipeline_v2.py new file mode 100644 index 0000000..fdd2b8c --- /dev/null +++ b/extensions/assistive_harness/phase_b/cam_pipeline_v2.py @@ -0,0 +1,2283 @@ +# -*- coding: utf-8 -*- +"""cam_tuner.py — ESP32 相机调参台 (形态1: 纯调参, 不接模型) + +目的: 一边看 TCP live 画面, 一边热调分辨率/对焦/quality, 实时看每帧的 + 拉普拉斯清晰度分 + 亮度 —— 为三级漏斗一级标定阈值。 + +用法: + python cam_tuner.py --ip 10.100.7.68 + 浏览器开 http://localhost:8080 + +注意 (硬约束): + 固件 TCP 5000 是单客户端串行。本工具独占 TCP —— 调参时不要同时跑主程序 + (pc_vlm / demo_esp32), 否则抢同一条 TCP 连接会错位。 + +依赖: aiohttp, opencv-python(cv2), numpy, requests +""" + +from __future__ import annotations + +import argparse +import asyncio +import socket +import struct +import threading +import time +from pathlib import Path +from typing import Optional + +import cv2 +import numpy as np +import requests +from aiohttp import web + + +# ============================================================ +# TCP 5000 抓帧 (复用已验证协议; 单连接 + 锁, 后台线程持有) +# ============================================================ +class TCPImageClient: + def __init__(self, ip: str, port: int = 5000, timeout: float = 2.0): + self.ip, self.port, self.timeout = ip, port, timeout + self._sock = None + self._lock = threading.Lock() + + def _connect_locked(self) -> bool: + try: + self._sock = socket.create_connection((self.ip, self.port), timeout=self.timeout) + self._sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self._sock.settimeout(self.timeout) + return True + except Exception: + self._sock = None + return False + + def _recv_exactly(self, n: int) -> Optional[bytes]: + buf = b"" + while len(buf) < n: + chunk = self._sock.recv(n - len(buf)) + if not chunk: + return None + buf += chunk + return buf + + def capture(self) -> Optional[bytes]: + with self._lock: + for attempt in (1, 2): + if self._sock is None and not self._connect_locked(): + return None + try: + self._sock.sendall(b"\x01") + hdr = self._recv_exactly(20) + if not hdr: + raise ConnectionError("no header") + magic = struct.unpack_from(" bool: + try: + r = self.sess.get(f"{self.base}/control", + params={"var": var, "val": val}, timeout=self.timeout) + return r.status_code == 200 + except Exception: + return False + + def set_resolution(self, name: str) -> bool: + v = FRAMESIZE.get(name.upper()) + return self.control("framesize", v) if v is not None else False + + def set_quality(self, q: int) -> bool: + return self.control("quality", max(0, min(63, int(q)))) + + def trigger_af(self) -> bool: + # 单次对焦: 直接写 OV5640 寄存器 0x3022=0x03 (single auto focus)。 + # 固件用标准 esp32-camera web server, 没有实现 /control?var=af (af 非标准变量), + # 之前先试 control("af") 会"假成功"(固件忽略)而不真对焦 —— 故直接走 /reg。 + # 固件端应设 g_auto_af=false, 关掉每秒定时硬对焦, 对焦完全由此按需触发。 + try: + r = self.sess.get(f"{self.base}/reg", + params={"reg": 0x3022, "mask": 0xff, "val": 0x03}, + timeout=self.timeout) + return r.status_code == 200 + except Exception: + return False + + +def rotate_jpeg(jpg: bytes, deg: int) -> bytes: + """后端真旋转 (顺时针 deg∈{0,90,180,270})。返回旋转后的 JPEG bytes。 + 真转而非只转显示: 因为最终自动旋转策略判断后会真喂给模型, 数据路径要一致; + 且清晰度分基于转后图算, 可验证'旋转不改变清晰度'(锐利度与朝向正交)。""" + d = deg % 360 + if d == 0: + return jpg + arr = np.frombuffer(jpg, dtype=np.uint8) + img = cv2.imdecode(arr, cv2.IMREAD_COLOR) + if img is None: + return jpg + if d == 90: + img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE) + elif d == 180: + img = cv2.rotate(img, cv2.ROTATE_180) + elif d == 270: + img = cv2.rotate(img, cv2.ROTATE_90_COUNTERCLOCKWISE) + else: + return jpg + ok, enc = cv2.imencode(".jpg", img, [cv2.IMWRITE_JPEG_QUALITY, 90]) + return enc.tobytes() if ok else jpg + + +def _decode_gray(jpg: bytes): + arr = np.frombuffer(jpg, dtype=np.uint8) + return cv2.imdecode(arr, cv2.IMREAD_GRAYSCALE) + + +def frame_motion(jpg_a: bytes, jpg_b: bytes) -> float: + """两帧灰度平均绝对差 (0-255)。大 = 画面在动 (快速移动/转头)。 + **先高斯模糊再算差**: 抹掉密集文字的高频细节, 只保留宏观位移 —— + 否则密集文字面(成分面)手持轻微移动时每个小字边缘都产生大量像素差, motion 虚高, + 把可读的图误判成"在动"(真机+OCR校准证实成分面可读却被全拒)。 + 漏斗二级用: 一批帧相邻 diff 都大 -> 画面不稳 -> 拒绝(念字必错, 让用户停稳)。""" + a, b = _decode_gray(jpg_a), _decode_gray(jpg_b) + if a is None or b is None: + return 0.0 + if a.shape != b.shape: + b = cv2.resize(b, (a.shape[1], a.shape[0])) + a = cv2.GaussianBlur(a, (7, 7), 0) + b = cv2.GaussianBlur(b, (7, 7), 0) + return float(np.abs(a.astype(np.int16) - b.astype(np.int16)).mean()) + + +def optical_flow_motion(jpg_a: bytes, jpg_b: bytes, downscale: float = 0.35) -> float: + """两帧稠密光流(Farneback)的平均位移幅度(像素)。物理含义明确、可解释: + 画面整体移动了多少像素。用于【严重晃动一刀切】—— 相机大幅位移时光流幅度大。 + 比 frame_motion(逐像素差)更鲁棒: 光流看运动矢量场, 不被密集文字的高频细节干扰 + (密集字轻微移动 -> 光流一致小位移; 逐像素差却虚高)。降采样提速(判"严重"够用)。""" + a, b = _decode_gray(jpg_a), _decode_gray(jpg_b) + if a is None or b is None: + return 0.0 + if a.shape != b.shape: + b = cv2.resize(b, (a.shape[1], a.shape[0])) + if downscale != 1.0: + a = cv2.resize(a, None, fx=downscale, fy=downscale, interpolation=cv2.INTER_AREA) + b = cv2.resize(b, None, fx=downscale, fy=downscale, interpolation=cv2.INTER_AREA) + flow = cv2.calcOpticalFlowFarneback(a, b, None, + pyr_scale=0.5, levels=3, winsize=15, + iterations=3, poly_n=5, poly_sigma=1.2, flags=0) + mag = np.sqrt(flow[..., 0] ** 2 + flow[..., 1] ** 2) + # 位移幅度按降采样比例还原到原图尺度, 便于用原图像素单位设阈值 + return float(mag.mean() / max(downscale, 1e-6)) + + +def bg_signals(jpg_bytes): + """候选'背景干扰'信号(都可解释, 检测画面有额外背景 -> 提示拿近/对准)。 + 干净白纸标签: 大片均匀白 + 中间少量黑字; 有背景(灯管/工位/天花板)时信号异常。 + 也含清晰度类(local_sharp/worst_block/sharp_ratio)供评分。返回 dict。""" + arr = np.frombuffer(jpg_bytes, np.uint8) + g = cv2.imdecode(arr, cv2.IMREAD_GRAYSCALE) + if g is None: + return {} + h, w = g.shape + # 信号1: 亮度直方图熵 —— 干净标签(白底+黑字)分布集中熵低; 背景杂->熵高 + hist = cv2.calcHist([g], [0], None, [32], [0, 256]).flatten() + p = hist / (hist.sum() + 1e-9) + entropy = float(-np.sum(p * np.log2(p + 1e-12))) + # 信号2: 最大均匀(低方差)区占比 —— 白纸有大片均匀白; 占比低=画面杂乱 + blur = cv2.GaussianBlur(g, (5, 5), 0) + localvar = cv2.blur((g.astype(np.float32) - blur.astype(np.float32)) ** 2, (15, 15)) + lowvar = (localvar < 30).astype(np.uint8) + n, labels, stats, _ = cv2.connectedComponentsWithStats(lowvar) + max_uniform = float(stats[1:, cv2.CC_STAT_AREA].max() / g.size) if n > 1 else 0.0 + # 信号3: 边缘空间分布 —— 干净标签边缘集中中间(文字); 背景使边缘散布/落外围 + edges = cv2.Canny(g, 50, 150) + ys, xs = np.where(edges > 0) + if len(xs) > 10: + cx, cy = w / 2, h / 2 + d = np.sqrt(((xs - cx) / w) ** 2 + ((ys - cy) / h) ** 2) + edge_spread = float(d.mean()) + edge_periph = float((d > 0.35).mean()) # 边缘落外围(远离中心)比例 + else: + edge_spread = 0.0; edge_periph = 0.0 + # 信号4: local_sharp / worst_block —— 有高对比背景(灯管)时 local虚高而整体糊 + lap = cv2.Laplacian(g, cv2.CV_64F) + local_sharp = float(lap.var()) + grid = 6 + vals = [] + for i in range(grid): + for j in range(grid): + blk = g[i*h//grid:(i+1)*h//grid, j*w//grid:(j+1)*w//grid] + if blk.size == 0: + continue + e = cv2.Canny(blk, 50, 150) + if (e > 0).mean() < 0.006: + continue + vals.append(float(cv2.Laplacian(blk, cv2.CV_64F).var())) + worst = min(vals) if vals else 0.0 + sharp_ratio = float(local_sharp / (worst + 1e-6)) if worst > 0 else 0.0 + return { + "bright_entropy": round(entropy, 2), + "max_uniform": round(max_uniform, 3), + "edge_spread": round(edge_spread, 3), + "edge_periph": round(edge_periph, 3), + "sharp_ratio": round(sharp_ratio, 1), + "local_sharp": round(local_sharp, 1), + "worst_block": round(worst, 1), + } + + +# ============================================================ +# 三级漏斗 (形态2核心) +# 输入: 一批 N 帧 (bytes list) +# 一级 单帧质量筛: 太糊/太暗/空白 的帧标记不合格 +# 二级 稳定性判定: 相邻帧 motion 都过大 -> 整批拒绝 (画面不稳) +# 三级 选优: 合格帧里选 sharpness 最高的 = best +# 全不合格 / 不稳 -> 返回拒绝 (安全闸, 不喂模型) +# +# 阈值说明: 分辨率固定 HD 后针对 HD 标定。这里给的是初始默认值, +# 你戴眼镜实测后按 CSV 数据调 (尤其 SHARP_MIN / MOTION_MAX)。 +# sharp 低 + edge 低 = 空白(拒绝, 说"没看到文字"); 用 EDGE_MIN 兜。 +# ============================================================ +import math + + +def _g_sat(x: float, x0: float) -> float: + """Michaelis-Menten 饱和: x/(x+x0) -> [0,1)。x0=半饱和点。""" + x = max(0.0, x) + return x / (x + x0) if (x + x0) > 0 else 0.0 + + +def _g_light(b: float, lo: float, hi: float, soft: float) -> float: + """亮度响应: 太暗/过曝降分, 中间平台=1。两 sigmoid 相夹。""" + def sig(z): + try: + return 1.0 / (1.0 + math.exp(-z)) + except OverflowError: + return 0.0 if z < 0 else 1.0 + return sig((b - lo) / soft) * (1 - sig((b - hi) / soft)) + + +def frame_score(metrics: dict, rel_change: float, cfg: "FunnelConfig") -> dict: + """对一帧算 quality + focus_gain + 各分量 (供解释/记录)。 + rel_change: 该帧相对上一帧的 sharp 相对变化 (稳定性输入); 首帧传 0。""" + # 清晰度用 worst_block(最糊的有内容块) 而非全局 sharpness —— + # 全局拉普拉斯被大面积白底稀释(白底无边缘, 拉普拉斯低), 把"清晰的白盒子标签"误判成糊, + # 触发假的 need_focus。worst_block 只看有内容的最糊块, 不被白底拉低。 + # 真机+OCR校准: 不可读帧 worst_block<4.6, 可读>5.5 -> 分界~5, 故 x0 用 worst_x0(~5量级)。 + worst = metrics.get("worst_block", None) + if worst is None: + worst = metrics.get("sharpness", 0.0) # 兜底: 无 worst_block 时退回全局 + g_sharp = _g_sat(worst, cfg.worst_x0) + g_stable = max(0.0, 1.0 - min(1.0, rel_change)) # rel_change 已归一化到[0,1] + g_content = _g_sat(metrics.get("edge_density", 0.0), cfg.edge_x0) + g_light = _g_light(metrics.get("brightness", 0.0), cfg.light_lo, cfg.light_hi, cfg.light_soft) + + wsum = cfg.w_sharp + cfg.w_stable + cfg.w_content + cfg.w_light + base = (cfg.w_sharp * g_sharp + cfg.w_stable * g_stable + + cfg.w_content * g_content + cfg.w_light * g_light) / wsum + sharp_gate = g_sharp ** cfg.sharp_gate_gamma # 念字硬要求: 糊则压分 + quality = base * sharp_gate + # focus_gain: 稳 + 糊 时该对焦。对 content 用"温和依赖"(sqrt)而非硬乘 —— + # 重度失焦会把边缘也糊没(g_content 低), 但它恰恰最该对焦, 不能因此归零; + # sqrt 让低内容时 focus_gain 降低但保留触发机会。真空白由 g_content 极低 + + # g_stable 高的组合另行识别(见 run_funnel 拒绝归因)。 + focus_gain = g_stable * (1 - g_sharp) * math.sqrt(max(0.0, g_content)) + return { + "quality": round(quality, 3), + "focus_gain": round(focus_gain, 3), + "g_sharp": round(g_sharp, 3), + "g_stable": round(g_stable, 3), + "g_content": round(g_content, 3), + "g_light": round(g_light, 3), + } + + +class FunnelConfig: + """评分函数 f 的参数 (替代硬阈值 if)。每帧算一个连续 quality 分 + focus_gain, + 决策基于分数而非离散阈值 —— 可解释、可 ablation、可用数据校准。 + + quality = [Σ wi·gi] · g_sharp^gamma (sharp 作乘性门, 念字硬要求) + g_sharp = sharp/(sharp+sharp_x0) 清晰度饱和响应 + g_stable = 1 - min(1, rel_change/change_ref) 帧间相对变化越小越稳 + g_content = edge/(edge+edge_x0) 有无内容(区分糊/空) + g_light = 亮度双sigmoid(暗/过曝都降) + focus_gain = g_stable·(1-g_sharp)·g_content 稳+糊+有内容 -> 该对焦 + 决策: max(quality) >= ACCEPT_Q -> 选该帧; 否则看 max(focus_gain) >= FOCUS_G + -> 触发对焦再采; 都低 -> 拒绝。 + 参数是物理动机初值, 戴眼镜采数据后可校准/拟合 (方法 vs 调参的区别)。 + """ + # g 响应形状 (半饱和点/参考值) + sharp_x0 = 120.0 # (记录用) 全局清晰度半饱和点; g_sharp 现改用 worst_block + worst_x0 = 6.5 # worst_block 半饱和点。调严(原5.0)要求更清晰。真机+OCR: 不可读<4.6/可读>5.5。 + # g_sharp = worst/(worst+worst_x0): worst=5时g_sharp=0.5, 白盒清晰标签 + # worst 正常(不被白底稀释)故不误判need_focus。阈值待白标签标准数据精校。 + change_ref = 0.5 # (旧, 已弃用: 稳定性改用帧间像素差) + motion_ref = 15.0 # 帧间模糊后平均像素差达此视为"完全在动"。校准依据: OCR证实可读的 + # 成分面(密集字)模糊后 motion 达~8.4 仍可读, 须能过; 故 ref 上调到15, + # 使可读帧 stable>=0.44 通过, 同时剧烈晃动(>15)仍被拒。下界待补采晃动样本精校。 + edge_x0 = 0.01 # 边缘密度半饱和 (区分有内容/空白) + light_lo = 40.0 # 亮度下沿 + light_hi = 220.0 # 亮度上沿(过曝) + light_soft = 25.0 # 亮度软边宽度 + # 权重 (语义清晰, 可做 ablation) + w_sharp = 1.0 + w_stable = 1.0 + w_content = 0.6 + w_light = 0.4 + sharp_gate_gamma = 0.5 # sharp 乘性门强度 (0=不否决, 大=糊帧强烈压分) + # 决策阈值 (作用在归一化综合分上, 只此两个, 远少于原来一堆硬阈值) + ACCEPT_Q = 0.55 # 可用性门: 调严(原0.40太松, 放过模糊图)。0.55 要求更清晰才放行。 + # 宁拒绝不念错: 送下游的必须够清晰, 模糊的宁可让用户重拍。 + # 放宽的理由(数据得出): 全局quality分不开'能OCR'与'轻微糊', + # 那是任务层的精判(局部/文字区清晰度), 通用层不越俎代庖。 + FOCUS_G = 0.35 # (保留) focus_gain 参考 + # 稳定性优先三出口的门限 (段级中位数上判定; 待真机 rerun 校准) + STABLE_MIN = 0.35 # 出口A: g_stable 低于此 = 持续在动 -> "请拿稳" (第一闸)。 + # 从0.45降到0.35: 配合 motion_ref=15, 让OCR证实可读的密集字面通过, 修复误拒。 + SEVERE_FLOW = 8.0 # 出口A0: 段级光流位移(像素)>=此 = 严重晃动一刀切拒绝。初值8, + # 待 shake 样本校准("多大位移算严重"); 光流位移物理可解释。 + LIGHT_MIN = 0.35 # 出口D(留): g_light 低于此且为最差 -> 太暗 + CONTENT_MIN = 0.30 # 出口E(留): g_content 低于此 + SHARP_OK_FOR_E = 0.45 # 出口E(留): 判"没对准"要求 sharp 够高(清晰却没字才算对错) + MIN_QUALIFIED = 1 + + +# ============================================================ +# 场景配置 (核心: 机制一套, 参数按场景标定 —— "权重由场景物理特征+安全等级决定") +# 药盒(medicine): 小字/高危/白底 -> 偏拒绝、要求清晰、零容忍。= 默认 FunnelConfig。 +# 文具(stationery): 中大字/低危/彩色包装 -> 可放松、允许更多放行。 +# 固有机制(对焦/多帧筛选/倒置/光流晃动)完全一致, 只有下面这些判定参数不同。 +# ============================================================ +def make_scene_config(scene: str = "medicine") -> "FunnelConfig": + """按场景返回标定好的 FunnelConfig。机制代码不变, 只改参数。""" + cfg = FunnelConfig() + if scene in ("medicine", "药盒", "drug"): + return cfg # 药盒 = 现有默认(小字/高危/严) + if scene in ("stationery", "文具", "supplies", "生活用品"): + # 生活用品/文具: 安全等级低(念错关系不大) -> 放行阈值放松, 少拒绝、少打扰。 + # 参数依据: 网络摄像头 clip_windows csv (routing 分析) + 安全等级。 + cfg.worst_x0 = 5.0 # 松(药盒6.5): 拦截判据, 药盒严文具松(安全等级低)。 + # 手动定: csv里 worst_block 对ok/blur区分很弱(图都较清晰, + # AUC~0.52), 算不出可靠值, 按"文具比药盒松"手动定松一档。 + cfg.ACCEPT_Q = 0.48 # 略放松(药盒0.55): 安全低, 容错高一点。 + cfg.SEVERE_FLOW = 8.0 # 沿用药盒: csv 里 seg_flow 是最强判据(AUC~0.71), >8 后 + # ok率骤降, 与药盒一致。晃动是通用物理约束, 不随场景放松。 + return cfg + # 未知场景 -> 退回药盒默认, 并提示 + print(f"[scene] 未知场景 '{scene}', 用药盒默认配置") + return cfg + + +class FunnelResult: + def __init__(self): + self.accepted = False + self.need_focus = False # 稳但糊: 上层应触发AF再采一批 + self.reject_reason = "" + self.best_index = -1 + self.best_jpg = None + self.per_frame = [] + self.motions = [] # 这里存 rel_change 序列 + self.components = {} # 段级各 gi 分量 (归因用) + self.timings = {} + + def to_dict(self): + def _fmt(v): + if isinstance(v, (int, float)): + return round(v, 1) + return v # list 之类原样 (如 grab_per_frame_ms) + return { + "accepted": self.accepted, + "need_focus": self.need_focus, + "reject_reason": self.reject_reason, + "best_index": self.best_index, + "per_frame": self.per_frame, + "rel_changes": [round(m, 3) for m in self.motions], + "components": self.components, + "timings": {k: _fmt(v) for k, v in self.timings.items()}, + } + + +def run_funnel(frames: list[bytes], cfg: FunnelConfig = FunnelConfig()) -> FunnelResult: + """评分函数版漏斗: + 1) 每帧算 metrics(sharp/亮度/edge) + 2) 帧间相对变化 -> 每帧 quality + focus_gain (连续分, 非硬阈值) + 3) 决策: max(quality)>=ACCEPT_Q -> 选该帧(选优); + 否则 max(focus_gain)>=FOCUS_G -> 建议对焦(need_focus); + 都低 -> 拒绝。 + 返回里带 need_focus 标志, 供上层决定"稳但糊->触发AF->再采"。""" + r = FunnelResult() + if not frames: + r.reject_reason = "no_frames" + return r + + # 1) 单帧 metrics + t0 = time.monotonic() + metrics_list = [frame_metrics(j) for j in frames] + r.timings["level1_metrics_ms"] = (time.monotonic() - t0) * 1000 + + # 2) 帧间"运动"(稳定性输入) + 每帧评分 + # 稳定性用真正的帧间像素差分(frame_motion)衡量 —— 位移/转动会改变画面内容, + # 但不一定改变清晰度, 故不能用 sharp 变化代替。归一化到 [0,1] 的 rel。 + t1 = time.monotonic() + per = [] + prev_jpg = None + flow_mags = [] # 相邻帧光流位移幅度(严重晃动一刀切用) + for i, m in enumerate(metrics_list): + if prev_jpg is None: + motion = 0.0 + else: + motion = frame_motion(prev_jpg, frames[i]) # 平均绝对像素差 0-255 + flow_mags.append(optical_flow_motion(prev_jpg, frames[i])) # 光流位移(像素) + prev_jpg = frames[i] + rel = min(1.0, motion / cfg.motion_ref) # 归一: motion>=motion_ref 视为完全在动 + sc = frame_score(m, rel, cfg) + per.append({**m, "rel_change": round(rel, 3), "motion": round(motion, 2), **sc}) + r.motions.append(rel) + r.per_frame = per + # 段级光流: 用中位数抗单帧噪声。大 = 整段相机在大幅移动 = 严重晃动。 + seg_flow = float(sorted(flow_mags)[len(flow_mags) // 2]) if flow_mags else 0.0 + r.timings["level2_score_ms"] = (time.monotonic() - t1) * 1000 + + # 3) 决策: 稳定性优先的三出口 (A/B/C), argmin(gi) 归因, D/E 留接口 + t2 = time.monotonic() + qualities = [p["quality"] for p in per] + # 段级聚合各分量 (用中位数抗单帧噪声) + def med(key): + vals = sorted(p[key] for p in per) + n = len(vals) + return vals[n // 2] if n % 2 else (vals[n // 2 - 1] + vals[n // 2]) / 2 + seg_stable = med("g_stable") + seg_sharp = med("g_sharp") + seg_content = med("g_content") + seg_light = med("g_light") + # best 帧选择: 只按【清晰度】选最能读的一帧, 不用综合 quality。 + # 原因: quality 含 stable 分量, 会让"晃动某瞬间恰好帧间motion低(转折点)但画面糊"的帧 + # quality 虚高, 压过"稍动但清晰可读"的帧 -> best 选成糊的(真机验证: f0糊被选/f2清晰没选)。 + # 念字能不能读只取决于清晰度, 与该帧瞬时稳不稳无关; 稳定性只用于整段是否拒绝(出口A)。 + # 用 worst_block(最糊有内容块)选: 选"最糊处也最清晰"的那帧 = 最可读。 + def _clarity(i): + return per[i].get("worst_block", per[i].get("sharpness", 0.0)) + best_q = max(range(len(per)), key=_clarity) + # quality 仍保留供出口B的接受阈值判断(下面 qualities[best_q]>=ACCEPT_Q) + r.timings["level3_decide_ms"] = (time.monotonic() - t2) * 1000 + + # 归因分量表 (供 argmin 路由 + 记录) + r.components = {"stable": round(seg_stable, 3), "sharp": round(seg_sharp, 3), + "content": round(seg_content, 3), "light": round(seg_light, 3), + "flow": round(seg_flow, 2)} # 段级光流位移(像素) + + # ---- 出口 A0: 严重晃动一刀切 —— 光流位移过大 = 相机大幅移动, 不看别的直接拒 ---- + # 可解释: 画面整段平均移动 >SEVERE_FLOW 像素 = 明显在晃, 念字必糊, 让用户停稳。 + # 光流比逐像素差鲁棒(不被密集字伪运动骗); 阈值待 shake 样本校准。 + if seg_flow >= cfg.SEVERE_FLOW: + r.reject_reason = "severe_shake" # 出口 A0: "晃得厉害, 请拿稳" + r.need_focus = False + r.best_index = best_q + return r + + # ---- 出口 A: 稳定性优先 —— 持续在动, 不看清晰度, 直接请拿稳 ---- + # (动的时候对焦也没用, 且用户没老实, 不该浪费时间去挑图) + if seg_stable < cfg.STABLE_MIN: + r.reject_reason = "unstable" # 出口 A: "请拿稳/对准" + r.need_focus = False + r.best_index = best_q # 仅供参考, 不喂模型 + return r + + # ---- 稳定。看有没有帧够清晰 ---- + if qualities[best_q] >= cfg.ACCEPT_Q: + # ---- 出口 B: 稳 + 有清晰帧 -> 给最清晰的 ---- + r.accepted = True + r.best_index = best_q + r.best_jpg = frames[best_q] + r.need_focus = False + return r + + # ---- 稳但没有够清晰的帧: 归因 argmin, 决定是 C(对焦) 还是 D/E ---- + # 在"稳定"前提下, 看哪个分量最拖累: sharp低=失焦(C); light低=暗(D); + # content低但sharp高=清晰却没内容=没对准(E)。argmin 连续路由, 非硬堆阈值。 + cand = {"sharp": seg_sharp, "light": seg_light, "content": seg_content} + worst = min(cand, key=cand.get) + + if worst == "light" and seg_light < cfg.LIGHT_MIN: + # 出口 D (留接口, 未激活干预): 太暗 -> 未来 CLAHE 数字增强 / 提示到亮处 + r.reject_reason = "too_dark" # TODO: 接 CLAHE 后改为可救 + r.need_focus = False + r.best_index = best_q + return r + + # 出口 E（aimed_wrong）已弃用：原设计（-v/OCR）用"清晰但无内容结构"判"没对准"， + # 目的是排除背景干扰。但 -o 不受背景影响、完全没字时会老实说"没看到字"， + # 不需要漏斗拦。故这种帧直接放行，交给模型自己判断，避免误判打断。 + # （真正危险的"像字又不是字"是另一个问题，待 720p 数据用别的信号重做。） + if worst == "content" and seg_sharp >= cfg.SHARP_OK_FOR_E and seg_content < cfg.CONTENT_MIN: + r.accepted = True + r.best_index = best_q + r.best_jpg = frames[best_q] + r.need_focus = False + return r + + # ---- 出口 C: 稳但糊 (sharp 主导拖累) -> 触发AF重采 ---- + r.accepted = False + r.need_focus = True + r.reject_reason = "need_focus" # 上层触发AF+重采, 仍不行才最终拒绝 + r.best_index = best_q + return r + + +# ============================================================ +# 方向处理 & 推理 —— 留桩, 下一步填 +# ============================================================ +def _center_crop(gray, frac=0.6): + """取画面中心区, 排掉四周背景(手/桌面/墙)。用户会把目标大致对准中心, + 故中心区更可能是文字主体; 在其上算判据可显著减轻背景污染。""" + h, w = gray.shape[:2] + y0, y1 = int(h * (1 - frac) / 2), int(h * (1 + frac) / 2) + x0, x1 = int(w * (1 - frac) / 2), int(w * (1 + frac) / 2) + return gray[y0:y1, x0:x1] + + +def _binv_fg(gray): + _, b = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + return (b > 0).astype(np.float64) + + +def _detect_sideways(gray): + """90°/270°侧向: 水平投影方差 vs 垂直投影方差。多档中心区(0.5/0.6/0.7)投票, + 比单一比例更稳(文字不在正中心时不易被单档背景污染带偏)。 + 返回 (verdict, ratio): 'upright'/'sideways'/'ambiguous'。ratio 取中位数。""" + ratios = [] + for frac in (0.5, 0.6, 0.7): + b = _binv_fg(_center_crop(gray, frac)) + hvar = float(np.var(b.sum(axis=1))) + vvar = float(np.var(b.sum(axis=0))) + ratios.append(hvar / (vvar + 1e-6)) + ratios.sort() + med = ratios[1] # 中位数, 抗单档异常 + votes = ["sideways" if r < 0.67 else ("upright" if r > 1.5 else "ambiguous") + for r in ratios] + # 多数票: 3档里≥2档一致才定, 否则 ambiguous + for v in ("sideways", "upright"): + if votes.count(v) >= 2: + return v, round(med, 2) + return "ambiguous", round(med, 2) + + +def _detect_upside_down(gray): + """180°正倒: '注水容量'判据(资料方法, 真图验证对中文有效)。 + 对每列: 最高文字点上方空白(top_cap) vs 最低文字点下方空白(bot_cap)。 + 正立版面文字整体下方留白多 -> (bot-top)>0; 倒置翻转 -> <0。 + **看版面级上下留白, 不依赖单字对称性, 绕开中文字对称死结。** + 真图验证: 正立+0.66/倒置-0.48, 抗±8°倾斜、抗裁剪比例(0.5~0.9)。必须裁剪(整图会被背景淹)。 + 返回 (verdict, score): 'upright'/'flipped'/'ambiguous'。""" + b = _binv_fg(_center_crop(gray, 0.7)) + h, w = b.shape + tc = bc = cnt = 0 + for x in range(0, w, 3): + col = np.where(b[:, x] > 0)[0] + if len(col) < 2: + continue + tc += col[0] + bc += (h - 1 - col[-1]) + cnt += 1 + if cnt < 10: + return "ambiguous", 0.0 + score = (bc - tc) / (bc + tc + 1e-9) + if score > 0.15: + return "upright", round(score, 3) + if score < -0.15: + return "flipped", round(score, 3) + return "ambiguous", round(score, 3) + + +def _detect_truncation(gray, margin=10): + # 先确认中心区有文字主体; 否则不判截断(避免背景边缘误报) + cb = _binv_fg(_center_crop(gray, 0.6)) + if cb.mean() < 0.01: + return {}, {} # 中心没内容, 不谈截断 + b = _binv_fg(gray) + edges = {"top": b[:margin, :].mean(), "bottom": b[-margin:, :].mean(), + "left": b[:, :margin].mean(), "right": b[:, -margin:].mean()} + overall = b.mean() + if overall < 1e-4: + return {}, {} + # 贴边判据收紧: 边缘密度需 >= 整图的一定比例, 且 >= 中心密度的一定比例 + # (排掉"背景纹理贴边但中心才是文字"的误报) + center_d = cb.mean() + touched = {k: (v >= overall * 0.6 and v >= center_d * 0.4 and v > 0.02) + for k, v in edges.items()} + return {k: round(v, 4) for k, v in edges.items()}, {k: t for k, t in touched.items() if t} + + +_PAN_HINT = {"top": "请向上看一点", "bottom": "请向下看一点", + "left": "请向左看一点", "right": "请向右看一点"} + + +_ORI_CLS = None # 方向分类器单例(全局只加载一次) +_ORI_CLS_TRIED = False # 是否已尝试加载(避免反复重试失败) + +def _get_orient_classifier(): + """懒加载 PaddleOCR 文档方向分类器(PP-LCNet, 6.75MB, 0/90/180/270)。 + 单例: 只在首次调用时加载一次(几秒), 之后每次推理仅几ms。 + 降级: 未安装/加载失败 -> 返回 None, process_orientation 回退到轻量CV判据, 不崩。""" + global _ORI_CLS, _ORI_CLS_TRIED + if _ORI_CLS is not None: + return _ORI_CLS + if _ORI_CLS_TRIED: + return None # 之前试过且失败, 不再重试 + _ORI_CLS_TRIED = True + # 加载方向分类器。优先用最简单的写法(实测 SmartGlasses 环境可用, 与 torch 不冲突 —— + # 当初冲突的是完整 PaddleOCR-GPU 识别, 不是这个轻量方向分类器)。 + # 若想显式指定设备, 后面的参数变体作为备选依次尝试。 + from paddleocr import DocImgOrientationClassification + last_err = None + for kw in (dict(model_name="PP-LCNet_x1_0_doc_ori"), # 最简单(原来能用的) + dict(model_name="PP-LCNet_x1_0_doc_ori", device="cpu"), # 显式CPU(可选) + dict()): # 全默认兜底 + try: + _ORI_CLS = DocImgOrientationClassification(**kw) + print(f"[orient] PaddleOCR 方向分类器已加载 (PP-LCNet, 参数={list(kw) or 'default'})") + return _ORI_CLS + except Exception as e: + last_err = e + continue # 这个参数组合不行, 换下一个 + print(f"[orient] 方向分类器加载失败, 回退到轻量CV判据: {last_err}") + _ORI_CLS = None + return None + + +# 文档方向标签(图像被顺时针旋转的角度) -> (状态, 给用户的转向提示) +# 注意: "图像旋转角"与"用户转药盒方向"相反。文案先按标准定义, 实机核对后固定。 +# 90°: 画面顺时针转了90° = 药盒被逆时针放了 -> 用户应顺时针转回? 需实机核对, 故保留 raw 标签调试。 +_ORIENT_MAP = { + "0": ("upright", None), + "90": ("sideways", "请把药盒顺时针转90°"), + "270": ("sideways", "请把药盒逆时针转90°"), + "180": ("flipped", "药盒拿反了，请上下翻转"), +} + + +def _classify_orientation(img_bgr): + """用分类器判方向。返回 (raw_label, conf) 或 (None, 0) 若不可用/低置信。""" + cls = _get_orient_classifier() + if cls is None: + return None, 0.0 + try: + res = cls.predict(img_bgr) # 传 ndarray, 免落盘 + r = res[0] + labels = r.get("label_names") or [str(x) for x in r.get("class_ids", [])] + scores = r.get("scores", [0.0]) + raw = str(labels[0]).replace("_degree", "").strip() + conf = float(max(scores)) if scores else 0.0 + return raw, conf + except Exception as e: + print(f"[orient] 推理失败: {e}") + return None, 0.0 + + +def process_orientation(best_jpg: bytes): + """第2级(方向) + 第3级(取景完整) —— 只对清晰的 best 帧做一次。 + 方向: 优先用 PaddleOCR 轻量方向分类器(PP-LCNet, 真图验证0/90/180/270全准, 置信~0.92); + 未装/低置信 -> 回退轻量CV判据。180°走多帧一致性('请确认正反'在handler触发)。 + 返回 (jpg, diag)。diag 含方向反馈 + 截断反馈 + incomplete 标志(传下游prompt)。 + [留桩] 圆柱曲面矫正 = 独立课题, 不在此处。""" + arr = np.frombuffer(best_jpg, np.uint8) + img = cv2.imdecode(arr, cv2.IMREAD_COLOR) + if img is None: + return best_jpg, {"ok": False} + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + + orient_hint = None + orient_state = "upright" + ud_score = 0.0 + side_ratio = "" + orient_raw = "" + orient_conf = 0.0 + CONF_MIN = 0.60 # 置信门控: 低于此不硬判, 走uncertain/兜底 + + raw, conf = _classify_orientation(img) + orient_raw, orient_conf = (raw or ""), conf + if raw is not None and conf >= CONF_MIN and raw in _ORIENT_MAP: + # 分类器可用且置信足够 -> 直接按映射给状态+提示 + # (实测180 conf~0.93很确定, 不会与0混淆, 故即时提示; 若需滤噪声在簇内多帧投票层做) + orient_state, orient_hint = _ORIENT_MAP[raw] + elif raw is not None: + # 分类器给了结果但低置信 -> 不硬判 + orient_state = "uncertain" + else: + # 分类器不可用 -> 回退轻量CV判据(整块投影/注水容量) + side, side_ratio = _detect_sideways(gray) + if side == "sideways": + orient_state, orient_hint = "sideways", "请把药盒转90°" + else: + ud, ud_score = _detect_upside_down(gray) + orient_state = {"flipped": "flipped", "upright": "upright"}.get(ud, "uncertain") + + # 取景完整性(截断) —— 显示层暂关, 接口保留 + edges, truncated = _detect_truncation(gray) + pan_hint = None + if truncated: + worst = max(truncated, key=lambda k: edges.get(k, 0)) + pan_hint = _PAN_HINT.get(worst) + + diag = { + "ok": True, + "orient_state": orient_state, # upright/sideways/flipped/uncertain + "orient_hint": orient_hint, # 侧向即时提示; 180的"请确认"由多帧一致性在handler加 + "orient_raw": orient_raw, # 分类器原始标签(0/90/180/270), 实机核对转向用 + "orient_conf": round(orient_conf, 3), + "sideways_ratio": side_ratio, + "flipped_frame": orient_state == "flipped", # 供多帧一致性累积 + "ud_score": ud_score, + "truncated_edges": list(truncated.keys()), + "pan_hint": pan_hint, + "incomplete": bool(truncated), # 传下游prompt: 只念可见部分 + } + return best_jpg, diag # 第一版不自动转正, 只诊断+提示(方向由用户调) + + +def infer(frame_jpg: bytes) -> Optional[str]: + """[留桩] 未来: 把 best 帧喂给 -v (或 Qwen-VL) 做念字/识别。 + 现在返回 None。接推理时把这里换成真的模型调用。""" + return None + + +# ============================================================ +# CV 指标: 拉普拉斯方差(清晰度) + 平均亮度 +# 这两个就是三级漏斗一级的判据; 在这里实时显示以标定阈值。 +# ============================================================ +def frame_metrics(jpg: bytes) -> dict: + """返回一帧的 CV 指标字典。记录器会动态把所有 key 当作 CSV 列 —— + 以后 CV 那步加新指标(如帧间差分测晃动、方向检测分), 只需在这里往 + dict 里加 key, CSV 自动多列, 记录代码不用改。""" + arr = np.frombuffer(jpg, dtype=np.uint8) + img = cv2.imdecode(arr, cv2.IMREAD_GRAYSCALE) + if img is None: + return {"sharpness": 0.0, "brightness": 0.0, "kb": round(len(jpg) / 1024, 1)} + lap_var = cv2.Laplacian(img, cv2.CV_64F).var() + edges = cv2.Canny(img, 80, 160) + edge_density = float((edges > 0).mean()) + + # --- 附加指标: 先记录不判定, 采数据后用 ok/blur 标注看哪个最能对齐可读性 --- + # local_sharp: N×N 分块取最清晰块。抓"局部有清晰区"; 但会被高频背景(木纹)骗高。 + ls = _local_sharp(img, grid=4) + # text_sharp / text_ratio: MSER 类文字候选区的清晰度 + 面积占比。 + # text_ratio 说"有没有像字的区域"(高频背景 ratio≈0, 骗不过它); + # 但糊到一定程度 MSER 找不到字 -> ratio→0 (自我矛盾, 已离线证实)。 + # 与 local_sharp 组合: local高+ratio低=清晰但非字; 两者皆低=糊。 + ts, tr = _text_sharp(img) + # worst_block / block_var: 匹配"一处糊即不可用(OCR 标准)"的悲观判据。 + # worst_block = 有内容的块里最糊的清晰度 (看最差, 非最好); + # block_std = 块间清晰度标准差 (大 = 部分清晰部分糊 = 局部糊)。 + # 对 OCR: 决定成败的是最糊的文字区, 不是最清晰的 —— 故看 worst 而非 best。 + wb, bstd = _block_sharp_stats(img, grid=4) + # contrast / dark_ratio: 区分"糊"与"暗/低对比"。暗+低对比时全局sharp假性偏低, + # 但 contrast 低会揭示真因 -> 对应"数字增强(CLAHE)"而非"对焦"这条干预路径。 + contrast = float(img.std()) + dark_ratio = float((img < 50).mean()) + + # [CLAHE 接口预留] 未来暗环境念字: 若 contrast 低 / dark_ratio 高, 可在此对 img + # 做 CLAHE/gamma 增强后重算指标, 作为"数字层干预"(区别于对焦的"物理层干预")。 + # 现在不实现 —— 待有暗环境测试条件再填。示意: + # if contrast < TH: img_enh = cv2.createCLAHE(...).apply(img); 重算 lap_var ... + + return { + "sharpness": round(float(lap_var), 1), # 全局拉普拉斯方差 + "brightness": round(float(img.mean()), 1), + "edge_density": round(edge_density, 4), + "local_sharp": round(float(ls), 1), # 最清晰块 (乐观, 记录) + "worst_block": round(float(wb), 1), # 最糊的有内容块 (悲观, 匹配OCR标准) + "block_std": round(float(bstd), 1), # 块间清晰度std (大=局部糊) + "text_sharp": round(float(ts), 1), # 文字区清晰度 (记录) + "text_ratio": round(float(tr), 4), # 文字区面积占比 (记录) + "contrast": round(contrast, 1), # 对比度(std) (记录) + "dark_ratio": round(dark_ratio, 4), # 暗像素占比 (记录) + "kb": round(len(jpg) / 1024, 1), + } + + +def _block_sharp_stats(gray, grid: int = 4): + """分块清晰度的 (最糊有内容块, 块间std)。 + 只统计"有内容"的块(edge 非极低), 避免纯背景块干扰; + worst = 有内容块里最低清晰度 (对应'一处糊即失败'); + std = 块间清晰度离散度 (局部糊 -> 大)。""" + h, w = gray.shape[:2] + vals = [] + for i in range(grid): + for j in range(grid): + blk = gray[i * h // grid:(i + 1) * h // grid, j * w // grid:(j + 1) * w // grid] + if blk.size == 0: + continue + # 只算"有文字/边缘结构"的块。关键: 用【边缘密度】而非 std 判有内容 —— + # std 只反映明暗起伏, 白纸的折痕/阴影/渐变 std 也 >8, 会被误当"内容块", + # 而白纸拉普拉斯极低(~1) -> 把 worst_block 拉到假性极低 -> 清晰标签被误判糊。 + # 文字块有大量边缘, 白纸光影块几乎无边缘 -> 用 Canny 边缘占比区分。 + edges = cv2.Canny(blk, 50, 150) + edge_frac = float((edges > 0).mean()) + if edge_frac < 0.006: # 边缘太少 = 无文字结构(白纸/背景) -> 跳过 + continue + vals.append(float(cv2.Laplacian(blk, cv2.CV_64F).var())) + if not vals: + return 0.0, 0.0 + worst = min(vals) + std = float(np.std(vals)) if len(vals) > 1 else 0.0 + return worst, std + + +def _local_sharp(gray, grid: int = 4) -> float: + """N×N 分块, 取最清晰块的拉普拉斯方差。""" + h, w = gray.shape[:2] + best = 0.0 + for i in range(grid): + for j in range(grid): + blk = gray[i * h // grid:(i + 1) * h // grid, j * w // grid:(j + 1) * w // grid] + if blk.size: + best = max(best, float(cv2.Laplacian(blk, cv2.CV_64F).var())) + return best + + +def _text_sharp(gray): + """MSER 找类文字候选区, 在候选区上算清晰度。返回 (清晰度, 面积占比)。 + 找不到 -> (0,0)。占比=0 提示'没找到字'(可能糊/无字), 与 local_sharp 组合判读。""" + try: + mser = cv2.MSER_create(delta=5, min_area=60, max_area=14400) + regions, _ = mser.detectRegions(gray) + except Exception: + return 0.0, 0.0 + if not regions: + return 0.0, 0.0 + mask = np.zeros(gray.shape[:2], np.uint8) + for pts in regions: + x, y, bw, bh = cv2.boundingRect(pts.reshape(-1, 1, 2)) + ar = bw / max(bh, 1) + if 0.1 < ar < 10 and bw < gray.shape[1] * 0.9: + cv2.fillPoly(mask, [cv2.convexHull(pts.reshape(-1, 1, 2))], 255) + area_ratio = float((mask > 0).mean()) + if area_ratio < 0.001: + return 0.0, area_ratio + lap = cv2.Laplacian(gray, cv2.CV_64F) + return float(lap[mask > 0].var()), area_ratio + + +# ============================================================ +# 后台抓帧线程: 独占 TCP, 持续抓最新帧 (供 /frame 取用) +# 前端按自己的刷新率来取, 后端只维护"最新一帧+指标"。 +# ============================================================ +class Recorder: + """记录一段一段的指标序列。每段带一个 label(触发来源, 如 af / res:UXGA / manual), + 每帧一行, 动态列(跟随 frame_metrics 的 key)。存磁盘 + 供前端下载。 + + 两种触发: (a) 点控制按钮自动录 duration 秒; (b) 手动开始/停止。 + 记录挂在后端抓帧线程上(每抓一帧记一行), 与前端显示刷新率解耦, 密度=真实抓帧率。""" + + def __init__(self, out_dir: Path): + self.out_dir = Path(out_dir) + self.out_dir.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + self._rows: list[dict] = [] # 所有已记录的行 (跨段, 全量) + self._active = False + self._seg_label = "" + self._seg_start_mono = 0.0 + self._seg_deadline = 0.0 # >0 表示定时录; 0 表示手动录(无限直到 stop) + self._seg_index = 0 + self._columns: list[str] = [] # 动态列, 首次记录时按 metrics key 确定 + + def start_segment(self, label: str, duration_s: float = 0.0): + """duration_s>0: 定时录; =0: 手动录(直到 stop_segment)。""" + with self._lock: + self._active = True + self._seg_index += 1 + self._seg_label = label + self._seg_start_mono = time.monotonic() + self._seg_deadline = (self._seg_start_mono + duration_s) if duration_s > 0 else 0.0 + + def stop_segment(self): + with self._lock: + self._active = False + + def feed(self, metrics: dict, grab_ms: float, res: str, rotate: int): + """抓帧线程每帧调一次。仅在 active 且未超时时记录。""" + with self._lock: + if not self._active: + return + now = time.monotonic() + if self._seg_deadline > 0 and now >= self._seg_deadline: + self._active = False + return + if not self._columns: + # 首次: 固定前缀列 + metrics 的动态列 + self._columns = (["seg", "label", "t_ms", "grab_ms", "res", "rotate"] + + list(metrics.keys())) + row = { + "seg": self._seg_index, + "label": self._seg_label, + "t_ms": round((now - self._seg_start_mono) * 1000, 1), + "grab_ms": round(grab_ms, 1), + "res": res, + "rotate": rotate, + } + row.update(metrics) + self._rows.append(row) + + def status(self) -> dict: + with self._lock: + remain = 0.0 + if self._active and self._seg_deadline > 0: + remain = max(0.0, self._seg_deadline - time.monotonic()) + return { + "active": self._active, + "label": self._seg_label, + "remain_s": round(remain, 1), + "n_rows": len(self._rows), + "n_segments": self._seg_index, + } + + def to_csv(self) -> str: + import csv, io + with self._lock: + rows = list(self._rows) + cols = list(self._columns) if self._columns else ["seg"] + # 动态列可能因未来指标增删而不齐: 用所有行 key 的并集补齐 + allcols = list(cols) + for r in rows: + for k in r: + if k not in allcols: + allcols.append(k) + buf = io.StringIO() + w = csv.DictWriter(buf, fieldnames=allcols, extrasaction="ignore") + w.writeheader() + for r in rows: + w.writerow(r) + return buf.getvalue() + + def save_disk(self) -> Optional[Path]: + csv_text = self.to_csv() + stamp = time.strftime("%Y%m%d_%H%M%S") + path = self.out_dir / f"tuning_{stamp}.csv" + try: + path.write_text(csv_text, encoding="utf-8-sig") # BOM 便于 Excel 直接开 + return path + except Exception as e: + print(f"[REC] 存盘失败: {e}") + return None + + def clear(self): + with self._lock: + self._rows.clear() + self._seg_index = 0 + self._columns = [] + self._active = False + + +class FrameGrabber: + def __init__(self, tcp: TCPImageClient, recorder: "Recorder", ctl_state: dict): + self.tcp = tcp + self.recorder = recorder + self.ctl_state = ctl_state # {"res":..., "rotate":...} 供记录标注当前状态 + self._latest_jpg: Optional[bytes] = None + self._latest_metrics: dict = {} + self._latest_grab_ms: float = 0.0 + self._rotate_deg = 0 + self._lock = threading.Lock() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def set_rotation(self, deg: int): + with self._lock: + self._rotate_deg = deg % 360 + + def start(self): + self._thread.start() + + def _run(self): + while not self._stop.is_set(): + t0 = time.monotonic() + jpg = self.tcp.capture() + dt = (time.monotonic() - t0) * 1000 + if jpg: + with self._lock: + deg = self._rotate_deg + if deg: + jpg = rotate_jpeg(jpg, deg) # 真转, 指标基于转后图 + mtr = frame_metrics(jpg) + with self._lock: + self._latest_jpg = jpg + self._latest_metrics = mtr + self._latest_grab_ms = dt + # 喂记录器 (仅在 active 时真记); 带当前分辨率/旋转状态 + self.recorder.feed(mtr, dt, + res=self.ctl_state.get("res", "?"), + rotate=deg) + else: + time.sleep(0.05) + + def latest(self): + with self._lock: + return self._latest_jpg, dict(self._latest_metrics), self._latest_grab_ms + + def grab_batch(self, n: int, rotate_apply: bool = True): + """连抓 n 帧 (复用 TCP 连接, 串行)。返回 (frames_list, grab_ms_list)。 + 对应真实链路 ASR 后'抓 N 张'那一步。""" + frames, lats = [], [] + with self._lock: + deg = self._rotate_deg + for _ in range(n): + t0 = time.monotonic() + jpg = self.tcp.capture() + dt = (time.monotonic() - t0) * 1000 + if jpg: + if rotate_apply and deg: + jpg = rotate_jpeg(jpg, deg) + frames.append(jpg) + lats.append(dt) + return frames, lats + + def stop(self): + self._stop.set() + self._thread.join(timeout=2.0) + self.tcp.close() + + +# ============================================================ +# Web 服务 +# ============================================================ +PAGE = r""" + + + + +相机调参台 + + + +
+
+ — +
+
Laplacian variance · 清晰度
+
+
+
+
+ +
等待画面…
+
—
+
+
+
漏斗选出的 BEST 帧 (喂给模型的那张)
+ +
+
+ + + + + + +""" + + +class PipelineStats: + """漏斗运行统计: 接受/拒绝计数(按原因)+ best帧存盘 + 最近 best 缓存。 + 拒绝率是安全指标的雏形: 该拒的拒了多少、各什么原因。""" + + def __init__(self, out_dir: Path): + self.out_dir = Path(out_dir) + self.out_dir.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + self.n_total = 0 + self.n_accepted = 0 + self.reasons: dict = {} # reason -> count + self.last_best: Optional[bytes] = None + self._batch_i = 0 + self._runs: list[dict] = [] # 每次抓拍一行汇总 (供标定用) + + def add_run_row(self, scene: str, res, focused: bool, best_path: str = "", geom: dict = None): + """每次抓拍追加一行汇总, 带用户标注的场景标签 + best帧路径。 + best_path: 供你事后人工看这张 best 清不清楚, 在 manual_label 列标真值 —— + 用人工判断(而非抓拍前意图标签)当 ground truth 校准阈值, 绕开标签错位。""" + per = res.per_frame or [] + max_q = max((p.get("quality", 0) for p in per), default=0) + max_fg = max((p.get("focus_gain", 0) for p in per), default=0) + # best 帧的完整 CV 指标 (供标定: 一张汇总表就能对比各指标 vs ok/blur) + bm = {} + if 0 <= res.best_index < len(per): + bm = per[res.best_index] + best_sharp = bm.get("sharpness", 0) + t = res.timings or {} + import os as _os + row = { + "time": time.strftime("%H:%M:%S"), + "scene": scene, + "manual_label": "", # ← 你看完 best 帧后手填: ok / blur (真值) + "n_frames": len(per), + "accepted": int(res.accepted), + "need_focus": int(res.need_focus), + "focused": int(focused), + "reject_reason": res.reject_reason, + "best_index": res.best_index, + "best_sharp": round(best_sharp, 1), + "best_local": bm.get("local_sharp", ""), + "best_worst_block": bm.get("worst_block", ""), + "best_block_std": bm.get("block_std", ""), + "best_text_sharp": bm.get("text_sharp", ""), + "best_text_ratio": bm.get("text_ratio", ""), + "best_contrast": bm.get("contrast", ""), + "best_dark_ratio": bm.get("dark_ratio", ""), + "seg_stable": (res.components or {}).get("stable", ""), + "seg_sharp": (res.components or {}).get("sharp", ""), + "seg_content": (res.components or {}).get("content", ""), + "seg_light": (res.components or {}).get("light", ""), + "max_quality": round(max_q, 3), + "max_focus_gain": round(max_fg, 3), + "grab_batch_ms": round(t.get("grab_batch_ms", 0), 1), + "funnel_ms": round(t.get("level1_metrics_ms", 0) + + t.get("level2_score_ms", 0) + + t.get("level3_decide_ms", 0), 1), + "total_ms": round(t.get("total_ms", 0), 1), + "best_file": _os.path.basename(best_path) if best_path else "", + "best_path": best_path, + # 方向分类器原始输出(定位方向不准: raw标签 vs 你实际摆放) + "orient_raw": (geom or {}).get("orient_raw", ""), + "orient_conf": (geom or {}).get("orient_conf", ""), + "orient_state": (geom or {}).get("orient_state", ""), + "orient_hint": (geom or {}).get("orient_hint", "") or "", + } + with self._lock: + self._runs.append(row) + + def runs_csv(self) -> str: + import csv, io + with self._lock: + runs = list(self._runs) + cols = ["time", "scene", "manual_label", "n_frames", "accepted", "need_focus", + "focused", "reject_reason", "best_index", "best_sharp", + "best_local", "best_worst_block", "best_block_std", + "best_text_sharp", "best_text_ratio", "best_contrast", "best_dark_ratio", + "seg_stable", "seg_sharp", "seg_content", "seg_light", + "max_quality", "max_focus_gain", "grab_batch_ms", "funnel_ms", "total_ms", + "orient_raw", "orient_conf", "orient_state", "orient_hint", + "best_file", "best_path"] + # 未来新字段自动并入 + for r in runs: + for k in r: + if k not in cols: + cols.append(k) + buf = io.StringIO() + w = csv.DictWriter(buf, fieldnames=cols, extrasaction="ignore") + w.writeheader() + for r in runs: + w.writerow(r) + return buf.getvalue() + + def save_runs_csv(self) -> Optional[Path]: + stamp = time.strftime("%Y%m%d_%H%M%S") + path = self.out_dir / f"pipeline_runs_{stamp}.csv" + try: + path.write_text(self.runs_csv(), encoding="utf-8-sig") + return path + except Exception as e: + print(f"[PSTATS] runs csv 存盘失败: {e}") + return None + + def record(self, accepted: bool, reason: str): + with self._lock: + self.n_total += 1 + if accepted: + self.n_accepted += 1 + else: + self.reasons[reason] = self.reasons.get(reason, 0) + 1 + + def set_last_best(self, jpg: bytes): + with self._lock: + self.last_best = jpg + + def save_batch(self, frames: list[bytes], best_index: int, res): + """存整批 N 帧 + 标出 best, 供人工核对。返回 (目录, best帧路径)。""" + with self._lock: + self._batch_i += 1 + bi = self._batch_i + d = self.out_dir / f"batch_{time.strftime('%H%M%S')}_{bi:03d}" + best_path = "" + try: + d.mkdir(parents=True, exist_ok=True) + for i, jpg in enumerate(frames): + tag = "_BEST" if i == best_index else "" + sharp = res.per_frame[i]["sharpness"] if i < len(res.per_frame) else 0 + fp = d / f"f{i}_sharp{sharp:.0f}{tag}.jpg" + fp.write_bytes(jpg) + if i == best_index: + best_path = str(fp) + import json + (d / "funnel.json").write_text( + json.dumps(res.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8") + return str(d), best_path + except Exception as e: + print(f"[PSTATS] save_batch 失败: {e}") + return "", "" + + def summary(self) -> dict: + with self._lock: + rej = self.n_total - self.n_accepted + return { + "total": self.n_total, + "accepted": self.n_accepted, + "rejected": rej, + "accept_rate": round(self.n_accepted / self.n_total, 3) if self.n_total else 0, + "reject_reasons": dict(self.reasons), + } + + +def make_app(grabber: FrameGrabber, ctl: CamControl, + recorder: Recorder, ctl_state: dict, default_dur: float, + funnel_cfg: "FunnelConfig", pstats: "PipelineStats") -> web.Application: + app = web.Application() + + async def h_index(request): + return web.Response(text=PAGE, content_type="text/html") + + async def h_frame(request): + jpg, m, grab_ms = grabber.latest() + if not jpg: + return web.Response(status=503, text="no frame") + headers = { + "X-Sharpness": str(m.get("sharpness", 0)), + "X-Brightness": str(m.get("brightness", 0)), + "X-Kb": str(m.get("kb", 0)), + "X-Grab-Ms": str(round(grab_ms, 0)), + "Cache-Control": "no-store", + } + return web.Response(body=jpg, content_type="image/jpeg", headers=headers) + + async def h_ctl(request): + op = request.query.get("op", "") + val = request.query.get("val", "") + dur = float(request.query.get("dur", default_dur) or default_dur) + loop = asyncio.get_event_loop() + label = None + if op == "res": + ok = await loop.run_in_executor(None, ctl.set_resolution, val) + ctl_state["res"] = val + label = f"res:{val}" + elif op == "af": + ok = await loop.run_in_executor(None, ctl.trigger_af) + label = "af" + elif op == "quality": + ok = await loop.run_in_executor(None, ctl.set_quality, int(val or 10)) + label = f"quality:{val}" + elif op == "rotate": + grabber.set_rotation(int(val or 0)) + ctl_state["rotate"] = int(val or 0) + ok = True + label = f"rotate:{val}" + else: + return web.json_response({"ok": False, "err": "unknown op"}, status=400) + # 控制操作自动开一段定时记录 (看该操作的效果曲线) + if label and dur > 0: + recorder.start_segment(label, dur) + return web.json_response({"ok": bool(ok), "recording": label, "dur": dur}) + + # ---- 记录控制 ---- + async def h_rec_start(request): + label = request.query.get("label", "manual") + dur = float(request.query.get("dur", 0) or 0) # 0=手动(直到 stop) + recorder.start_segment(label, dur) + return web.json_response({"ok": True, **recorder.status()}) + + async def h_rec_stop(request): + recorder.stop_segment() + return web.json_response({"ok": True, **recorder.status()}) + + async def h_rec_status(request): + return web.json_response(recorder.status()) + + async def h_rec_save(request): + path = recorder.save_disk() + return web.json_response({"ok": path is not None, + "path": str(path) if path else None, + **recorder.status()}) + + async def h_rec_download(request): + csv_text = recorder.to_csv() + recorder.save_disk() # 下载同时也存盘一份, 双保险 + return web.Response( + body=csv_text.encode("utf-8-sig"), + headers={"Content-Type": "text/csv; charset=utf-8", + "Content-Disposition": "attachment; filename=tuning.csv"}) + + async def h_rec_clear(request): + recorder.clear() + return web.json_response({"ok": True, **recorder.status()}) + + # ---- 形态2: 抓批 -> 评分漏斗 -> (稳但糊则对焦再采) -> best/拒绝 ---- + async def h_pipeline_run(request): + n = int(request.query.get("n", 5) or 5) + focus_n = int(request.query.get("focus_n", 3) or 3) # 对焦后补采几帧 + af_settle_ms = int(request.query.get("af_settle_ms", 120) or 120) # 对焦生效等待(ms) + # OV5640 单次对焦(0x3022=0x03)锁定通常在~100ms量级(目标近焦时更快), 非几百ms。 + # 默认120ms为初值, 实测用 ?af_settle_ms=N 扫描找最短有效等待。 + scene = request.query.get("scene", "") # 用户标注场景 + loop = asyncio.get_event_loop() + + def _work(): + t0 = time.monotonic() + frames, grab_lats = grabber.grab_batch(n) + grab_ms = (time.monotonic() - t0) * 1000 + if not frames: + return {"ok": False, "err": "no_frames"}, None, None + + res = run_funnel(frames, funnel_cfg) + focused = False + af_on = ctl_state.get("af_enabled", True) + # "自动对焦"开关 af_enabled 的语义(用户设计): + # ON(默认): 自主判断+按需触发对焦(路线A, 正常使用)。 + # OFF: 不自动触发对焦, 只保留 res.need_focus 的判断结果 —— + # 用于录制实验时验证"判据说该对焦"这个判断本身对不对, 不让对焦动作干扰。 + # (手动"触发单次AF"按钮走 /op?op=af, 不受此开关限制, 任何时候可手动触发。) + # 固件侧须 g_auto_af=false, 否则固件每秒硬对焦会架空此开关。 + if res.need_focus and af_on: + tf = time.monotonic() + ctl.trigger_af() # 写 /reg 0x3022=0x03 单次对焦 + # 等对焦生效再抓 —— trigger_af 后马达物理对焦需时间(OV5640单次对焦~100ms量级), + # 立刻 grab_batch 会抓到"对焦过程中"的糊帧。af_settle_ms 可调, 实测最短有效等待。 + if af_settle_ms > 0: + time.sleep(af_settle_ms / 1000.0) # _work 在 executor 线程, sleep 不阻塞事件循环 + extra, extra_lats = grabber.grab_batch(focus_n) + res.timings["focus_af_regrab_ms"] = (time.monotonic() - tf) * 1000 + if extra: + frames = frames + extra + grab_lats = grab_lats + extra_lats + res = run_funnel(frames, funnel_cfg) + focused = True + + res.timings["grab_batch_ms"] = grab_ms + res.timings["grab_per_frame_ms"] = [round(x, 1) for x in grab_lats] + res.timings["focused"] = 1 if focused else 0 + + infer_text = None + # 可读性多帧一致性: 本窗口 accepted 只是"候选送下游"; 复用方向那套跨窗口累积, + # 连续窗口多数都候选送, 才真正送下游 —— 滤掉"单窗口偶然可读"(晃动中偶抓清晰帧)。 + # 严重晃动(severe_shake)已在 run_funnel 单窗口即拒, 不进这里, 不受一致性影响。 + # 纯CV判定累积(不喂下游), 与方向一致性同机制。N=3 多数。 + rhist = ctl_state.setdefault("readable_hist", []) + rhist.append(1 if res.accepted else 0) + del rhist[:-3] # 只留最近3窗口 + readable_consistent = (len(rhist) >= 3 and sum(rhist) > 3 / 2) # 3窗口多数 + res_dict_extra = {"readable_hist": list(rhist), + "readable_consistent": readable_consistent} + if res.accepted and res.best_jpg is not None and readable_consistent: + to = time.monotonic() + oriented, geom_diag = process_orientation(res.best_jpg) + res.timings["orientation_ms"] = (time.monotonic() - to) * 1000 + # 多帧一致性(方向): 累积最近若干次抓拍的"疑似倒置"标志, 连续/多数才弹"请确认正反"。 + # 纯CV判据的多帧累积(不喂VLM/OCR), 滤单帧噪声, 避免正常图偶发误报。 + if geom_diag and geom_diag.get("ok"): + hist = ctl_state.setdefault("flip_hist", []) + hist.append(1 if geom_diag.get("flipped_frame") else 0) + del hist[:-5] # 只留最近5次 + # 5次里≥3次疑似倒置, 且本次也疑似 -> 才弹确认(多数一致) + if len(hist) >= 3 and sum(hist) >= 3 and geom_diag.get("flipped_frame"): + if not geom_diag.get("orient_hint"): + geom_diag["orient_hint"] = "请确认药盒正反" + geom_diag["flip_consistent"] = True + geom_diag["flip_hist"] = list(hist) + ti = time.monotonic() + infer_text = infer(oriented) + res.timings["infer_ms"] = (time.monotonic() - ti) * 1000 + pstats.record(accepted=True, reason="") + saved_dir, best_path = pstats.save_batch(frames, res.best_index, res) + res.timings["total_ms"] = (time.monotonic() - t0) * 1000 + rd = res.to_dict(); rd.update(res_dict_extra) + return ({"ok": True, "result": rd, "focused": focused, + "infer": infer_text, "geom": geom_diag, "saved_dir": saved_dir}, + res.best_jpg, (res, focused, best_path)) + elif res.accepted and not readable_consistent: + # 本窗口看着可读, 但连续一致性未达(可能偶然/刚开始) -> 暂不送下游, 等确认 + pstats.record(accepted=False, reason="await_consistency") + res.timings["total_ms"] = (time.monotonic() - t0) * 1000 + rd = res.to_dict(); rd.update(res_dict_extra) + rd["reject_reason"] = "await_consistency" + return ({"ok": True, "result": rd, "focused": focused, + "infer": None, "await_consistency": True}, + res.best_jpg, (res, focused, None)) + else: + # 若对焦后仍 need_focus, 归为最终拒绝(对焦也没救回来) + reason = res.reject_reason if res.reject_reason != "need_focus" else "all_blurry" + pstats.record(accepted=False, reason=reason) + res.reject_reason = reason + res.timings["total_ms"] = (time.monotonic() - t0) * 1000 + # 拒绝时也存证 + 显示本次批次里最清晰的一帧(而非留旧图), + # 让你能看到"系统这次面对的到底是什么画面", 复盘判定冤不冤。 + rej_best = res.best_index if 0 <= res.best_index < len(frames) else 0 + saved_dir, best_path = pstats.save_batch(frames, rej_best, res) + show_jpg = frames[rej_best] if frames else None + return ({"ok": True, "result": res.to_dict(), "focused": focused, + "infer": None, "geom": None, "rejected_shown": True, + "saved_dir": saved_dir}, show_jpg, (res, focused, best_path)) + + result = await loop.run_in_executor(None, _work) + if len(result) == 3: + payload, best_jpg, run_info = result + else: + payload, best_jpg, run_info = result[0], result[1], None + if best_jpg is not None: + pstats.set_last_best(best_jpg) + if run_info is not None: + res_obj, focused, best_path = run_info + pstats.add_run_row(scene, res_obj, focused, best_path, payload.get("geom")) + return web.json_response({**payload, "stats": pstats.summary()}) + + async def h_pipeline_best(request): + jpg = pstats.last_best + if not jpg: + return web.Response(status=404) + return web.Response(body=jpg, content_type="image/jpeg", + headers={"Cache-Control": "no-store"}) + + async def h_pipeline_stats(request): + return web.json_response(pstats.summary()) + + async def h_pipeline_download(request): + csv_text = pstats.runs_csv() + pstats.save_runs_csv() # 同时存盘 + return web.Response( + body=csv_text.encode("utf-8-sig"), + headers={"Content-Type": "text/csv; charset=utf-8", + "Content-Disposition": "attachment; filename=pipeline_runs.csv"}) + + # ---- 录制一段纯原始帧流(按秒, 不掺对焦), 供 rerun 看过程判定(问题1) ---- + async def h_record(request): + name = request.query.get("name", "").strip() or time.strftime("clip_%H%M%S") + dur = float(request.query.get("dur", 8) or 8) + dur = max(2.0, min(30.0, dur)) + safe = "".join(ch for ch in name if ch.isalnum() or ch in "_-") + clip_dir = pstats.out_dir.parent / "clips" / safe + loop = asyncio.get_event_loop() + try: + rec = await loop.run_in_executor( + None, record_stream, grabber, clip_dir, dur) + except Exception as e: + return web.json_response({"ok": False, "err": str(e)}, status=500) + return web.json_response({ + "ok": True, "dir": str(rec.dir), "name": safe, + "n_frames": len(rec.frames), "duration_s": dur, + }) + + app.router.add_get("/", h_index) + app.router.add_get("/frame", h_frame) + app.router.add_get("/ctl", h_ctl) + app.router.add_get("/rec/start", h_rec_start) + app.router.add_get("/rec/stop", h_rec_stop) + app.router.add_get("/rec/status", h_rec_status) + app.router.add_get("/rec/save", h_rec_save) + app.router.add_get("/rec/download", h_rec_download) + app.router.add_get("/rec/clear", h_rec_clear) + app.router.add_get("/pipeline/run", h_pipeline_run) + app.router.add_get("/pipeline/best", h_pipeline_best) + app.router.add_get("/pipeline/stats", h_pipeline_stats) + async def h_af_toggle(request): + val = request.query.get("on", "") + if val in ("1", "true", "on"): + ctl_state["af_enabled"] = True + elif val in ("0", "false", "off"): + ctl_state["af_enabled"] = False + return web.json_response({"ok": True, "af_enabled": ctl_state.get("af_enabled", True)}) + + app.router.add_get("/pipeline/download", h_pipeline_download) + app.router.add_get("/record", h_record) + app.router.add_get("/af_toggle", h_af_toggle) + return app + + +# ============================================================ +# 录制 + rerun (受控对比基础) +# 录制: 采一段(未对焦) -> 触发真AF -> 再采一段(对焦后), 两段都存, 标对焦点。 +# rerun: 从录制回放帧喂 run_funnel, 同一段反复扫参数, 输入固定=受控对比。 +# 解法一: 对焦效果也录进去(对焦后的帧真实存在), 故 C 出口的对焦可受控复现。 +# ============================================================ +class Recording: + """一段录制: 原始帧序列 + 每帧时间戳 + 对焦触发点标记。""" + + def __init__(self, clip_dir: Path): + self.dir = Path(clip_dir) + self.frames: list[bytes] = [] + self.timestamps: list[float] = [] + self.af_index = -1 # 对焦触发点: 此索引及之后的帧是"对焦后" + self.meta = {} + + def save(self): + self.dir.mkdir(parents=True, exist_ok=True) + for i, jpg in enumerate(self.frames): + (self.dir / f"frame_{i:04d}.jpg").write_bytes(jpg) + import json + meta = { + "n_frames": len(self.frames), + "timestamps": self.timestamps, + "af_index": self.af_index, # -1=无对焦点 + **self.meta, + } + (self.dir / "meta.json").write_text( + json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") + return self.dir + + @classmethod + def load(cls, clip_dir): + import json + d = Path(clip_dir) + rec = cls(d) + meta = json.loads((d / "meta.json").read_text(encoding="utf-8")) + rec.af_index = meta.get("af_index", -1) + rec.timestamps = meta.get("timestamps", []) + rec.meta = meta + n = meta.get("n_frames", 0) + for i in range(n): + fp = d / f"frame_{i:04d}.jpg" + if fp.exists(): + rec.frames.append(fp.read_bytes()) + return rec + + def pre_af_frames(self): + """对焦前的帧 (rerun 时先用这些跑漏斗)。""" + if self.af_index < 0: + return self.frames + return self.frames[:self.af_index] + + def post_af_frames(self): + """对焦后的帧 (rerun 时 C 出口触发对焦后回放这些)。""" + if self.af_index < 0: + return [] + return self.frames[self.af_index:] + + +def record_stream(grabber: "FrameGrabber", clip_dir: Path, + duration_s: float = 8.0, gap_s: float = 0.05) -> Recording: + """按秒录一段纯原始帧流 (不掺对焦)。专为问题1: rerun 回放看整个过程的 + 判定序列, 验证 CV 判的'该对焦'对不对。af_index=-1 表示无对焦点。""" + rec = Recording(clip_dir) + rec.af_index = -1 + t0 = time.monotonic() + while (time.monotonic() - t0) < duration_s: + jpg = grabber.tcp.capture() + if jpg: + rec.frames.append(jpg) + rec.timestamps.append(time.monotonic() - t0) + time.sleep(gap_s) + rec.meta["duration_s"] = duration_s + rec.meta["kind"] = "stream" # 区别于对焦对比的 clip + rec.save() + return rec + + +def rerun_stream(clip_dir, cfg: "FunnelConfig" = None, n: int = 5, stride: int = 3): + """对一段连续录制, 滑动窗口跑判定, 输出'整个过程的判定序列'。 + 每 stride 帧滑一次, 窗口大小 n。看物品进->停->出时判定(A/B/C)怎么变。 + 对焦命令天然关闭(回放死图), 即问题1: 验证'该对焦'的判断对不对。 + + 产出(供人工核对): 每窗口存 best 帧(文件名带判定) + 一个 CSV(每窗口一行, + 带判定/各分量/best帧路径)。你筛 reason=need_focus 的行, 点开原图看是否真需调焦。""" + import csv, json + cfg = cfg or FunnelConfig() + rec = Recording.load(clip_dir) + frames = rec.frames + out_dir = Path(clip_dir) / "rerun" + out_dir.mkdir(parents=True, exist_ok=True) + print(f"\n=== 过程判定序列 {Path(clip_dir).name} " + f"({len(frames)}帧, ~{rec.meta.get('duration_s','?')}s, 窗口N={n} 步长={stride}) ===") + print(f"{'win':<5}{'t(s)':<7}{'判定':<10}{'reason':<14}{'stable':<8}{'sharp':<8}{'content':<8}{'best图'}") + print("-" * 78) + rows = [] + i = 0 + win = 0 + flip_hist = [] # 跨窗口多帧一致性: 累积"疑似倒置"标志 + while i + n <= len(frames): + window = frames[i:i + n] + res = run_funnel(window, cfg) + t = rec.timestamps[i] if i < len(rec.timestamps) else i * 0.05 + comp = res.components or {} + if res.accepted: + verdict = "B_usable" + elif res.reject_reason == "need_focus": + verdict = "C_needfocus" + elif res.reject_reason == "unstable": + verdict = "A_moving" + else: + verdict = res.reject_reason + # 存 best 帧 (窗口内 best_index; 拒绝时也存参考帧) + bidx = res.best_index if 0 <= res.best_index < len(window) else 0 + best_name = f"w{win:03d}_t{t:.1f}_{verdict}.jpg" + (out_dir / best_name).write_bytes(window[bidx]) + # best 帧的完整指标 + bm = res.per_frame[bidx] if 0 <= bidx < len(res.per_frame) else {} + # === 方向判据 (只对被接受的 best 帧跑; 拒绝的帧朝向无意义) === + orient_state = ""; ud_score = ""; side_ratio = ""; orient_hint = ""; flip_consistent = "" + if res.accepted: + _, gd = process_orientation(window[bidx]) + if gd.get("ok"): + orient_state = gd.get("orient_state", "") + ud_score = gd.get("ud_score", "") + side_ratio = gd.get("sideways_ratio", "") + orient_hint = gd.get("orient_hint") or "" + # 多帧一致性(按窗口顺序累积, 与 live 同逻辑): 5窗内≥3窗疑倒置且本窗也疑 -> 触发 + flip_hist.append(1 if gd.get("flipped_frame") else 0) + del flip_hist[:-5] + if len(flip_hist) >= 3 and sum(flip_hist) >= 3 and gd.get("flipped_frame"): + flip_consistent = 1 + if not orient_hint: + orient_hint = "请确认药盒正反" + else: + flip_consistent = 0 + print(f"{win:<5}{t:<7.1f}{verdict:<10}{res.reject_reason:<14}" + f"{comp.get('stable',0):<8.2f}{comp.get('sharp',0):<8.2f}{comp.get('content',0):<8.2f}{best_name}") + rows.append({ + "win": win, "t_s": round(t, 2), "verdict": verdict, + "reject_reason": res.reject_reason, "accepted": int(res.accepted), + "need_focus": int(res.need_focus), + "g_stable": comp.get("stable", ""), "g_sharp": comp.get("sharp", ""), + "g_content": comp.get("content", ""), "g_light": comp.get("light", ""), + "seg_flow": comp.get("flow", ""), + "best_sharp": bm.get("sharpness", ""), "best_local": bm.get("local_sharp", ""), + "best_worst_block": bm.get("worst_block", ""), "best_text_ratio": bm.get("text_ratio", ""), + "best_frame": best_name, + # 方向判据输出 (供核对方向准确性) + "orient_state": orient_state, "ud_score": ud_score, + "sideways_ratio": side_ratio, "flip_consistent": flip_consistent, + "orient_hint": orient_hint, + "manual_check": "", # ← 你看完 best 图后填: 该对焦的真该吗? ok/wrong + "manual_orient": "", # ← 你填这窗 best 的真实朝向: upright/sideways/flipped + }) + i += stride + win += 1 + # 存 CSV + cols = ["win", "t_s", "verdict", "reject_reason", "accepted", "need_focus", + "g_stable", "g_sharp", "g_content", "g_light", "seg_flow", + "best_sharp", "best_local", "best_worst_block", "best_text_ratio", + "best_frame", + "orient_state", "ud_score", "sideways_ratio", "flip_consistent", "orient_hint", + "manual_check", "manual_orient"] + csv_path = out_dir / "sequence.csv" + with open(csv_path, "w", encoding="utf-8-sig", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols) + w.writeheader() + w.writerows(rows) + from collections import Counter + cnt = Counter(r["verdict"] for r in rows) + print(f"\n判定分布: {dict(cnt)}") + print(f"CSV: {csv_path}") + print(f"best图: {out_dir}/ (筛 need_focus 的窗口, 看对应 best图是否真需调焦)") + return rows + + +def record_clip(grabber: "FrameGrabber", ctl: "CamControl", clip_dir: Path, + pre_n: int = 8, post_n: int = 8, gap_s: float = 0.05) -> Recording: + """录一段: 抓 pre_n 帧(未对焦) -> 触发真AF -> 抓 post_n 帧(对焦后)。 + 含对焦点标记, 供 rerun 复现 C 出口对焦效果 (解法一)。""" + rec = Recording(clip_dir) + t0 = time.monotonic() + # 未对焦段 + for _ in range(pre_n): + jpg = grabber.tcp.capture() + if jpg: + rec.frames.append(jpg) + rec.timestamps.append(time.monotonic() - t0) + time.sleep(gap_s) + # 触发真 AF + rec.af_index = len(rec.frames) + ctl.trigger_af() + time.sleep(0.2) # 给对焦一点生效时间 + # 对焦后段 + for _ in range(post_n): + jpg = grabber.tcp.capture() + if jpg: + rec.frames.append(jpg) + rec.timestamps.append(time.monotonic() - t0) + time.sleep(gap_s) + rec.meta["pre_n"] = pre_n + rec.meta["post_n"] = post_n + rec.save() + return rec + + +def rerun_clip(rec: "Recording", cfg: "FunnelConfig", n: int, focus_n: int = 3) -> dict: + """对一段录制跑漏斗: 用前 n 帧(对焦前); 若判 need_focus, 用对焦后帧重评。 + 输入固定, 只变 cfg/n/focus_n -> 受控对比。返回判定 + 是否用了对焦。""" + pre = rec.pre_af_frames() + if len(pre) < n: + n = len(pre) + batch = pre[:n] + res = run_funnel(batch, cfg) + focused = False + if res.need_focus: + post = rec.post_af_frames() + if post: + extra = post[:focus_n] + res = run_funnel(batch + extra, cfg) # 合并重评, best 应来自对焦后帧 + focused = True + return { + "accepted": res.accepted, + "reject_reason": res.reject_reason, + "need_focus": res.need_focus, + "focused": focused, + "best_index": res.best_index, + "components": res.components, + "n": n, "focus_n": focus_n, + } + + +def rerun_sweep(clip_dir, cfg: "FunnelConfig" = None, + n_values=(3, 5, 7), focus_values=(0, 3)): + """对一段录制扫参数, 打印结果表 (找 N 拐点 / 对焦是否值得)。 + 受控对比: 同一段帧, 只变 N 和 focus_n。""" + cfg = cfg or FunnelConfig() + rec = Recording.load(clip_dir) + print(f"\n=== rerun 扫参 {Path(clip_dir).name} " + f"({len(rec.frames)}帧, 对焦点@{rec.af_index}) ===") + print(f"{'N':<5}{'focus_n':<9}{'判定':<12}{'用对焦':<8}{'best':<6}{'原因'}") + print("-" * 50) + for n in n_values: + for fn in focus_values: + r = rerun_clip(rec, cfg, n, fn) + verdict = "接受" if r["accepted"] else "拒绝" + print(f"{n:<5}{fn:<9}{verdict:<12}{'是' if r['focused'] else '否':<8}" + f"{r['best_index']:<6}{r['reject_reason']}") + return rec + + +def main(): + ap = argparse.ArgumentParser(description="ESP32 相机流水线台 (形态2: 漏斗+录制+rerun)") + ap.add_argument("--ip", help="眼镜 IP (live/record 模式必需)") + ap.add_argument("--tcp-port", type=int, default=5000) + ap.add_argument("--port", type=int, default=8080, help="网页端口") + ap.add_argument("--record-dir", default="./tuning_logs", help="CSV/批次 存盘目录") + ap.add_argument("--duration", type=float, default=5.0, help="点控制按钮后自动记录秒数") + ap.add_argument("--quality", type=int, default=4, help="启动默认 JPEG 质量(小=清晰, 默认4最高)") + # 录制 / rerun 模式 + ap.add_argument("--record", metavar="CLIP_DIR", help="录一段(含对焦点)到指定目录, 然后退出") + ap.add_argument("--pre-n", type=int, default=8, help="录制: 对焦前帧数") + ap.add_argument("--post-n", type=int, default=8, help="录制: 对焦后帧数") + ap.add_argument("--rerun", metavar="CLIP_DIR", help="对一段录制扫参数(N/focus_n), 然后退出") + args = ap.parse_args() + + # ---- rerun 模式: 纯离线, 不连眼镜 ---- + if args.rerun: + rec = Recording.load(args.rerun) + if rec.meta.get("kind") == "stream" or rec.af_index < 0: + rerun_stream(args.rerun) # 连续过程 -> 过程判定序列(问题1) + else: + rerun_sweep(args.rerun) # 对焦对比 clip -> 扫参数 + return + + # ---- record 模式: 连眼镜录一段 ---- + if args.record: + if not args.ip: + print("record 模式需要 --ip"); return + tcp = TCPImageClient(args.ip, args.tcp_port) + grabber = FrameGrabber(tcp, Recorder(Path(args.record_dir)), {"res": "HD", "rotate": 0}) + grabber.start(); time.sleep(0.3) + ctl = CamControl(args.ip) + print(f"[REC] 录制中: {args.pre_n}帧(未对焦) -> AF -> {args.post_n}帧(对焦后)...") + rec = record_clip(grabber, ctl, Path(args.record), args.pre_n, args.post_n) + print(f"[REC] 已存 {rec.dir} ({len(rec.frames)}帧, 对焦点@{rec.af_index})") + print(f"[REC] rerun: python cam_pipeline.py --rerun {rec.dir}") + grabber.stop() + return + + # ---- live 模式: 网页 ---- + if not args.ip: + print("live 模式需要 --ip"); return + recorder = Recorder(Path(args.record_dir)) + ctl_state = {"res": "HD", "rotate": 0, "af_enabled": True} + funnel_cfg = FunnelConfig() + pstats = PipelineStats(Path(args.record_dir) / "batches") + + tcp = TCPImageClient(args.ip, args.tcp_port) + grabber = FrameGrabber(tcp, recorder, ctl_state) + grabber.start() + ctl = CamControl(args.ip) + # 启动默认: HD + 最高画质(quality=4), 省得每次进网页手调 + ctl.set_resolution("HD"); ctl_state["res"] = "HD" + ctl.set_quality(args.quality) + time.sleep(0.15); ctl.trigger_af() # 切完补一次对焦 + print(f"[PIPE] 启动默认: HD + quality={args.quality} + 对焦") + + app = make_app(grabber, ctl, recorder, ctl_state, args.duration, funnel_cfg, pstats) + print(f"[PIPE] http://localhost:{args.port} (TCP {args.ip}:{args.tcp_port})") + print(f"[PIPE] 录制受控对比: python cam_pipeline.py --ip {args.ip} --record clips/case1") + print("[PIPE] 独占 TCP: 调参时不要同时跑主程序") + try: + web.run_app(app, host="0.0.0.0", port=args.port, print=None) + finally: + grabber.stop() + + +if __name__ == "__main__": + main() diff --git a/extensions/assistive_harness/phase_b/esp32_runtime.py b/extensions/assistive_harness/phase_b/esp32_runtime.py new file mode 100644 index 0000000..d59c1a6 --- /dev/null +++ b/extensions/assistive_harness/phase_b/esp32_runtime.py @@ -0,0 +1,1767 @@ +"""ESP32 Phase B Harness runtime. + +复用 rokid_runtime 的全部 harness 骨架（HarnessClient / GatewaySessionManager / +GatewayDuplexSession / PCSpeaker / OutputGate / 控制逻辑），只把设备 I/O 从 +"PC 起 web server 等 Rokid 推" 换成 "PC 主动连 ESP32 拉音视频"。 + +数据方向对比： + Rokid : 眼镜(APK) --push--> PC 的 aiohttp server + ESP32 : PC --pull--> ESP32(CameraWebServer, /ws_audio_v2 + TCP:5000) + +汇聚点完全一致： + 音频 -> audio_queue(给 gateway session) + harness.send_audio(镜像给 8021 ASR) + 图像 -> latest_frame.set() + harness.send_frame(镜像给 8021) + AI 音频输出 -> PCSpeaker +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import signal +import ssl +import struct +import base64 +import os +import time +from pathlib import Path +from typing import Any, Optional + +import aiohttp +import numpy as np + +from .timing_probe import TimingProbe +from .funnel_gate import FunnelGate +from .session_recorder import SessionRecorder +from .recorder_live import LiveRecorder +try: + from .bridge_ui import WebUIServer +except Exception: + WebUIServer = None + +# ── 复用 rokid_runtime 的 harness 骨架（一个字不改）────────────────────── +from .rokid_runtime import ( + RokidRuntimeConfig, + GatewaySessionManager, + GatewayDuplexSession, + HarnessClient, + PCSpeaker, + NullSpeaker, + OutputGate, + DropOldestAudioQueue, + AudioMirrorChunker, + LatestFrame, + RuntimeStats, + CtrlCExitWatchdog, + SkillRegistry, + default_skills_config, + pcm16le_to_float32, + apply_pcm16_gain, + now_ms, + SAMPLE_RATE_IN, +) + +LOG = logging.getLogger("assistive_harness.phase_b.esp32") + +# ESP32 音频包头： = seq(4) ts_ms(4) n_samples(2) reserved(2) +# 注意：第4字段是 reserved(pad)，固件真丢包计数 g_ring_drops 未发到 wire， +# PC 端用 seq 跳变推断真丢包。 +ESP32_PKT_HDR = 12 +# TCP 图像帧头 magic +_TCP_IMG_MAGIC = 0x55AA55AA + +# 句子边界标点（作为"保护单位"的边界） +def _install_asr_tap(harness, live_rec) -> None: + """把 8021 的 asr.transcript 截下来落盘（评分取提问时刻 t0 用）。 + + 为什么不是另开一条连接：8021 每个 client_id 有独立的 runtime 和 outbound 队列， + asr.transcript 只投递给"送音频进来的那个 client"（即 esp32-phase-b）， + 另开的 esp32-asr-tap 虽然能连上，但永远收不到任何消息。 + 所以只能挂在同一条 HarnessClient 上——rokid_runtime 里加了个可选的 on_message + 旁路（默认 None，不影响原行为），这里赋值即可。 + + 8021 推的字段（server.py 148-154）： + {"type":"asr.transcript", "asr_event_id":..., "utterance":"这上面写的是什么", + "confidence":..., "final_at_ms":..., + "suppressed": true/false, "reason": "..."} ← 被回声抑制时带这两个 + 注意：**被 echo drop 的也照样推**（抑制发生在 route/echo 之后），所以这里全都记， + 由评分侧按问句筛，不在这里过滤。 + """ + if live_rec is None or harness is None: + return + + def _on_message(payload): + if not isinstance(payload, dict): + return + if payload.get("type") != "asr.transcript": + return + utt = str(payload.get("utterance") or "").strip() + if not utt: + return + try: + live_rec.log_asr(utt, payload.get("final_at_ms"), + payload.get("confidence")) + LOG.info("[ASR] %s%s", utt, + " (suppressed)" if payload.get("suppressed") else "") + except Exception as e: + LOG.warning("[ASR] 落盘失败: %s", e) + + harness.on_message = _on_message + LOG.info("[ASR] transcript 旁路已挂在 HarnessClient 上") + + +class TurnPrinter: + """把逐 chunk 的 text 碎片聚合成整段 turn（移植自 demo_esp32_duplex_0703）。 + + -o 每个 chunk 只吐几个字，评分要的是"这次回答说了什么"。规则： + text 非空 且（不在 turn 内 或 listen/speak 发生切换）→ 开新 turn，turn_idx +1 + end_of_turn → 返回 (turn_idx, 整段文本, is_listen) 并结束本 turn + is_listen=True 的 turn 是 8021 ASR 识别出的用户说话（用于取提问结束时刻 t0）。 + """ + + def __init__(self) -> None: + self.turn_idx = 0 + self._in_turn = False + self._turn_is_listen = None + self._turn_text: list[str] = [] + + def _reset(self) -> None: + self._in_turn = False + self._turn_text.clear() + + def feed(self, is_listen: bool, end_of_turn: bool, text: str): + """返回 (turn_idx, 完整文本或"", is_listen)。 + + 除 end_of_turn 外，**listen/speak 切换时也会把上一段吐出来** —— + 原实现在切换时直接 _reset() 丢掉，模型一路流式说、end_of_turn 迟迟不来时 + 整段就没了（bare 组实测 transcript 全空）。 + """ + flushed = None + if text: + if (not self._in_turn) or (self._turn_is_listen != is_listen): + if self._in_turn and self._turn_text: + flushed = (self.turn_idx, "".join(self._turn_text), + bool(self._turn_is_listen)) + self._reset() + self.turn_idx += 1 + self._in_turn = True + self._turn_is_listen = is_listen + self._turn_text.append(text) + if flushed is not None and not end_of_turn: + return flushed + if end_of_turn: + full_text = "".join(self._turn_text) + cur_idx = self.turn_idx + cur_listen = bool(self._turn_is_listen) + self._reset() + return cur_idx, full_text, cur_listen + return self.turn_idx, "", is_listen + + +_SENT_PUNCT = "，。！？,.!?" + + +def _emit_chunk_img(web_ui, jpeg: bytes, idx: int, img_sent: bool, age_ms: int = 0): + """把一帧图推给 bridge_ui 第一视角（type=chunk, img_b64）。无客户端时零负担。""" + if web_ui is None or not getattr(web_ui, "live_clients", None): + return + try: + b64 = base64.b64encode(jpeg).decode("ascii") if jpeg else None + import asyncio as _a + _a.create_task(web_ui.emit({ + "type": "chunk", + "idx": int(idx), + "img_b64": b64, + "img_sent": bool(img_sent), + "img_age_ms": int(age_ms), + })) + except Exception: + pass + + +class SentenceTracker: + """跟踪模型输出的句子边界，供漏斗"标点保护"策略用。 + + handle_result 每个 chunk 调 update()：检测句末标点，累计标点序号。 + image_loop：reject 挂起时 snapshot() 记当前标点序号；之后 + boundary_reached(snap, speaker) 判断"自挂起后是否出现了新句末标点， + 且该段语音已经播完（speaker 队列基本清空）"。 + + 关键：模型文字生成远快于语音播放，文字标点会早于语音到达。所以"语音播到 + 标点"不能靠文字标点时刻或墙钟估算，而要看 speaker.pending_ms —— 队列里还 + 没播的语音降到很低时，才说明当前这段（含标点前的字）真的播完了。这样打断点 + 落在句末标点、语音播完那一刻，不会把句子从中间截断。 + """ + def __init__(self): + self._speaking = False + self._punct_seq = 0 # 句末标点计数（每出现一个 +1） + + def update(self, text: str, audio_ms: float, is_listen: bool): + self._speaking = not is_listen + if text and any(p in text for p in _SENT_PUNCT): + self._punct_seq += 1 + + @property + def speaking(self) -> bool: + return self._speaking + + def snapshot(self): + return {"punct_seq": self._punct_seq} + + def boundary_reached(self, snap, speaker=None): + """自 snap 之后：出现了新句末标点，且该段语音已播完（speaker 队列近空）。""" + if self._punct_seq <= snap["punct_seq"]: + return False # 还没出现新的句末标点 → 仍在保护当前句 + # 出现了新标点：等语音真的播到这里（speaker 队列基本清空） + if speaker is not None: + try: + if speaker.pending_ms() > 150.0: # 还有 >150ms 没播完 → 语音还没到标点 + return False + except Exception: + pass + return True + +# ── 强制措施：播放离线预生成的 reject 提示 wav（模型音色，运行时不碰 chat/KV）── +_REJECT_WAV_CACHE = {} # reason -> float32 ndarray（进程内缓存，只读一次盘） + + +def _load_reject_wav(wav_dir, reason): + """读预生成的 {reason}.wav（24kHz 16-bit mono）→ float32 ndarray，带缓存。 + 找不到返回 None（跳过播报，不阻断主流程）。""" + import wave as _wave + if reason in _REJECT_WAV_CACHE: + return _REJECT_WAV_CACHE[reason] + path = os.path.join(wav_dir, str(reason) + ".wav") + if not os.path.isfile(path): + _REJECT_WAV_CACHE[reason] = None + return None + try: + with _wave.open(path, "rb") as wf: + n = wf.getnframes() + raw = wf.readframes(n) + pcm16 = np.frombuffer(raw, dtype=np.int16) + pcm = (pcm16.astype(np.float32) / 32768.0) + _REJECT_WAV_CACHE[reason] = pcm + return pcm + except Exception: + _REJECT_WAV_CACHE[reason] = None + return None + +# ── 强制措施（防幻觉）：reject 时用模型音色念提示 ───────────────────────── +# 复用 test_tts_speak 验证过的链路：走 gateway 的 /ws/chat(wss)，zero-shot TTS， +# 模型音色念任意文本，返回 float32 24kHz 音频 → PCSpeaker 播。 +# 停/恢复走 8021(funnel.stop/funnel.resume)，和真人「停一下」同一条 controller。 +_TTS_SYS_PROMPT = ( + "模仿音频样本的音色并生成新的内容。请用这种声音风格来为用户提供帮助。" + "直接作答，不要有冗余内容。" +) + + +async def _speak_hint_via_chat( + gateway_host: str, + gateway_port: int, + hint_text: str, + speaker, + ssl_ctx, +) -> bool: + """用 chat zero-shot TTS 让模型用自己音色念 hint_text，播到 PCSpeaker。 + + 返回 True=念出并已入队播放；False=失败（不阻断主流程）。 + """ + url = f"wss://{gateway_host}:{gateway_port}/ws/chat" + req = { + "messages": [ + {"role": "system", "content": _TTS_SYS_PROMPT}, + {"role": "user", "content": "请朗读以下内容：" + hint_text}, + ], + "streaming": False, + "generation": {"max_new_tokens": 128, "length_penalty": 1.1}, + "tts": {"enabled": True}, + "use_tts_template": True, + "omni_mode": False, + } + try: + async with aiohttp.ClientSession() as sess: + async with sess.ws_connect(url, ssl=ssl_ctx, max_msg_size=128 * 1024 * 1024) as ws: + await ws.send_json(req) + audio_b64 = None + sr = 24000 + # 加超时：念提示走 /ws/chat,若 duplex 占着 worker 会排队。 + # 超时返回,避免永久阻塞 image_loop(否则后续抓图全停)。 + deadline = time.monotonic() + 8.0 + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + LOG.warning("[强制措施] 念提示超时(可能 /ws/chat 排队,duplex 占用 worker)") + return False + try: + msg = await asyncio.wait_for(ws.receive(), timeout=remaining) + except asyncio.TimeoutError: + LOG.warning("[强制措施] 念提示超时(等 chat 响应)") + return False + if msg.type != aiohttp.WSMsgType.TEXT: + if msg.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR): + return False + continue + data = json.loads(msg.data) + if data.get("type") == "done": + audio_b64 = data.get("audio_data") + sr = data.get("audio_sample_rate") or 24000 + break + if data.get("type") == "error": + LOG.warning("[强制措施] chat TTS error: %s", data.get("error")) + return False + if not audio_b64: + LOG.warning("[强制措施] chat TTS 无音频返回") + return False + # audio_data 是 float32 base64（24kHz mono）→ PCSpeaker(也是 24kHz float32) + import base64 as _b64 + pcm = np.frombuffer(_b64.b64decode(audio_b64), dtype=np.float32) + # block_and_flush 之后 PCSpeaker.blocked=True，enqueue 会被丢；先 resume 解除 + await speaker.resume() + await speaker.enqueue(pcm, generation=0) + LOG.info("[强制措施] 念提示: %r (%d samples, %.2fs)", + hint_text, pcm.size, pcm.size / float(sr)) + return True + except Exception as e: + LOG.warning("[强制措施] 念提示失败: %s", e) + return False + + +# ============================================================ +# ESP32 音频输入：PC 主动连 ESP32 的 /ws_audio_v2，收 int16 PCM +# （摘自 demo_esp32_duplex_0703 的 esp32_audio_reader，去掉 ring/live_rec， +# 直接把每包音频喂给 harness 骨架的 audio_queue + harness.send_audio） +# ============================================================ +async def esp32_audio_reader( + host: str, + port: int, + manager: GatewaySessionManager, + harness: HarnessClient, + audio_queue: DropOldestAudioQueue, + audio_mirror: AudioMirrorChunker, + stats: RuntimeStats, + input_gain: float, + stop_evt: asyncio.Event, + probe=None, + live_rec=None, +) -> None: + url = f"ws://{host}:{port}/ws_audio_v2" + LOG.info("[ESP32] audio WS: %s", url) + backoff = 1.0 + last_log = time.monotonic() + + while not stop_evt.is_set(): + try: + async with aiohttp.ClientSession() as session: + async with session.ws_connect(url, heartbeat=30, max_msg_size=0) as ws: + LOG.info("[ESP32] audio WS connected") + backoff = 1.0 + stats.audio_clients = 1 + async for msg in ws: + if stop_evt.is_set(): + break + if msg.type != aiohttp.WSMsgType.BINARY: + if msg.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR): + break + continue + data = msg.data + if len(data) < ESP32_PKT_HDR: + continue + # 固件包头: seq(4) ts_ms(4) n_samples(2) reserved(2) + # 注意: 第4字段是 reserved(pad)，不是丢包数；真丢包用 seq 跳变推断 + seq, ts_ms, n_samples, reserved = struct.unpack(" 0.0: + stats.non_silent_audio_packets += 1 + amplified = apply_pcm16_gain(raw, input_gain) + audio_queue.put_nowait(amplified) # -> gateway session + for frame in audio_mirror.feed(amplified): + await harness.send_audio(frame) # -> 8021 ASR 镜像 + # -o record：录 user 音（原始 samples，未放大） + if live_rec is not None: + try: + live_rec.feed_user_raw(samples) + except Exception: + pass + + if probe is not None: + probe.mark_audio(seq, n_samples, packet_rms) + + now = time.monotonic() + if now - last_log >= 5.0: + LOG.info( + "[ESP32] rx=%d pkts seq=%d ts=%d rsv=%d rms=%.4f", + stats.audio_packets, seq, ts_ms, reserved, stats.audio_rms, + ) + last_log = now + except asyncio.CancelledError: + raise + except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as e: + LOG.warning("[ESP32] audio WS error: %s, retry in %.1fs", e, backoff) + finally: + stats.audio_clients = 0 + if not stop_evt.is_set(): + try: + await asyncio.wait_for(stop_evt.wait(), timeout=backoff) + except asyncio.TimeoutError: + pass + backoff = min(backoff * 2, 10.0) + + LOG.info("[ESP32] audio reader stopped") + + +# ============================================================ +# rerun 音频 reader：读 live_user.wav，按 40ms 包喂 audio_queue + harness +# （移植自 rerun_source.local_pcm_reader，下游从 ring 改为 esp32 的 +# audio_queue + harness，其余节奏/回放逻辑一致） +# ============================================================ +async def rerun_audio_reader( + session_dir: str, + manager: GatewaySessionManager, + harness: HarnessClient, + audio_queue: DropOldestAudioQueue, + audio_mirror: AudioMirrorChunker, + stats: RuntimeStats, + input_gain: float, + stop_evt: asyncio.Event, + speed: float = 1.0, + live_rec=None, + ready_evt: Optional[asyncio.Event] = None, + replay_t0=None, + hard_stop: Optional[asyncio.Event] = None, + done_evt: Optional[asyncio.Event] = None, +) -> None: + if hard_stop is None: + hard_stop = stop_evt + # 等 duplex session 就绪再推，否则排队期间的音频全丢（见 _wait_gateway_ready） + if ready_evt is not None: + await ready_evt.wait() + import wave as _wave + wav_path = os.path.join(session_dir, "live_user.wav") + if not os.path.isfile(wav_path): + LOG.error("[RERUN] live_user.wav 不存在: %s", wav_path) + stop_evt.set() + return + with _wave.open(wav_path, "rb") as w: + sr_in = w.getframerate() + pcm_i16 = np.frombuffer(w.readframes(w.getnframes()), dtype=np.int16) + _SR = 16000 + _PKT = 640 # 40ms @16k + total = pcm_i16.size + LOG.info("[RERUN] audio: %s samples=%d dur=%.2fs sr=%d", + wav_path, total, total / _SR, sr_in) + + packet_interval = (_PKT / _SR) / max(speed, 0.01) + ts_ms = 0 + cursor = 0 + pkt = 0 + # next_tick 必须是"本函数真正开始推包的那一刻"。 + # 曾经这里写成 replay_t0()（就绪门放行的时刻），但那之后还要经过任务调度、 + # 读 wav、打日志，等跑到循环里时 next_tick 已经是过去时刻 → + # sleep_for 恒为负 → 每轮都不 sleep → 564 个包在几十毫秒内全灌进 audio_queue + # → 队列容量 96、DropOldest 把开头的提问全丢了，模型只听到最后 3.8 秒的尾音。 + # （stats 里的 audio=25.0pps 统计的是"推进队列"的数量，正好掩盖了这个问题。） + # 图那边按 frames.jsonl 的 t 等待、以 replay_t0 为零点，两边起点差几十毫秒， + # 对 1s 粒度的 chunk 没有影响。 + next_tick = time.monotonic() + # 只受 hard_stop 控制，不再被 stop_evt 掐断。 + # 原来是 `while cursor < total and not stop_evt.is_set()`： + # 图那条路径放完图会 set stop_evt，而素材的图轮数常少于音频秒数 + # （drugbox 16 轮图 vs 22.5s 音频）→ 图先放完就把音频推送掐了， + # 后半段提问根本没进模型。两条流应各自跑完自己的。 + while cursor < total and not hard_stop.is_set(): + end = min(cursor + _PKT, total) + chunk_i16 = pcm_i16[cursor:end] + cursor = end + pkt += 1 + raw = chunk_i16.astype(" 0: + # 这里等的是 hard_stop，不是 stop_evt。 + # 之前 while 条件改成了 hard_stop，但循环体里仍 await stop_evt 并 break， + # 等于没改——图放完 set stop_evt 照样把音频推送掐断。 + try: + await asyncio.wait_for(hard_stop.wait(), timeout=sleep_for) + break + except asyncio.TimeoutError: + pass + + if done_evt is not None: + done_evt.set() + LOG.info("[RERUN] audio 推完 %d 包 (%.2fs)，等模型说完…", pkt, ts_ms / 1000.0) + # 音频推完不立即 stop。注意：这里**不能** await stop_evt 就 break —— + # 图那条路径放完图也会 set stop_evt，两条互相踩，结果谁先到谁把另一条掐了 + # （图少的素材尤其明显：图很快放完 → 直接收工 → 模型话没说完）。 + # 这里独立按时间等，让模型有机会把最后一句说完；真正的结束由 image_loop + # 的静默判据 + 尾巴决定，或走这里的兜底。 + waited = 0.0 + while waited < 120.0: + if stop_evt.is_set(): + # 另一条已判定结束，再宽限一小段让 TTS 播完 + await asyncio.sleep(2.0) + break + await asyncio.sleep(0.5) + waited += 0.5 + if not stop_evt.is_set(): + stop_evt.set() + LOG.info("[RERUN] audio reader stopped") + + +# ============================================================ +# rerun 图像 loop：用 LocalImageSource 按 chunk 顺序读 best 图，直接喂模型 +# （-o rerun 不重跑 funnel——best 已是筛过的，直接 send_frame） +# ============================================================ +async def rerun_image_loop( + session_dir: str, + latest_frame, + harness: HarnessClient, + stats: RuntimeStats, + interval_s: float, + stop_evt: asyncio.Event, + web_ui=None, + live_rec=None, + ready_evt: Optional[asyncio.Event] = None, + replay_t0=None, + done_evt: Optional[asyncio.Event] = None, + peer_done: Optional[asyncio.Event] = None, + speaker=None, + sent_tracker=None, + tail_wait_s: float = 30.0, +) -> None: + if ready_evt is not None: + await ready_evt.wait() + try: + from .rerun_source import LocalImageSource + except Exception as e: + LOG.error("[RERUN] 无法导入 LocalImageSource: %s", e) + stop_evt.set() + return + from pathlib import Path as _Path + try: + src = LocalImageSource(_Path(session_dir)) + except Exception as e: + LOG.error("[RERUN] LocalImageSource 初始化失败: %s", e) + stop_evt.set() + return + # 按 chunk_idx 升序回放 + idxs = sorted(src._map.keys()) + LOG.info("[RERUN] image loop: %d 帧待回放", len(idxs)) + for cidx in idxs: + if stop_evt.is_set(): + break + jpeg = await src.capture(cidx) + if jpeg: + ts = now_ms() + latest_frame.set(jpeg, ts) + stats.image_count += 1 + await harness.send_frame(jpeg, latest_frame.sequence, ts) + LOG.info("[RERUN] send frame idx=%d (%d bytes)", cidx, len(jpeg)) + _emit_chunk_img(web_ui, jpeg, latest_frame.sequence, True) + if live_rec is not None: + try: + live_rec.on_frame(jpeg, latest_frame.sequence) + except Exception: + pass + try: + await asyncio.wait_for(stop_evt.wait(), timeout=interval_s) + break + except asyncio.TimeoutError: + pass + LOG.info("[RERUN] image loop 回放完毕") + if done_evt is not None: + done_evt.set() + # -o rerun 的收尾。之前这条路径图放完就什么都不做，结束全靠 + # rerun_audio_reader 里 `while waited < 120.0` 的兜底干等两分钟 + #（日志表现：音频推完后一串 audio=0.0pps 的 STATS，两分钟才退）， + # 而且 --rerun-tail-wait-s 对它无效。 + # 判定与 esp32_image_loop 保持一致：先等音频也推完，再看模型说完没有 + #（队列空 且 不在 speak，连续保持 2s 才算），最后加 2s 尾巴。 + if peer_done is not None and not peer_done.is_set(): + LOG.info("[RERUN] 图放完，等音频推完…") + try: + await asyncio.wait_for(peer_done.wait(), timeout=180.0) + except asyncio.TimeoutError: + LOG.warning("[RERUN] 等音频超时") + _QUIET_HOLD_S, _TAIL_S = 2.0, 2.0 + _quiet_since = None + _deadline = time.monotonic() + tail_wait_s + while time.monotonic() < _deadline and not stop_evt.is_set(): + try: + pending = speaker.pending_ms() if speaker is not None else 0.0 + except Exception: + pending = 0.0 + speaking = bool(getattr(sent_tracker, "speaking", False)) if sent_tracker else False + if pending <= 50 and not speaking: + if _quiet_since is None: + _quiet_since = time.monotonic() + elif time.monotonic() - _quiet_since >= _QUIET_HOLD_S: + break + else: + _quiet_since = None + await asyncio.sleep(0.25) + try: + await asyncio.wait_for(stop_evt.wait(), timeout=_TAIL_S) + except asyncio.TimeoutError: + pass + LOG.info("[RERUN] 结束（静默保持%.1fs + 尾巴%.1fs）", _QUIET_HOLD_S, _TAIL_S) + stop_evt.set() + + +# ============================================================ +# ESP32 图像输入：TCP 5000 持久连接，请求-响应取裸 JPEG +# （摘自 demo_esp32_duplex_0703 的 TcpImageClient，接口不变） +# 帧头(20B 小端): magic(4) frame_id(4) w(2) h(2) fmt(1) reserved(3) len(4) +# ============================================================ +class TcpImageClient: + def __init__(self, host: str, port: int = 5000): + self.host = host + self.port = port + self._reader: Optional[asyncio.StreamReader] = None + self._writer: Optional[asyncio.StreamWriter] = None + self._lock = asyncio.Lock() + + async def _ensure_conn(self, timeout_s: float) -> bool: + if self._reader is not None and self._writer is not None and not self._writer.is_closing(): + return True + try: + self._reader, self._writer = await asyncio.wait_for( + asyncio.open_connection(self.host, self.port), timeout=timeout_s) + sock = self._writer.get_extra_info("socket") + if sock is not None: + import socket as _s + sock.setsockopt(_s.IPPROTO_TCP, _s.TCP_NODELAY, 1) + LOG.info("[TCP-IMG] connected to %s:%d", self.host, self.port) + return True + except Exception as e: + LOG.debug("[TCP-IMG] connect failed: %s", e) + await self._close() + return False + + async def _close(self) -> None: + if self._writer is not None: + try: + self._writer.close() + except Exception: + pass + self._reader = None + self._writer = None + + async def capture(self, timeout_s: float = 1.0) -> Optional[bytes]: + async with self._lock: + if not await self._ensure_conn(timeout_s): + return None + try: + self._writer.write(b"C") + await self._writer.drain() + hdr = await asyncio.wait_for(self._reader.readexactly(20), timeout=timeout_s) + magic, frame_id, w, h = struct.unpack_from(" 4 * 1024 * 1024: + LOG.warning("[TCP-IMG] insane len=%d, reconnect", length) + await self._close() + return None + data = await asyncio.wait_for(self._reader.readexactly(length), timeout=timeout_s) + return bytes(data) + except asyncio.CancelledError: + raise + except Exception as e: + LOG.warning("[TCP-IMG] capture failed: %r, reconnect", e) + await self._close() + return None + + +# ============================================================ +# funnel rerun：把录制的整簇多帧当图源，复用 esp32_image_loop 重跑漏斗+播报 +# RecordedImageClient.capture() 按 frames.jsonl 每轮整簇顺序吐帧， +# funnel.run_once 调 N 次凑一簇 → 重判 send/reject → 播报/标点保护，逻辑全复用。 +# ============================================================ +class RecordedImageClient: + def __init__(self, session_dir: str, n_frames: int, no_funnel: bool = False): + import json as _json + self.dir = session_dir + self.images_dir = os.path.join(session_dir, "images") + self.n_frames = max(1, int(n_frames)) + self._rounds = [] # 每轮: [jpg_bytes, ...] + fr_path = os.path.join(session_dir, "frames.jsonl") + self._round_names = [] # 每轮的帧文件名（供 record 复用） + self._round_t = [] # 每轮录制时刻 t（秒），用于按原始时间轴回放 + with open(fr_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rec = _json.loads(line) + except Exception: + continue + names = rec.get("frames") or [] + jpgs = [] + keep_names = [] + for nm in names: + p = os.path.join(self.images_dir, nm) + if os.path.isfile(p): + with open(p, "rb") as fp: + jpgs.append(fp.read()) + keep_names.append(nm) + if jpgs: + self._rounds.append(jpgs) + self._round_names.append(keep_names) + self._round_t.append(float(rec.get("t", len(self._rounds) - 1))) + LOG.info("[RERUN-IMG] 载入 %d 轮，每轮≤%d帧%s", + len(self._rounds), self.n_frames, + "（无漏斗：每轮取中间张代表帧）" if no_funnel else "（funnel：整簇逐帧）") + self._round_i = 0 + self._frame_i = 0 + self._no_funnel = no_funnel + self.exhausted = False + self._t0 = None # 回放起点（优先用外部注入的统一零点） + self.replay_t0 = None # 由 runtime 注入，与音频共用 + # 整份 record 一张图都没有（图像链路当时丢了 / 手动删空 images）： + # 这是有效素材（对照：无漏斗→模型 0 图编场景；有漏斗→全程 reject no_frames）。 + # 此时不能置 exhausted，否则会被当成"图放完了"秒退；让音频推完来结束整场。 + self._no_images = (len(self._rounds) == 0) + if self._no_images: + LOG.info("[RERUN-IMG] 这份 record 没有可用图像 → 全程无图回放" + "（由音频长度决定结束）") + + def current_round_cluster(self): + """返回当前轮的整簇 (jpg_list, name_list)，供无漏斗 rerun 存 record 用。""" + i = self._round_i + if 0 <= i < len(self._rounds): + return self._rounds[i], self._round_names[i] + return [], [] + + async def _wait_round_time(self): + """按录制时刻回放：等到墙钟走到本轮的 t 再吐这一轮的帧。 + + 录制时每轮耗时并不均匀（对焦轮 ~1.9s、reject 播报轮 2.3~4s），若 rerun 按 + image_loop 的固定 interval_s 匀速吃图，就会以数倍速把图吃光（实测 33s 的录制 + 11s 就放完）。音频是按真实时长回放的，用 frames.jsonl 的 t 对齐，两边同一条 + 时间轴。 + """ + if self._t0 is None: + # 优先用统一回放零点（与音频同源）；没有才退回"第一次 capture 那一刻" + self._t0 = self.replay_t0 if self.replay_t0 is not None else time.monotonic() + i = self._round_i + if 0 <= i < len(self._round_t): + target = self._round_t[i] + delay = target - (time.monotonic() - self._t0) + if delay > 0: + await asyncio.sleep(min(delay, 10.0)) + + async def capture(self, timeout_s: float = 1.0): + if self._no_images: + # 全程无图：一直返回 None，但不置 exhausted（不提前结束）。 + # 有漏斗 → run_once 拿到 0 帧 → reject(no_frames) → 播"没有拿到画面"； + # 无漏斗 → 不 send → 模型 0 图 → 暴露编场景。 + await asyncio.sleep(0.05) + return None + if self._round_i >= len(self._rounds): + self.exhausted = True + return None + # 只在一轮的第一帧上等待，同一轮内的多帧连续吐（模拟原始高频抓帧） + if self._frame_i == 0: + await self._wait_round_time() + rnd = self._rounds[self._round_i] + if self._no_funnel: + # 无漏斗（对照组）：每轮取一张代表帧（中间张，模拟"随手抓一张就发"）， + # 每次 capture 直接进入下一轮。含模糊图会照发 → 暴露幻觉。 + jpg = rnd[len(rnd) // 2] + self._round_i += 1 + return jpg + # funnel：一帧帧吐整簇，供 run_once 调 N 次凑一簇 + jpg = rnd[self._frame_i % len(rnd)] + self._frame_i += 1 + if self._frame_i >= self.n_frames: + self._frame_i = 0 + self._round_i += 1 + return jpg + + async def _close(self): + pass + + +# ============================================================ +# ESP32 图像轮询循环：定时用 TCP 取图 -> latest_frame + harness.send_frame +# ============================================================ +async def esp32_image_loop( + client: TcpImageClient, + latest_frame: LatestFrame, + harness: HarnessClient, + stats: RuntimeStats, + interval_s: float, + timeout_s: float, + stop_evt: asyncio.Event, + probe=None, + funnel=None, + recorder=None, + speaker=None, + gateway_host: str = "127.0.0.1", + gateway_port: int = 8040, + ssl_ctx=None, + force_measure: bool = False, + manager=None, + reject_wav_dir: str = "assets/reject_wav", + sent_tracker=None, + live_rec=None, + web_ui=None, + ready_evt: Optional[asyncio.Event] = None, + tail_wait_s: float = 30.0, + no_reject: bool = False, + peer_done: Optional[asyncio.Event] = None, + done_evt: Optional[asyncio.Event] = None, +) -> None: + if ready_evt is not None: + await ready_evt.wait() + LOG.info("[ESP32] image loop start (interval=%.2fs, funnel=%s)", + interval_s, "on" if funnel else "off") + _last_grab_mono = None + + # ── 强制措施状态 ── + _funnel_stopped = False # 是否已发过 funnel.stop（在停住态，等好图 resume） + _last_hint_mono = 0.0 # 上次念提示的时刻（节流用） + _HINT_THROTTLE_S = 5.0 # 同类连续 reject 的念提示节流间隔 + _pending_reject = None # 标点保护：挂起中的 reject（等标点/超时再复查） + # ── 「持续坏」检测：3s 窗口内 reject≥2 → 判定输出基于坏图=幻觉，直接打断+压制 ── + # 覆盖"错错错对"场景（前半也错，标点保护不适用）。窗口单位=判定/chunk(每≈1s)。 + # 依据：模型拿图→输出约 1-1.5s；2s 都坏则幻觉必已在输出，必须闭嘴。 + _judge_history = [] # 最近判定时刻+是否reject: [(mono, is_reject), ...] + _WINDOW_S = 3.0 # 窗口长度 + _BAD_THRESH = 2 # 窗口内 reject≥此 → 持续坏，直接打断(跳过标点保护) + # _funnel_active：念提示临界区标志（预留给兜底：念提示那几秒真人喊话的处理） + # TODO(兜底,下次实现): 念提示期间若收到真人 STOP/RESUME/RESET， + # 先掐掉提示音(speaker.block_and_flush)+清此标志，再执行真人指令，避免撞车。 + # 当前只占位，不实现逻辑。 + _funnel_active = False # noqa: F841 (预留) + + async def _capture_one(): + return await client.capture(timeout_s=timeout_s) + + while not stop_evt.is_set(): + t0 = time.monotonic() + # funnel rerun：录制的整簇多帧放完了 → 优雅结束（不报 no_frames） + if getattr(client, "exhausted", False): + LOG.info("[FUNNEL-RERUN] 录制图已放完") + # 图轮数常少于音频秒数（drugbox 16 轮 vs 22.5s），不能图一放完就收工， + # 否则后半段音频（可能正含提问）根本没推给模型。先等音频也推完。 + if peer_done is not None and not peer_done.is_set(): + LOG.info("[FUNNEL-RERUN] 等音频推完…") + try: + await asyncio.wait_for(peer_done.wait(), timeout=180.0) + except asyncio.TimeoutError: + LOG.warning("[FUNNEL-RERUN] 等音频超时") + LOG.info("[FUNNEL-RERUN] 两条流都完事，等模型说完后结束") + # 等模型把话说完再 stop。判据不能只看"队列空"——模型是流式产出的， + # 某一瞬间队列排空只是"下一段还没到"，不代表这轮结束（图少的素材尤其明显： + # 图很快放完，恰好撞上队列空，就把还没说完的话掐了）。 + # 改为：队列必须连续 _QUIET_HOLD_S 保持空，且 sent_tracker 不在 speak 中， + # 再加一段固定尾巴，给最后一句 TTS 留出播放时间。 + _QUIET_HOLD_S = 2.0 + _TAIL_S = 2.0 + _quiet_since = None + _deadline = time.monotonic() + tail_wait_s + while time.monotonic() < _deadline and not stop_evt.is_set(): + try: + pending = speaker.pending_ms() if speaker is not None else 0.0 + except Exception: + pending = 0.0 + speaking = False + if sent_tracker is not None: + speaking = bool(getattr(sent_tracker, "speaking", False)) + if pending <= 50 and not speaking: + if _quiet_since is None: + _quiet_since = time.monotonic() + elif time.monotonic() - _quiet_since >= _QUIET_HOLD_S: + break + else: + _quiet_since = None + await asyncio.sleep(0.25) + # 尾巴：让最后一句 TTS 播完，也给模型一点补充的机会 + try: + await asyncio.wait_for(stop_evt.wait(), timeout=_TAIL_S) + except asyncio.TimeoutError: + pass + LOG.info("[FUNNEL-RERUN] 结束（静默保持%.1fs + 尾巴%.1fs）", + _QUIET_HOLD_S, _TAIL_S) + stop_evt.set() + break + try: + if funnel is None: + # ── 无漏斗：取 1 帧直发（含模糊图，暴露幻觉）── + # record 仍存整簇（mp4 用全部采集帧，和有漏斗一致，只是无 reject 标注） + cluster_jpgs, cluster_names = ([], []) + if hasattr(client, "current_round_cluster"): + cluster_jpgs, cluster_names = client.current_round_cluster() + jpeg = await client.capture(timeout_s=timeout_s) + grab_ms = (time.monotonic() - t0) * 1000.0 + if jpeg: + ts = now_ms() + latest_frame.set(jpeg, ts) + stats.image_count += 1 + await harness.send_frame(jpeg, latest_frame.sequence, ts) + _emit_chunk_img(web_ui, jpeg, latest_frame.sequence, True) + if probe is not None: + since_last = (t0 - _last_grab_mono) * 1000.0 if _last_grab_mono else 0.0 + probe.mark_grab(latest_frame.sequence, grab_ms, len(jpeg), since_last) + _last_grab_mono = t0 + # 存整簇给 record（无漏斗：send=True 无 reason） + if live_rec is not None and cluster_jpgs: + _dec = type("D", (), {"frames": cluster_jpgs, "best_index": len(cluster_jpgs)//2, + "send": True, "reason": ""})() + try: + await asyncio.to_thread( + live_rec.on_funnel_round, _dec, latest_frame.sequence) + except Exception as e: + LOG.warning("[LIVE] on_funnel_round(no-funnel) err: %s", e) + else: + # ── 有漏斗：一轮判定，合格才 send_frame ── + decision = await funnel.run_once(_capture_one) + # 统一 record：每轮整簇多帧 + 判定 → live_rec（供三种 rerun）。 + # 必须放在所有分支之前：持续坏分支末尾有 continue，放在最后会把 + # 触发压制的那些轮整簇漏录（实测 42s 只落下 8 轮）。录制是原始素材， + # 不该因为走了哪条处理路径而缺失。 + if live_rec is not None: + try: + await asyncio.to_thread( + live_rec.on_funnel_round, decision, latest_frame.sequence) + except Exception as e: + LOG.warning("[LIVE] on_funnel_round err: %s", e) + grab_ms = decision.timings.get("grab_ms", 0.0) + async def _do_reject_interrupt(reason): + """真正执行打断：停 duplex → 播 wav → 恢复。""" + wav_pcm = _load_reject_wav(reject_wav_dir, reason) + if wav_pcm is None: + LOG.warning("[强制措施] 无 %s.wav，跳过播报", reason) + return + try: + await harness.send({"type": "funnel.stop", "reason": reason}) + LOG.info("[强制措施] funnel.stop 已发 (%s)", reason) + except Exception as e: + LOG.warning("[强制措施] funnel.stop 失败: %s", e) + await asyncio.sleep(0.35) # 等 STOP 经 8021→rokid→block_and_flush + try: + await speaker.resume() + await speaker.enqueue(wav_pcm, generation=0) + dur = len(wav_pcm) / 24000.0 + LOG.info("[强制措施] 播 reject wav: %s (%.2fs)", reason, dur) + if live_rec is not None: + live_rec.log_event("HINT", f"{reason} ({dur:.2f}s)") + await asyncio.sleep(dur + 0.2) + except Exception as e: + LOG.warning("[强制措施] 播 wav 失败: %s", e) + try: + await harness.send({"type": "funnel.resume", "reason": "hint_done"}) + LOG.info("[强制措施] funnel.resume 已发（提示播完）") + except Exception as e: + LOG.warning("[强制措施] funnel.resume 失败: %s", e) + + # ── 消融用：选 best 但永远放行（关闭工作③的拒绝/播报）── + # 用途：把"多帧选 best"(工作②)的收益从"拒绝坏图"(工作③)里剥出来。 + # 对照 bare（每轮取中间帧直发），本 arm 每轮取 best 直发， + # 下游模型侧完全一致，差异只来自选图。 + if no_reject: + _b = decision.best + if _b: + ts = now_ms() + latest_frame.set(_b, ts) + stats.image_count += 1 + await harness.send_frame(_b, latest_frame.sequence, ts) + _emit_chunk_img(web_ui, _b, latest_frame.sequence, True) + if live_rec is not None: + try: + await asyncio.to_thread( + live_rec.on_frame, _b, latest_frame.sequence) + except Exception as e: + LOG.warning("[LIVE] on_frame err: %s", e) + LOG.info("[漏斗-放行] best (原判定=%s)", decision.reason) + # 消融 arm 不执行拒绝，但**判别结果要留痕**： + # 这一轮漏斗本来会不会拦、拦的理由是什么，是分析判别倾向的依据。 + if live_rec is not None and not decision.send: + live_rec.log_event( + "WOULD-REJECT", + f"{decision.reason} {decision.hint or ''}") + else: + LOG.info("[漏斗-放行] 本轮无 best（%s），跳过", decision.reason) + elapsed = time.monotonic() - t0 + try: + await asyncio.wait_for(stop_evt.wait(), + timeout=max(0.0, interval_s - elapsed)) + except asyncio.TimeoutError: + pass + continue + + # ── 「持续坏」检测（优先于标点保护）── + # 3s 窗口内"真坏"reject≥2 → 输出基于坏图=幻觉。此时不走标点保护 + # （前半也错，不值得保护），直接触发"停→播提示→恢复"。 + # need_focus 不算坏：它是系统正在对焦(run_once 已发 /reg 重抓)， + # 属于内部自愈，不是用户造成的持续坏，算进去会误判、干扰对焦流程。 + # "真坏" = reject 且 reason 不是 need_focus（severe_shake/unstable/ + # too_dark/orient 等，要用户动手的）。 + _now = time.monotonic() + _is_real_bad = (not decision.send) and (decision.reason != "need_focus") + _judge_history.append((_now, _is_real_bad)) + _judge_history[:] = [(t, r) for (t, r) in _judge_history + if _now - t <= _WINDOW_S] + _bad_in_window = sum(1 for (_t, r) in _judge_history if r) + + if _bad_in_window >= _BAD_THRESH and _is_real_bad: + _pending_reject = None # 持续坏优先，取消标点保护挂起 + # 到这里必是"真坏"(severe_shake/unstable/too_dark/orient)。 + # 直接触发完整"停→播提示→恢复"（复用 _do_reject_interrupt， + # 停模型输出不杀死、播完恢复）。耗时≈播报时长，天然间隔。 + LOG.info("[持续坏] 3s内%d次真坏reject，打断+播提示（停→播→恢复）", + _bad_in_window) + if live_rec is not None: + live_rec.log_event( + "PERSIST-BAD", + f"3s内{_bad_in_window}次真坏 → 打断 ({decision.reason})") + await _do_reject_interrupt(decision.reason) + elapsed = time.monotonic() - t0 + try: + await asyncio.wait_for(stop_evt.wait(), + timeout=max(0.0, interval_s - elapsed)) + except asyncio.TimeoutError: + pass + continue + + # ── 「标点保护」核心：突发 reject 不立即打断，保护当前段语音播放到 + # 下一个句末标点，到标点那一刻复查当时判定；仍 reject 才打断，否则无事。 + # 挂起期间的 send/reject 都不算数，只看"到标点那一刻"的判定。 + # （snapshot 记标点序号，boundary_reached 判断新标点+其语音已播完） + if force_measure and speaker is not None and _pending_reject is not None: + # 正在保护中：只判断"是否到标点(语音播完)或超时"，到了才用当时判定复查 + now_mono = time.monotonic() + reached = (sent_tracker is not None + and sent_tracker.boundary_reached( + _pending_reject["snap"], speaker)) + timed_out = (now_mono - _pending_reject["since"]) >= 1.5 + if reached or timed_out: + # 到标点/超时 → 复查此刻判定 + if decision.send: + LOG.info("[标点保护] %s，此刻已恢复(send)，无事发生", + "到标点" if reached else "超时1.5s") + _pending_reject = None + elif decision.reason == "need_focus": + _pending_reject = None # 复查是对焦，静默 + else: + # 仍 reject → 打断（受节流约束） + LOG.info("[标点保护] %s，仍 reject(%s)，打断", + "到标点" if reached else "超时1.5s", decision.reason) + if now_mono - _last_hint_mono >= _HINT_THROTTLE_S: + _last_hint_mono = now_mono + _pending_reject = None + await _do_reject_interrupt(decision.reason) + else: + _pending_reject = None # 被节流 + + if decision.send and decision.best: + ts = now_ms() + latest_frame.set(decision.best, ts) + stats.image_count += 1 + await harness.send_frame(decision.best, latest_frame.sequence, ts) + LOG.info("[漏斗] send (%s)", decision.reason) + _emit_chunk_img(web_ui, decision.best, latest_frame.sequence, True) + # -o record：只录真发送给模型的 best 图（chunk_idx = sequence） + if live_rec is not None: + try: + await asyncio.to_thread( + live_rec.on_frame, decision.best, latest_frame.sequence) + except Exception as e: + LOG.warning("[LIVE] on_frame err: %s", e) + # 注意：send 不清 _pending_reject，保护期只看到标点那一刻 + else: + LOG.info("[漏斗] reject(%s) -> hint: %s", + decision.reason, decision.hint) + if live_rec is not None: + live_rec.log_event("REJECT", f"{decision.reason} {decision.hint or ''}") + if force_measure and speaker is not None and decision.reason != "need_focus": + now_mono = time.monotonic() + if _pending_reject is not None: + pass # 已在保护中，上面已处理，不重复挂起 + elif sent_tracker is not None and sent_tracker.speaking: + # 模型正念字：挂起，保护到下一个标点再复查 + _pending_reject = { + "since": now_mono, + "reason": decision.reason, + "snap": sent_tracker.snapshot(), + } + LOG.info("[标点保护] speak中，reject(%s)挂起，等语音播到标点或超时1.5s", + decision.reason) + else: + # 模型没念字（listen/静默）→ 无需保护，直接打断（受节流） + if now_mono - _last_hint_mono >= _HINT_THROTTLE_S: + _last_hint_mono = now_mono + await _do_reject_interrupt(decision.reason) + if probe is not None: + since_last = (t0 - _last_grab_mono) * 1000.0 if _last_grab_mono else 0.0 + jb = len(decision.best) if decision.best else 0 + tm = decision.timings + probe.mark_grab( + latest_frame.sequence, grab_ms, jb, since_last, + reason=decision.reason, + judge_ms=tm.get("judge_ms"), + af_ms=tm.get("af_ms"), + n_frames=tm.get("n_frames"), + ) + _last_grab_mono = t0 + except Exception as e: + LOG.warning("[ESP32] image loop error: %s", e, exc_info=True) + # 控制取图节奏 + elapsed = time.monotonic() - t0 + try: + await asyncio.wait_for(stop_evt.wait(), timeout=max(0.0, interval_s - elapsed)) + except asyncio.TimeoutError: + pass + LOG.info("[ESP32] image loop stopped") + + +# ============================================================ +# ESP32 Runtime：继承 rokid 的 PhaseBRokidRuntime，复用全部 harness 骨架， +# 只覆盖 start()：不起 web server，改起两个 ESP32 主动拉取 task。 +# ============================================================ +from .rokid_runtime import PhaseBRokidRuntime + + +def _insecure_ssl_for_wss(url: str) -> Optional[ssl.SSLContext]: + """wss:// 且自签名证书时，返回一个不校验证书的 ssl context；ws:// 返回 None。 + + 与 rokid GatewayDuplexSession._ssl_context 同款做法（check_hostname=False, + verify_mode=CERT_NONE），用于连本地自签名的 8021。""" + if not url.lower().startswith("wss://"): + return None + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +class TlsHarnessClient(HarnessClient): + """与 HarnessClient 完全一致，仅在 ws_connect 时对 wss:// 传入自签名 ssl。 + + 覆盖 run()：逐行照抄父类，唯一区别是 ws_connect 多了 ssl= 参数。""" + + async def run(self) -> None: + ssl_ctx = _insecure_ssl_for_wss(self.url) + while not self._stop.is_set(): + try: + async with aiohttp.ClientSession() as client: + async with client.ws_connect( + self.url, heartbeat=30, max_msg_size=0, ssl=ssl_ctx + ) as ws: + self.ws = ws + self.connected = True + LOG.info("Harness connected: %s", self.url) + async for message in ws: + if message.type != aiohttp.WSMsgType.TEXT: + if message.type in ( + aiohttp.WSMsgType.CLOSED, + aiohttp.WSMsgType.ERROR, + ): + break + continue + payload = json.loads(message.data) + message_type = payload.get("type") + if message_type == "harness.ready": + await self.manager.emit_recovery_sync() + elif message_type == "control.intent": + task = asyncio.create_task( + self.manager.handle_control(payload) + ) + self._control_tasks.add(task) + task.add_done_callback(self._control_tasks.discard) + except asyncio.CancelledError: + raise + except Exception as exc: + if not self._stop.is_set(): + LOG.warning("Harness connection failed: %s", exc) + finally: + self.connected = False + self.ws = None + if not self._stop.is_set(): + await asyncio.sleep(self.reconnect_s) + + +class PhaseBEsp32Runtime(PhaseBRokidRuntime): + """ESP32 版 runtime：I/O 从 'PC 起 server 被动收' 换成 'PC 主动拉 ESP32'。 + + __init__ / close / health / _initial_session_loop / _stats_loop 全部继承。 + 仅覆盖 start()：把 Rokid 的 web server 换成 esp32_audio_reader + esp32_image_loop。 + """ + + def __init__(self, config, esp32_host, esp32_port, image_tcp_port, + image_interval_s, image_timeout_s, probe=None, + funnel=None, recorder=None, force_measure=False, + gateway_host="127.0.0.1", gateway_port=8040, + reject_wav_dir="assets/reject_wav", + live_record_dir=None, rerun_from=None, + funnel_rerun_from=None, web_ui_port=None, + record_no_media: bool = False, + rerun_tail_wait_s: float = 30.0, + no_reject: bool = False, **kwargs): + super().__init__(config, **kwargs) + # 用支持自签名 wss 的 harness client 替换父类建好的普通 HarnessClient， + # 并同步更新 manager.harness 引用（发遥测/镜像走同一个）。 + self.harness = TlsHarnessClient( + config.harness_url, + config.harness_client_id, + self.manager, + config.reconnect_s, + ) + self.manager.harness = self.harness + self._esp32_host = esp32_host + self._esp32_port = esp32_port + self._image_tcp_port = image_tcp_port + self._image_interval_s = image_interval_s + self._image_timeout_s = image_timeout_s + self._tcp_img = TcpImageClient(esp32_host, image_tcp_port) + self._stop_evt = asyncio.Event() + self._probe = probe + self._funnel = funnel + self._recorder = recorder + self._force_measure = force_measure + self._reject_wav_dir = reject_wav_dir + self._gateway_host = gateway_host + self._gateway_port = gateway_port + # 念提示走 gateway wss /ws/chat，自签名证书 → 复用同款 insecure ssl + self._chat_ssl = _insecure_ssl_for_wss(f"wss://{gateway_host}:{gateway_port}/ws/chat") + + self._rerun_from = rerun_from + self._funnel_rerun_from = funnel_rerun_from + self._rerun_tail_wait_s = rerun_tail_wait_s + self._no_reject = no_reject + self._replay_t0 = None # rerun 统一回放零点（就绪时设） + self._rec_client = None + # 回放两条流各自的完成标志：图轮数常少于音频秒数，谁先完都不能掐死对方 + self._audio_done = asyncio.Event() + self._image_done = asyncio.Event() + # bridge_ui 第一视角（8080）。live/rerun/funnel-rerun 都可推 img_b64。 + self._web_ui = None + if web_ui_port and WebUIServer is not None: + try: + self._web_ui = WebUIServer( + port=web_ui_port, + sessions_root=Path(live_record_dir).parent if live_record_dir + else Path("live_sessions"), + mode_info={"mode": "rerun" if (rerun_from or funnel_rerun_from) + else "live"}, + ) + except Exception as e: + LOG.warning("[UI] WebUIServer 创建失败: %s", e) + self._web_ui = None + # -o record（音+图，只存真发送的 best）：和漏斗 record 并存。 + self.live_rec = (LiveRecorder(live_record_dir, no_media=record_no_media) + if live_record_dir else None) + if self.live_rec is not None: + try: + self.live_rec.attach_to_player(self.speaker) # 录 AI 音 + except Exception as e: + LOG.warning("[LIVE] attach_to_player 失败: %s", e) + + # 句子边界追踪器：包装 manager.handle_result，每个模型 chunk 更新句子/语音进度， + # 供 image_loop 的"标点保护"策略读取（reject 时保护当前段到下一个标点再复查）。 + self.sent_tracker = SentenceTracker() + self._turn_printer = TurnPrinter() + self._turn_start_ms = None # 当前 turn 的起点（session 内相对 ms） + _orig_handle_result = self.manager.handle_result + + async def _wrapped_handle_result(session, result): + try: + is_listen = bool(result.get("is_listen")) + text = str(result.get("text") or "") + end_of_turn = bool(result.get("end_of_turn")) + ab64 = str(result.get("audio_data") or "") + audio_ms = 0.0 + if ab64 and not is_listen: + n = len(base64.b64decode(ab64)) // 4 # float32 + audio_ms = n * 1000.0 / 24000.0 + self.sent_tracker.update(text, audio_ms, is_listen) + + # ── 评分用：把 chunk 碎片聚合成整段 turn，落 transcript/subtitles ── + if self.live_rec is not None: + # 先记原始逐 chunk（不依赖 end_of_turn，保证一定有东西可评分） + self.live_rec.log_model_chunk(text, is_listen, end_of_turn, audio_ms) + prev_idx = self._turn_printer.turn_idx + turn_idx_cur, full_text, turn_listen = self._turn_printer.feed( + is_listen, end_of_turn, text) + if text and self._turn_printer.turn_idx > prev_idx: + self._turn_start_ms = self.live_rec.session_ms() + # 只要拿到完整段就落盘（end_of_turn 或 listen/speak 切换冲出的） + if full_text: + end_ms = self.live_rec.session_ms() + 800 # 多挂 0.8s + start_ms = (self._turn_start_ms + if self._turn_start_ms is not None + else max(end_ms - 3000, 0)) + self.live_rec.log_turn_text(turn_idx_cur, full_text, turn_listen) + self.live_rec.log_subtitle( + start_ms=start_ms, end_ms=end_ms, text=full_text, + is_listen=turn_listen, turn_idx=turn_idx_cur) + self._turn_start_ms = None + + # 推模型文字到 bridge_ui 第一视角（type=result） + if self._web_ui is not None and getattr(self._web_ui, "live_clients", None): + asyncio.create_task(self._web_ui.emit({ + "type": "result", + "is_listen": is_listen, + "end_of_turn": end_of_turn, + "text": text, + })) + except Exception: + pass + return await _orig_handle_result(session, result) + + self.manager.handle_result = _wrapped_handle_result + + async def _gate_open_when_ready(self, ready_evt: asyncio.Event) -> None: + """等 gateway 就绪后放行回放任务。""" + try: + await self._wait_gateway_ready() + finally: + # 录制的 t0 必须和回放起点一致，否则 frames.jsonl/subtitles 的时间轴 + # 会把排队那几十秒也算进去，两个 arm 无法对齐。 + if self.live_rec is not None: + self.live_rec.start() + LOG.info("[LIVE] rerun 录制已开（就绪后启动，结束自动出 mp4）") + # 统一回放零点：音频按 40ms 绝对时钟推、图按 frames.jsonl 的 t 等待， + # 两者必须用同一个 t0，否则各自以"自己被调度到的那一刻"为零点， + # 起跑差多少全看事件循环，音图就对不齐。 + self._replay_t0 = time.monotonic() + _rc = getattr(self, "_rec_client", None) + if _rc is not None: + _rc.replay_t0 = self._replay_t0 + ready_evt.set() + + async def _wait_gateway_ready(self, timeout_s: float = 180.0) -> bool: + """等 duplex session 真正 prepared 之后再开始回放。 + + 实测：连着跑两次 rerun 时，后一次会在 gateway 排队（[GW] queue position=1） + 长达 17 秒才 prepared。而回放任务从第 0 秒就推音频和图 —— 这 17 秒的输入 + 全部推给一个还不存在的 session，直接丢掉，提问就在里面，模型自然没反应。 + 更要命的是两个 arm 被吞掉的长度不一样，配对设计直接失效，而且事后从结果上 + 看不出来（长得就像"模型没回答"）。所以回放前必须等就绪。 + """ + t0 = time.monotonic() + warned = False + while time.monotonic() - t0 < timeout_s: + if self._stop_evt.is_set(): + return False + try: + status = str(self.manager.health().get("gateway_status") or "") + except Exception: + status = "" + if status == "running": + waited = time.monotonic() - t0 + if waited > 1.0: + LOG.info("[RERUN] gateway 就绪（等了 %.1fs），开始回放", waited) + return True + if not warned and time.monotonic() - t0 > 3.0: + warned = True + LOG.info("[RERUN] 等 gateway session 就绪…（status=%s）", status or "?") + await asyncio.sleep(0.25) + LOG.warning("[RERUN] 等 gateway 就绪超时 %.0fs，仍开始回放（本次结果可能不可用）", + timeout_s) + return False + + async def start(self) -> None: + await self.speaker.start() + _install_asr_tap(self.harness, self.live_rec) + if self._web_ui is not None: + try: + await self._web_ui.start() + LOG.info("[UI] bridge_ui 第一视角已启动: http://localhost:%d", + self._web_ui.port) + except Exception as e: + LOG.warning("[UI] bridge_ui 启动失败: %s", e) + self._web_ui = None + # 方向分类器预热（原设计，-o 迁移时漏了）：首次 process_orientation 含模型加载+ + # oneDNN 编译（~2s，甚至 5s+）。不预热的话它会推迟到第一次 accept 才现场加载， + # 卡住那一轮，且在此之前链路出不了 send（模型长时间拿不到图）。 + if self._funnel is not None: + try: + ms = await asyncio.to_thread(self._funnel.warmup_orient) + LOG.info("[预热] 方向分类器就绪 (首次 %.0fms，运行时应降到 ~10ms)", ms) + except Exception as e: + LOG.warning("[预热] 方向分类器预热跳过: %s", e) + # ── rerun 模式：用录制的 session 回放（音频+best图）重跑 -o，不接实时设备 ── + if self._rerun_from: + LOG.info("[RERUN] 模式启动，回放 session: %s", self._rerun_from) + _ready = asyncio.Event() + asyncio.create_task(self._gate_open_when_ready(_ready)) + self._tasks = [ + asyncio.create_task(self.harness.run()), + asyncio.create_task(self._initial_session_loop()), + asyncio.create_task(self._stats_loop()), + asyncio.create_task(rerun_audio_reader( + self._rerun_from, + self.manager, self.harness, + self.audio_queue, self.audio_mirror, self.stats, + self.config.input_gain, self._stop_evt, + live_rec=self.live_rec, ready_evt=_ready, + replay_t0=lambda: self._replay_t0, + done_evt=self._audio_done, + )), + asyncio.create_task(rerun_image_loop( + self._rerun_from, self.latest_frame, self.harness, + self.stats, self._image_interval_s, self._stop_evt, + web_ui=self._web_ui, + live_rec=self.live_rec, ready_evt=_ready, + replay_t0=lambda: self._replay_t0, + done_evt=self._image_done, + peer_done=self._audio_done, + speaker=self.speaker, sent_tracker=self.sent_tracker, + tail_wait_s=self._rerun_tail_wait_s, + )), + ] + return + # ── funnel rerun：录制的整簇多帧重跑漏斗；不加 --funnel 则为无漏斗对照组 ── + if self._funnel_rerun_from: + no_funnel = (self._funnel is None) + _ready = asyncio.Event() + asyncio.create_task(self._gate_open_when_ready(_ready)) + LOG.info("[RERUN] %s 模式启动: %s", + "无漏斗对照组" if no_funnel else "funnel 重跑漏斗+播报", + self._funnel_rerun_from) + n_frames = getattr(self._funnel, "n_frames", 3) if self._funnel else 3 + rec_client = RecordedImageClient(self._funnel_rerun_from, n_frames, + no_funnel=no_funnel) + self._rec_client = rec_client # 就绪时注入统一零点 + self._tasks = [ + asyncio.create_task(self.harness.run()), + asyncio.create_task(self._initial_session_loop()), + asyncio.create_task(self._stats_loop()), + asyncio.create_task(rerun_audio_reader( + self._funnel_rerun_from, + self.manager, self.harness, + self.audio_queue, self.audio_mirror, self.stats, + self.config.input_gain, self._stop_evt, + live_rec=self.live_rec, ready_evt=_ready, + replay_t0=lambda: self._replay_t0, + done_evt=self._audio_done, + )), + # 复用完整 esp32_image_loop（run_once 重判 + reject 停播恢复 + 标点保护）， + # 只把图源从 TCP 换成 RecordedImageClient（读录制整簇多帧）。 + asyncio.create_task(esp32_image_loop( + rec_client, self.latest_frame, self.harness, self.stats, + self._image_interval_s, self._image_timeout_s, self._stop_evt, + probe=self._probe, funnel=self._funnel, recorder=None, + speaker=self.speaker, + gateway_host=self._gateway_host, gateway_port=self._gateway_port, + ssl_ctx=self._chat_ssl, force_measure=self._force_measure, + manager=self.manager, + reject_wav_dir=self._reject_wav_dir, + sent_tracker=self.sent_tracker, + live_rec=self.live_rec, # 开 --record-live 则录 rerun 输出→自动出 mp4 + web_ui=self._web_ui, ready_evt=_ready, + tail_wait_s=self._rerun_tail_wait_s, + no_reject=self._no_reject, + peer_done=self._audio_done, done_evt=self._image_done, + )), + ] + return + # 设 ESP32 分辨率为 HD(1280×720)。固件默认 SVGA(800×600)，但漏斗阈值是按 HD + # 标定的（cam_pipeline_v2 注释：分辨率固定 HD 后针对 HD 标定）。-o 迁移时漏了这步， + # 导致一直跑 SVGA、画质与阈值不匹配。这里启动时补上。 + if self._funnel is not None: + ok = await asyncio.to_thread(self._funnel.set_resolution_hd) + LOG.info("[ESP32] 设分辨率 1280×720(UXGA档): %s", + "成功" if ok else "失败(检查ESP32 /control)") + await asyncio.sleep(0.3) # 切分辨率后固件重配，稍等 + if self.live_rec is not None: + self.live_rec.start() + LOG.info("[LIVE] -o record started") + self._tasks = [ + # ── 继承自 rokid 的三个骨架 task ── + asyncio.create_task(self.harness.run()), + asyncio.create_task(self._initial_session_loop()), + asyncio.create_task(self._stats_loop()), + # ── ESP32 特有：主动拉取音视频 ── + asyncio.create_task(esp32_audio_reader( + self._esp32_host, self._esp32_port, + self.manager, self.harness, + self.audio_queue, self.audio_mirror, self.stats, + self.config.input_gain, self._stop_evt, + probe=self._probe, + live_rec=self.live_rec, + )), + asyncio.create_task(esp32_image_loop( + self._tcp_img, self.latest_frame, self.harness, self.stats, + self._image_interval_s, self._image_timeout_s, self._stop_evt, + probe=self._probe, funnel=self._funnel, recorder=self._recorder, + speaker=self.speaker, + gateway_host=self._gateway_host, gateway_port=self._gateway_port, + ssl_ctx=self._chat_ssl, force_measure=self._force_measure, + manager=self.manager, + reject_wav_dir=self._reject_wav_dir, + sent_tracker=self.sent_tracker, + live_rec=self.live_rec, + web_ui=self._web_ui, + no_reject=self._no_reject, + )), + ] + + async def close(self) -> None: + self._stop_evt.set() + if self._web_ui is not None: + try: + await self._web_ui.stop() + except Exception: + pass + if self.live_rec is not None: + try: + # 冲掉还没等到 end_of_turn 的那段（bare 组常见：模型一路流式说， + # 没有 stop/resume 打断，end_of_turn 迟迟不来 → 整段丢失） + tp = getattr(self, "_turn_printer", None) + if tp is not None and getattr(tp, "_in_turn", False): + pending = "".join(getattr(tp, "_turn_text", [])) + if pending: + end_ms = self.live_rec.session_ms() + start_ms = (self._turn_start_ms + if self._turn_start_ms is not None + else max(end_ms - 3000, 0)) + self.live_rec.log_turn_text(tp.turn_idx, pending, + bool(tp._turn_is_listen)) + self.live_rec.log_subtitle( + start_ms=start_ms, end_ms=end_ms, text=pending, + is_listen=bool(tp._turn_is_listen), turn_idx=tp.turn_idx) + LOG.info("[LIVE] 冲出未收尾的 turn #%d (%d字)", + tp.turn_idx, len(pending)) + except Exception as e: + LOG.warning("[LIVE] flush pending turn err: %s", e) + try: + self.live_rec.stop() + LOG.info("[LIVE] -o record stopped") + except Exception as e: + LOG.warning("[LIVE] stop err: %s", e) + await self._tcp_img._close() + if self._probe is not None: + self._probe.close() + if self._recorder is not None: + self._recorder.close() + await super().close() + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="ESP32 Phase B Harness runtime") + # ── ESP32 设备 ── + p.add_argument("--esp32-host", required=True, help="ESP32 IP address") + p.add_argument("--esp32-port", type=int, default=80, help="ESP32 HTTP/WS port") + p.add_argument("--image-tcp-port", type=int, default=5000, help="ESP32 TCP image port") + p.add_argument("--image-interval-s", type=float, default=1.0, help="取图间隔(秒)") + p.add_argument("--image-timeout-s", type=float, default=1.0, help="单次取图超时(秒)") + # ── gateway / harness（与 rokid 一致）── + p.add_argument("--gateway", default="localhost:8040") + p.add_argument("--gateway-tls", action="store_true", default=True) + p.add_argument("--no-tls", dest="gateway_tls", action="store_false") + p.add_argument("--harness-url", default="ws://127.0.0.1:8021/ws/control") + p.add_argument("--client-id", default="esp32-phase-b") + p.add_argument("--skills-config", default=default_skills_config()) + p.add_argument("--cleanup-mode", choices=("light", "full"), default="light") + p.add_argument("--chunk-ms", type=int, default=1_000) + p.add_argument("--force-listen-count", type=int, default=3) + p.add_argument("--audio-queue-packets", type=int, default=96) + p.add_argument("--input-gain", type=float, default=12.0) + p.add_argument("--prompt", default=None, help="system prompt（可选）") + p.add_argument("--no-play", action="store_true") + p.add_argument("--log-level", default="INFO") + # ── 时序探针（纯旁路）── + p.add_argument("--timing-probe", action="store_true", help="开启时序探针，写 CSV") + p.add_argument("--timing-csv", default=None, help="探针 CSV 路径（默认自动带时间戳）") + # ── CV 漏斗 ── + p.add_argument("--funnel", action="store_true", help="开启 CV 漏斗（抓N帧判定，合格才送模型）") + p.add_argument("--scene", default="medicine", choices=("medicine", "stationery"), + help="漏斗场景参数") + p.add_argument("--funnel-frames", type=int, default=3, help="每轮抓帧数（3选2）") + p.add_argument("--no-focus", dest="funnel_focus", action="store_false", default=True, + help="关闭漏斗内的自动对焦触发") + # ── session 录制（对齐 -v 格式）── + p.add_argument("--record", action="store_true", help="录 session（整簇帧+判定到 sessions/）") + p.add_argument("--sessions-root", default="sessions", help="session 根目录") + p.add_argument("--no-reject", action="store_true", + help="消融：漏斗只选 best、永远放行，不拒绝不播报" + "（用于把工作②选图的收益与工作③拒绝分开）") + p.add_argument("--rerun-tail-wait-s", type=float, default=30.0, + help="rerun 图放完后，最多再等模型说完的秒数（默认30）") + p.add_argument("--record-no-media", action="store_true", + help="批量评分用：只写 wav+jsonl+transcript，跳过 jpg 落盘和 mp4 拼接") + p.add_argument("--record-live", action="store_true", + help="-o record：录音+图(只存真发送的best)到 live_sessions/，供 -o rerun") + p.add_argument("--rerun-from", default=None, + help="-o rerun：从 live_sessions/ 回放(音+best图)重跑 -o 模型") + p.add_argument("--funnel-rerun-from", default=None, + help="funnel rerun：从 live_sessions/ 回放(音+整簇多帧)重跑漏斗+播报+-o") + p.add_argument("--web-ui-port", type=int, default=None, + help="开 bridge_ui 第一视角(如 8080)，live/rerun/funnel-rerun 都可看") + # ── 强制措施（防幻觉）：reject 时用模型音色念提示 + 停/恢复走 8021 ── + p.add_argument("--force-measure", action="store_true", + help="开启强制措施：reject→停duplex→播预生成wav→恢复duplex") + p.add_argument("--reject-wav-dir", default="assets/reject_wav", + help="预生成的 reject 提示 wav 目录（gen_reject_wavs.py 生成）") + return p.parse_args() + + +def main() -> None: + args = parse_args() + logging.basicConfig( + level=getattr(logging, args.log_level.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + force=True, + ) + config = RokidRuntimeConfig( + gateway=args.gateway, + gateway_tls=args.gateway_tls, + harness_url=args.harness_url, + harness_client_id=args.client_id, + skills_config=args.skills_config, + cleanup_mode=args.cleanup_mode, + chunk_ms=args.chunk_ms, + force_listen_count=args.force_listen_count, + audio_queue_packets=args.audio_queue_packets, + input_gain=args.input_gain, + play_audio=not args.no_play, + ) + probe = TimingProbe( + enabled=args.timing_probe, + path=args.timing_csv, + chunk_ms=args.chunk_ms, + ) + funnel = None + if args.funnel: + # rerun 时强制关对焦：rerun 没有真实相机，trigger_af 对录制的图无意义， + # 且对焦重抓会让 run_once 多抓一簇 → RecordedImageClient 多消耗一个 round + # → 图提前用光（有漏斗比无漏斗先用光的根因）。 + _is_rerun = bool(args.rerun_from or args.funnel_rerun_from) + _focus = args.funnel_focus and not _is_rerun + funnel = FunnelGate( + args.esp32_host, + scene=args.scene, + n_frames=args.funnel_frames, + enable_focus=_focus, + ) + LOG.info("CV 漏斗已开启: scene=%s frames=%d focus=%s%s", + args.scene, args.funnel_frames, _focus, + "（rerun 强制关对焦）" if (_is_rerun and args.funnel_focus) else "") + recorder = None # 旧 SessionRecorder 已废弃，统一走 LiveRecorder + # 统一 record（音+整簇多帧+best标记）：--record 或 --record-live 都触发。 + live_record_dir = None + if args.record or args.record_live or args.record_no_media: + import time as _t + live_record_dir = os.path.join("live_sessions", _t.strftime("%Y%m%d_%H%M%S")) + # 日志同时落进 session 目录：评分时要按时间轴对齐 reject/播报/模型输出， + # 控制台日志是滚动的、和 session 分离，跑多组 rerun 极易对错。 + try: + os.makedirs(live_record_dir, exist_ok=True) + _fh = logging.FileHandler( + os.path.join(live_record_dir, "run.log"), encoding="utf-8") + _fh.setFormatter(logging.Formatter( + "%(asctime)s %(levelname)s %(name)s: %(message)s")) + logging.getLogger().addHandler(_fh) + LOG.info("[LIVE] 日志同时写入 %s", os.path.join(live_record_dir, "run.log")) + except Exception as e: + LOG.warning("[LIVE] run.log 创建失败: %s", e) + os.makedirs(live_record_dir, exist_ok=True) + LOG.info("统一 record 已开启: %s（音+整簇多帧+best标记，供三种 rerun）", live_record_dir) + # 解析 gateway "host:port"（念提示的 chat TTS 走同一个 gateway 的 wss） + _gw = args.gateway.split("://")[-1] + _gw_host, _, _gw_port = _gw.partition(":") + _gw_host = _gw_host or "127.0.0.1" + _gw_port = int(_gw_port or "8040") + runtime = PhaseBEsp32Runtime( + config, + esp32_host=args.esp32_host, + esp32_port=args.esp32_port, + image_tcp_port=args.image_tcp_port, + image_interval_s=args.image_interval_s, + image_timeout_s=args.image_timeout_s, + probe=probe, + funnel=funnel, + recorder=recorder, + force_measure=args.force_measure, + reject_wav_dir=args.reject_wav_dir, + gateway_host=_gw_host, + gateway_port=_gw_port, + live_record_dir=live_record_dir, + rerun_from=args.rerun_from, + funnel_rerun_from=args.funnel_rerun_from, + web_ui_port=args.web_ui_port, + record_no_media=args.record_no_media, + rerun_tail_wait_s=args.rerun_tail_wait_s, + no_reject=args.no_reject, + ) + if args.force_measure: + LOG.info("强制措施已开启: reject→funnel.stop+念提示, good→funnel.resume " + "(节流5s, chat TTS via wss://%s:%d)", _gw_host, _gw_port) + + LOG.info("ESP32 Phase B input: ws://%s:%d/ws_audio_v2 + TCP:%d", + args.esp32_host, args.esp32_port, args.image_tcp_port) + LOG.info("Harness: %s", config.harness_url) + LOG.info("Gateway: %s://%s", "wss" if config.gateway_tls else "ws", config.gateway) + + async def _run() -> None: + await runtime.start() + stop = asyncio.Event() + + def _sig(*_a): + stop.set() + try: + loop = asyncio.get_running_loop() + for s in (signal.SIGINT, getattr(signal, "SIGBREAK", signal.SIGINT)): + try: + loop.add_signal_handler(s, stop.set) + except (NotImplementedError, ValueError): + signal.signal(s, _sig) + except Exception: + signal.signal(signal.SIGINT, _sig) + # 等"信号(Ctrl+C)"或"runtime 内部 stop_evt(rerun 图放完自动结束)"任一触发 + _internal = getattr(runtime, "_stop_evt", None) + waiters = [asyncio.create_task(stop.wait())] + if _internal is not None: + waiters.append(asyncio.create_task(_internal.wait())) + await asyncio.wait(waiters, return_when=asyncio.FIRST_COMPLETED) + for w in waiters: + w.cancel() + await runtime.close() + + try: + asyncio.run(_run()) + except KeyboardInterrupt: + LOG.info("Interrupted; shutting down") + + +if __name__ == "__main__": + main() diff --git a/extensions/assistive_harness/phase_b/funnel_gate.py b/extensions/assistive_harness/phase_b/funnel_gate.py new file mode 100644 index 0000000..fe683a7 --- /dev/null +++ b/extensions/assistive_harness/phase_b/funnel_gate.py @@ -0,0 +1,218 @@ +"""漏斗闸门（接入 -o 链路）。 + +把 cam_pipeline_v2 的 run_funnel / process_orientation / CamControl 封装成 +"一轮漏斗判定"，供 esp32_runtime 在图像循环里调用： + + gate = FunnelGate(esp32_host, scene="medicine", n_frames=3) + result = await gate.run_once(capture_fn) # capture_fn: async ()->Optional[bytes] + +返回 FunnelDecision： + send : bool 是否放行给模型 + best : bytes|None 放行的最佳帧（已做倒置纠正） + reason : str 决策原因（accepted/severe_shake/unstable/need_focus/orient...） + hint : str|None 拒绝时给用户的提示文本（先输出 text，TTS 通道待定） + timings: dict grab_ms / judge_ms / orient_ms / af_ms / n_frames + +设计依据（防漂移文档 + cam_pipeline_v2）： + - 判定唯一 = run_funnel 的 frame_score 三出口 + argmin 归因 + - need_focus → 触发单次 AF(0x3022) → 等 AF_SETTLE_MS → 重抓一簇 → 重判 + - 倒置 = process_orientation，flipped 时纠正或提示 + - 拒绝 reason → HINTS 文本 +本模块只做"输入端把关+引导"，不碰模型输出侧（那是 harness 的事）。 +""" +from __future__ import annotations + +import asyncio +import time +from typing import Awaitable, Callable, Optional + +from . import cam_pipeline_v2 as cp + +AF_SETTLE_MS = 120 # OV5640 单次对焦 settle（与 v5 一致） + +# reject_reason -> 用户提示（先输出 text；TTS 通道确定后接同一份文本） +HINTS = { + "no_frames": "没有拿到画面，请稍等", + "severe_shake": "画面晃得厉害，请先保持不动", + "unstable": "画面还在晃动，请保持不动一会儿", + "too_dark": "光线太暗，请到亮一点的地方", + "aimed_wrong": "好像没对准，请对准目标", + "need_focus": "对焦中，请拿稳一下", + "orient": "画面好像反了，请倒过来", +} + + +class FunnelDecision: + __slots__ = ("send", "best", "reason", "hint", "timings", + "frames", "best_index", "af_triggered", "af_ok") + + def __init__(self, send, best, reason, hint, timings, + frames=None, best_index=-1, af_triggered=False, af_ok=None): + self.send = send + self.best = best + self.reason = reason + self.hint = hint + self.timings = timings + self.frames = frames or [] # 整簇帧（存 session 用） + self.best_index = best_index # best 在簇里的下标 + self.af_triggered = af_triggered # 本轮有没有触发 AF + self.af_ok = af_ok # AF /reg 请求是否返回成功（None=没触发） + + +class FunnelGate: + def __init__(self, esp32_host: str, scene: str = "medicine", + n_frames: int = 3, frame_gap_s: float = 0.0, + enable_focus: bool = True): + self.scene = scene + self.n_frames = n_frames + self.frame_gap_s = frame_gap_s + self.enable_focus = enable_focus + self._cfg = (cp.make_scene_config(scene) + if hasattr(cp, "make_scene_config") else cp.FunnelConfig()) + # 相机 HTTP 控制走 /control、/reg 到 ESP32（同 esp32_host）。 + # camctl 总是构造：分辨率(set_resolution)独立于对焦，即使不对焦也要设 HD。 + # 对焦(trigger_af)另受 enable_focus 控制。 + self._camctl = cp.CamControl(esp32_host) + self._last_af_mono = 0.0 # 上次真正触发对焦的时刻（冷却用） + self._af_cooldown_s = 2.0 # 对焦冷却：2s 内最多触发 1 次（避免每秒对焦） + + def warmup_orient(self) -> float: + """方向分类器预热（同步，调用方放线程里）。 + + 原设计（-v pc_vlm_v5_funnel）：首次 process_orientation 含模型加载 + oneDNN + 编译开销（实测 ~2s，本次日志里甚至 5.76s），启动时先跑一张假图，把这笔一次性 + 开销挪到启动阶段，运行时 orient_ms 就是纯推理（md 记录：2143ms → 9ms）。 + -o 迁移时漏了这步 → orient 直到第一次 accept 才现场加载，卡住那一轮， + 且在此之前链路出不了 send。返回首次耗时(ms)。 + """ + import numpy as _np + import cv2 as _cv2 + _warm = _np.full((480, 640, 3), 255, dtype=_np.uint8) + ok, buf = _cv2.imencode(".jpg", _warm) + if not ok: + return -1.0 + t0 = time.monotonic() + cp.process_orientation(buf.tobytes()) + return (time.monotonic() - t0) * 1000.0 + + def set_resolution_hd(self, retries: int = 5, gap_s: float = 0.6) -> bool: + """设 1280×720（UXGA档，这块 OV5640 枚举非标准，HD(11)无效、UXGA(13)才 720p）。 + live 启动时 ESP32 可能刚就绪，/control 偶发失败 → 重试几次，避免要手动进网页设。 + 失败时打详细原因（status/异常），不再静默。""" + import logging as _lg + _log = _lg.getLogger("funnel_gate") + for i in range(max(1, retries)): + try: + ok = self._camctl.set_resolution("UXGA") + if ok: + if i > 0: + _log.info("[funnel_gate] set_resolution(UXGA) 第%d次重试成功", i + 1) + return True + _log.warning("[funnel_gate] set_resolution(UXGA) 返回非200 (第%d/%d次)", + i + 1, retries) + except Exception as e: + _log.warning("[funnel_gate] set_resolution(UXGA) 异常 (第%d/%d次): %r", + i + 1, retries, e) + time.sleep(gap_s) + return False + + async def _grab_burst( + self, capture_fn: Callable[[], Awaitable[Optional[bytes]]], n: int + ) -> list[bytes]: + frames: list[bytes] = [] + for i in range(n): + jpeg = await capture_fn() + if jpeg: + frames.append(jpeg) + if self.frame_gap_s > 0 and i < n - 1: + await asyncio.sleep(self.frame_gap_s) + return frames + + async def run_once( + self, capture_fn: Callable[[], Awaitable[Optional[bytes]]] + ) -> FunnelDecision: + """跑一轮漏斗：抓 N 帧 -> run_funnel -> (need_focus 则对焦重抓) -> 倒置 -> 决策。""" + timings: dict = {} + + # 1) 抓一簇 + t0 = time.monotonic() + frames = await self._grab_burst(capture_fn, self.n_frames) + timings["grab_ms"] = round((time.monotonic() - t0) * 1000, 1) + timings["n_frames"] = len(frames) + if not frames: + return FunnelDecision(False, None, "no_frames", HINTS["no_frames"], timings) + + # 2) run_funnel 判定（run_funnel 是同步的，放线程池避免阻塞事件循环） + tj = time.monotonic() + res = await asyncio.to_thread(cp.run_funnel, frames, self._cfg) + timings["judge_ms"] = round((time.monotonic() - tj) * 1000, 1) + timings["af_ms"] = 0.0 + af_triggered = False + af_ok = None + + # 3) need_focus -> 触发单次 AF -> 等 settle -> 重抓 -> 重判 + # 对焦冷却：need_focus 但距上次对焦 < _af_cooldown_s(2s) 时不重复触发， + # 给对焦时间生效（每秒对焦太快，马达反复动反而对不好）。冷却内仍 need_focus + # 就用当前帧判定（该 reject 就 reject，不重抓）。 + if res.need_focus and self.enable_focus and self._camctl is not None: + _now = time.monotonic() + if _now - self._last_af_mono >= self._af_cooldown_s: + self._last_af_mono = _now + taf = _now + af_triggered = True + af_ok = await asyncio.to_thread(self._camctl.trigger_af) # True/False + await asyncio.sleep(AF_SETTLE_MS / 1000.0) + frames2 = await self._grab_burst(capture_fn, self.n_frames) + if frames2: + res = await asyncio.to_thread(cp.run_funnel, frames2, self._cfg) + frames = frames2 + timings["af_ms"] = round((time.monotonic() - taf) * 1000, 1) + timings["n_frames"] = len(frames) + else: + # 冷却期内：不重复对焦，用当前判定结果 + timings["af_cooldown"] = round(self._af_cooldown_s - (_now - self._last_af_mono), 2) + timings["af_ok"] = af_ok + + def _mk(send, best, reason, hint): + bi = frames.index(best) if (best in frames) else -1 + return FunnelDecision(send, best, reason, hint, timings, + frames=frames, best_index=bi, + af_triggered=af_triggered, af_ok=af_ok) + + # 4) 拒绝出口 + if not res.accepted: + reason = res.reject_reason or "reject" + # cam_pipeline 的拒绝出口只设 best_index（注释：仅供参考, 不喂模型）， + # 不设 best_jpg —— 这是原设计意图：拒绝就不送图。 + # 但消融 arm（--no-reject：只选 best、永远放行）需要拿到这一帧， + # 所以这里按 best_index 把帧补出来。**send 仍然是 False**， + # 正常链路完全不受影响：只有显式开了 --no-reject 才会去用它。 + _bj = res.best_jpg + if _bj is None: + bi = getattr(res, "best_index", -1) + if isinstance(bi, int) and 0 <= bi < len(frames): + _bj = frames[bi] + return _mk(False, _bj, reason, + HINTS.get(reason, "看不清楚，请调整一下")) + + # 5) 接受帧的方向检测（OCR 方向分类器优先，见 cam_pipeline.process_orientation） + # 原则「宁拒绝不念错」：方向不正 -> 拒绝 + 提示用户转正，绝不自动纠正、绝不送倒图。 + # process_orientation 只诊断方向、返回的 jpg 未旋转，故不能拿它当"纠正后"送模型。 + best = res.best_jpg + to_ = time.monotonic() + try: + _jpg, geom = await asyncio.to_thread(cp.process_orientation, best) + except Exception: + geom = {"ok": False} + timings["orient_ms"] = round((time.monotonic() - to_) * 1000, 1) + + orient_state = geom.get("orient_state") if isinstance(geom, dict) else None + orient_hint = geom.get("orient_hint") if isinstance(geom, dict) else None + + # upright 才放行；flipped/sideways 拒绝并提示；uncertain 也放行（不硬拦，避免误拒） + if orient_state in ("flipped", "sideways"): + hint = orient_hint or HINTS["orient"] + return _mk(False, best, "orient_" + orient_state, hint) + + # upright / uncertain / 检测不可用 -> 放行原图（不做任何旋转） + return _mk(True, best, "send", None) diff --git a/extensions/assistive_harness/phase_b/recorder_live.py b/extensions/assistive_harness/phase_b/recorder_live.py new file mode 100644 index 0000000..3e17528 --- /dev/null +++ b/extensions/assistive_harness/phase_b/recorder_live.py @@ -0,0 +1,860 @@ +"""recorder_live.py — 录屏式实时录制器 v5.0 +================================================= + +v5.0 改动:user 轨从"ring buffer 切片拼接"改成"WS 原始包直录"。 +和蓝牙耳机录通话语义一致:丢包 = 该段缺失,不再补零去对齐墙钟。 +和 AI 轨完全对称——AI 轨录的是 PortAudio DAC 实际写出的样本。 + +调用顺序: + live_rec = LiveRecorder(session_dir) + live_rec.attach_to_player(speaker) # 必须在 speaker.start() 之前 + speaker.start() + live_rec.start() # 此后 user/ai/frame 才会被记录 + + # ESP32 reader 每收一包就调: + live_rec.feed_user_raw(pcm_f32) # 16kHz 原始,丢包就丢 + + # 每发一个 chunk 配的图: + live_rec.on_frame(jpeg, chunk_idx) + + live_rec.stop() + live_rec.finalize_mp4() +""" + +from __future__ import annotations + +import json +import logging +import shutil +import subprocess +import threading +import time +import wave +from pathlib import Path +from typing import Optional + +import numpy as np + +LOGGER = logging.getLogger("recorder_live") + + +class LiveRecorder: + def __init__( + self, + session_dir: Optional[Path], + user_sr: int = 16000, + ai_sr: int = 24000, + no_media: bool = False, + ): + self.enabled = session_dir is not None + self.dir: Optional[Path] = Path(session_dir) if session_dir else None + self.user_sr = user_sr + self.ai_sr = ai_sr + # no_media：批量 rerun 评分只需要 wav + jsonl + transcript， + # 跳过 jpg 落盘和 mp4 拼接（150 次 rerun 会重复复制整簇图、跑 450 次 ffmpeg）。 + self.no_media = bool(no_media) + + self._t0: Optional[float] = None + + # User 轨:ESP32 WS 来的原始 PCM,丢包 = 缺失,不补零 + self._user_t_start: Optional[float] = None + # self._user_chunks: list[np.ndarray] = [] + self._user_chunks: list[tuple[float, np.ndarray]] = [] + # AI 轨:PortAudio DAC 实际输出(含 underrun 时填的零) + self._ai_t_start: Optional[float] = None + #self._ai_chunks: list[np.ndarray] = [] + self._ai_chunks: list[tuple[float, np.ndarray]] = [] + # Frames + self._frames: list[tuple[float, str, int]] = [] + self._funnel_rounds: list[dict] = [] # 每轮整簇多帧+判定（统一record） + self._round_counter: int = -1 # on_funnel_round 轮次计数（qid，每轮+1） + self._transcript = None # transcript.txt 句柄 + self._subtitles = None # subtitles.jsonl 句柄 + self._chunks_f = None # model_chunks.jsonl 句柄 + + self._lock = threading.Lock() + self._started = threading.Event() + self._stopping = threading.Event() + + self._user_calls = 0 + self._ai_calls = 0 + + # ------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------ + + def attach_to_player(self, player) -> None: + if not self.enabled: + return + orig_enqueue = player.enqueue + rec = self + + # PCSpeaker.enqueue 是 async，签名 (pcm, generation)。包装必须同签名 + async， + # 否则模型音频 enqueue 报 "takes 1 positional argument but 2 were given"， + # 导致 AI 音播不出、也录不进（live 听不到声音）。 + async def wrapped_enqueue(pcm_f32, generation=0): + await orig_enqueue(pcm_f32, generation) + if (not rec._started.is_set() or rec._stopping.is_set() + or rec._t0 is None or pcm_f32 is None or pcm_f32.size == 0): + return + try: + pcm = pcm_f32 if pcm_f32.dtype == np.float32 else pcm_f32.astype(np.float32) + pcm = pcm.copy() + t = max(0.0, time.monotonic() - rec._t0) + with rec._lock: + if rec._ai_t_start is None: + rec._ai_t_start = t + rec._ai_chunks.append((t, pcm)) + rec._ai_calls += 1 + except Exception as e: + LOGGER.warning("[LIVE] enqueue hook err: %r", e) + + player.enqueue = wrapped_enqueue + player._orig_enqueue = orig_enqueue # 暴露原始方法，绕过 hook 的场景使用 + LOGGER.info("[LIVE] attached to SPK enqueue (async, pcm+generation)") + + def start(self) -> None: + if not self.enabled or self._started.is_set(): + return + assert self.dir is not None + self.dir.mkdir(parents=True, exist_ok=True) + #(self.dir / "live_images").mkdir(exist_ok=True) + self._t0 = time.monotonic() + self._started.set() + # 评分用：模型/用户的成段文本 + 时间轴（移植自 demo_esp32_duplex_0703 的 + # SessionRecorder.log_turn_text / log_subtitle）。 + # transcript.txt —— 人读，[AI #n] / [LISTEN #n] 整段 + # subtitles.jsonl —— 机读，{start_ms,end_ms,text,is_listen,turn_idx} + # is_listen=True 的 turn 就是 8021 ASR 识别出的用户说话 → 提问结束时刻 t0 从这拿 + try: + self._transcript = (self.dir / "transcript.txt").open("w", encoding="utf-8") + self._transcript.write(f"# session {self.dir.name}\n") + self._transcript.flush() + self._subtitles = (self.dir / "subtitles.jsonl").open("w", encoding="utf-8") + self._chunks_f = (self.dir / "model_chunks.jsonl").open("w", encoding="utf-8") + except Exception as e: + LOGGER.warning("[LIVE] transcript/subtitles 打开失败: %s", e) + self._transcript = None + self._subtitles = None + self._chunks_f = None + LOGGER.info( + "[LIVE] recording started: %s (user=%dHz ai=%dHz)", + self.dir, self.user_sr, self.ai_sr, + ) + + def session_ms(self) -> int: + """session 内相对毫秒（与 frames.jsonl 的 t 同一条时间轴）。""" + if self._t0 is None: + return 0 + return int((time.monotonic() - self._t0) * 1000.0) + + def log_model_chunk(self, text: str, is_listen: bool, end_of_turn: bool, + audio_ms: float = 0.0) -> None: + """逐 chunk 原始记录 —— **一条都不过滤**。 + + 这是诊断用的原始流水：-o 是 1Hz 决策，listen 期间大量 chunk 是 + text="" 且 end_of_turn=False。之前这里写了 + if not text and not end_of_turn: return + 把它们全扔了，结果 24 秒的会话只剩 5 条，看上去像"模型 5.7 秒才出一个 + chunk / 输入分块坏了"——那是记录函数造出来的假象，不是事实。 + 要判断模型节奏是否正常，必须看到每一个 chunk。 + """ + if not self.enabled or not self._started.is_set() or self.dir is None: + return + if self._chunks_f is None: + return + try: + self._chunks_f.write(json.dumps({ + "session_ms": self.session_ms(), + "text": text, + "is_listen": bool(is_listen), + "end_of_turn": bool(end_of_turn), + "audio_ms": round(float(audio_ms), 1), + }, ensure_ascii=False) + "\n") + self._chunks_f.flush() + except Exception: + pass + + def log_event(self, tag: str, text: str) -> None: + """把漏斗侧事件写进 transcript，和 USER/AI 同一条时间轴。 + + 评分时要判"这次拦截对应哪次回答"，三者必须在同一条人读时间轴上： + [USER @12480ms] 这上面写的是什么 + [REJECT @13520ms] severe_shake 画面晃得厉害，请先保持不动 + [HINT @13900ms] severe_shake (2.52s) + [AI #3 @16100ms] 上面写着阿莫西林胶囊... + 机读的一份仍在 frames.jsonl（每轮 send/reason）。 + """ + if not self.enabled or not self._started.is_set() or self._transcript is None: + return + try: + self._transcript.write(f"[{tag} @{self.session_ms()}ms] {text}\n") + self._transcript.flush() + except Exception: + pass + + def log_asr(self, utterance: str, final_at_ms=None, confidence=None) -> None: + """8021 ASR 识别出的用户说话。评分取 t0 用。 + + 写两处：transcript.txt 的 [USER] 行（人读）、subtitles.jsonl 的 + is_listen=true 记录（机读，与模型 turn 同一条时间轴、同一个 session_ms 基准）。 + final_at_ms 是 8021 侧的时钟，和本 session 的 _t0 不同源，只作参考存进 + asr_events.jsonl；对齐一律用本地 session_ms()。 + """ + if not self.enabled or not self._started.is_set(): + return + t_ms = self.session_ms() + try: + if self._transcript is not None: + self._transcript.write(f"[USER @{t_ms}ms] {utterance}\n") + self._transcript.flush() + except Exception: + pass + # 复用 subtitles 的 turn 序列（负数 turn_idx 标识 ASR，避免和模型 turn 撞号） + self._asr_seq = getattr(self, "_asr_seq", 0) + 1 + self.log_subtitle(start_ms=t_ms, end_ms=t_ms, text=utterance, + is_listen=True, turn_idx=-self._asr_seq) + try: + if self.dir is not None: + with (self.dir / "asr_events.jsonl").open("a", encoding="utf-8") as f: + f.write(json.dumps({ + "session_ms": t_ms, + "utterance": utterance, + "harness_final_at_ms": final_at_ms, + "confidence": confidence, + }, ensure_ascii=False) + "\n") + except Exception: + pass + + def log_turn_text(self, turn_idx: int, text: str, is_listen: bool) -> None: + if not self.enabled or self._transcript is None: + return + tag = "LISTEN" if is_listen else "AI" + try: + self._transcript.write(f"[{tag} #{turn_idx} @{self.session_ms()}ms] {text}\n") + self._transcript.flush() + except Exception: + pass + + def log_subtitle(self, start_ms: int, end_ms: int, text: str, + is_listen: bool, turn_idx: int) -> None: + if not self.enabled or self._subtitles is None: + return + try: + self._subtitles.write(json.dumps({ + "start_ms": int(start_ms), + "end_ms": int(end_ms), + "text": text, + "is_listen": bool(is_listen), + "turn_idx": int(turn_idx), + }, ensure_ascii=False) + "\n") + self._subtitles.flush() + except Exception: + pass + + def stop(self) -> None: + if not self.enabled or not self._started.is_set() or self._stopping.is_set(): + return + self._stopping.set() + assert self._t0 is not None and self.dir is not None + total = time.monotonic() - self._t0 + + with self._lock: + user_chunks = list(self._user_chunks) + user_t_start = self._user_t_start + ai_chunks = list(self._ai_chunks) + ai_t_start = self._ai_t_start + n_frames = len(self._frames) + + u_path = self.dir / "live_user.wav" + a_path = self.dir / "live_ai.wav" + u_dur = self._write_track_wav(u_path, user_t_start, user_chunks, + self.user_sr, total) + a_dur = self._write_track_wav(a_path, ai_t_start, ai_chunks, + self.ai_sr, total) + + u_size = u_path.stat().st_size if u_path.exists() else 0 + a_size = a_path.stat().st_size if a_path.exists() else 0 + + # 写 events.jsonl（供 rerun 的 LocalImageSource 读：chunk_idx → img 映射） + try: + import json as _json + with self._lock: + frames_snap = list(self._frames) + ev_path = self.dir / "events.jsonl" + with ev_path.open("w", encoding="utf-8") as ef: + for (ft, frel, fidx) in frames_snap: + ef.write(_json.dumps( + {"kind": "chunk_sent", "idx": int(fidx), + "img": Path(frel).name, "t": round(ft, 3)}, + ensure_ascii=False) + "\n") + LOGGER.info("[LIVE] events.jsonl written: %d frames", len(frames_snap)) + except Exception as e: + LOGGER.warning("[LIVE] write events.jsonl err: %r", e) + + # 写 frames.jsonl（整簇多帧 + 判定，供 funnel rerun 重判 / 无漏斗 rerun 取代表帧） + try: + import json as _json2 + with self._lock: + rounds_snap = list(self._funnel_rounds) + fr_path = self.dir / "frames.jsonl" + with fr_path.open("w", encoding="utf-8") as ff: + for rec in rounds_snap: + ff.write(_json2.dumps(rec, ensure_ascii=False) + "\n") + LOGGER.info("[LIVE] frames.jsonl written: %d rounds", len(rounds_snap)) + except Exception as e: + LOGGER.warning("[LIVE] write frames.jsonl err: %r", e) + + u_nz = self._nonzero_ratio(user_chunks) + a_nz = self._nonzero_ratio(ai_chunks) + + LOGGER.info( + "[LIVE] stopped: wall=%.2fs " + "user(calls=%d audio=%.2fs t_start=%.2fs nonzero=%.1f%%) " + "ai(calls=%d audio=%.2fs t_start=%.2fs nonzero=%.1f%%) " + "frames=%d user_wav=%dB ai_wav=%dB", + total, + self._user_calls, u_dur, + user_t_start if user_t_start is not None else -1, u_nz * 100, + self._ai_calls, a_dur, + ai_t_start if ai_t_start is not None else -1, a_nz * 100, + n_frames, u_size, a_size, + ) + + if self._user_calls == 0: + LOGGER.warning( + "[LIVE] feed_user_raw() was NEVER called — " + "esp32_audio_reader 没把 live_rec 接进来。" + ) + if self._ai_calls == 0: + LOGGER.warning( + "[LIVE] AI callback NEVER fired — attach_to_player 没装上、" + "speaker 未启动、或开了 --no-play。" + ) + + # 关闭评分用文本落盘 + for _fh in (self._transcript, self._subtitles, self._chunks_f): + try: + if _fh is not None: + _fh.close() + except Exception: + pass + self._transcript = None + self._subtitles = None + self._chunks_f = None + + # 结束自动落盘 mp4（不用手动）： + # ① 整簇多帧 + 角落 reject 标注（演示主用，流畅 + 可视化漏斗决策） + # ② best 1fps 版（兼容旧用途） + try: + self.finalize_multiframe_mp4() + except Exception as e: + LOGGER.warning("[LIVE] multiframe mp4 finalize failed: %s", e) + try: + self.finalize_mp4() + except Exception as e: + LOGGER.warning("[LIVE] mp4 finalize failed: %s", e) + + # ------------------------------------------------------------ + # Inputs + # ------------------------------------------------------------ + + def feed_user_raw(self, pcm_f32: np.ndarray,t_override: Optional[float] = None) -> None: + if (not self.enabled or not self._started.is_set() + or self._stopping.is_set() or self._t0 is None): + return + if pcm_f32 is None or pcm_f32.size == 0: + return + if pcm_f32.dtype != np.float32: + pcm_f32 = pcm_f32.astype(np.float32) + # 新增:允许调用者提供精确的音频时间轴 t 戳(rerun 场景); + # 否则按 wall clock 来(ESP32 场景) + if t_override is not None: + t = max(0.0, t_override) + else: + t = max(0.0, time.monotonic() - self._t0) + #t = max(0.0, time.monotonic() - self._t0) + with self._lock: + if self._user_t_start is None: + self._user_t_start = t + rms = float(np.sqrt(np.mean(pcm_f32 ** 2))) + LOGGER.info("[LIVE] first feed_user_raw(): %d samples rms=%.4f t=%.2fs", + pcm_f32.size, rms, t) + self._user_chunks.append((t, pcm_f32.copy())) + self._user_calls += 1 + + def on_frame(self, jpeg_bytes: Optional[bytes], chunk_idx: int) -> None: + if (not self.enabled or not self._started.is_set() + or self._stopping.is_set() or not jpeg_bytes + or self._t0 is None or self.dir is None): + return + t = time.monotonic() - self._t0 + rel = f"images/img_{chunk_idx:05d}.jpg" + # 实际写 jpg 落盘（供 rerun 的 LocalImageSource 读）。 + try: + img_dir = self.dir / "images" + img_dir.mkdir(parents=True, exist_ok=True) + if not self.no_media: + (self.dir / rel).write_bytes(jpeg_bytes) + except Exception as e: + LOGGER.warning("[LIVE] write frame jpg err: %r", e) + return + with self._lock: + self._frames.append((t, rel, int(chunk_idx))) + + def on_funnel_round(self, decision, chunk_idx: int, text: str = "") -> None: + """存漏斗一轮：整簇多帧(筛前原始)落盘 q{NNN}_f{MM}.jpg + frames.jsonl 记一条。 + 统一 record 的核心：一份 record 供三种 rerun—— + 无漏斗 rerun(每轮取代表帧)/ funnel rerun(整簇重判)/ -o rerun(取当时best)。 + """ + if (not self.enabled or not self._started.is_set() + or self._stopping.is_set() or self._t0 is None or self.dir is None): + return + t = time.monotonic() - self._t0 + frames = getattr(decision, "frames", None) + if not frames: + b = getattr(decision, "best", None) or getattr(decision, "best_jpg", None) + frames = [b] if b else [] + # qid 用独立轮次计数（每轮 +1），不能用 latest_frame.sequence—— + # 那个只在 send 时递增，一直 reject（晃动/糊）时不变 → qid 不变 → 整簇 jpg + # 反复用同名 q{同值}_fNN 覆盖旧图，record 存不下、rerun 没图。 + self._round_counter = getattr(self, "_round_counter", -1) + 1 + qid = self._round_counter + frame_names = [] + try: + img_dir = self.dir / "images" + img_dir.mkdir(parents=True, exist_ok=True) + for i, jpg in enumerate(frames): + if not jpg: + continue + name = f"q{qid:05d}_f{i:02d}.jpg" + if not self.no_media: + (img_dir / name).write_bytes(jpg) + frame_names.append(name) + except Exception as e: + LOGGER.warning("[LIVE] on_funnel_round write err: %r", e) + return + rec = { + "qid": qid, + "t": round(t, 3), + "chunk_idx": int(chunk_idx), + "frames": frame_names, + "best_index": getattr(decision, "best_index", None), + "send": bool(getattr(decision, "send", False)), + "reason": getattr(decision, "reason", ""), + "text": text, + } + with self._lock: + self._funnel_rounds.append(rec) + + # ------------------------------------------------------------ + # WAV writing + # ------------------------------------------------------------ + + @staticmethod + def _nonzero_ratio(chunks: list[tuple[float, np.ndarray]]) -> float: + if not chunks: + return 0.0 + total, nz = 0, 0 + for _, pcm in chunks: + total += pcm.size + nz += int(np.sum(np.abs(pcm) > 1e-4)) + return (nz / total) if total > 0 else 0.0 + + @staticmethod + def _write_track_wav( + path: Path, + t_start: Optional[float], # 仅用于日志兼容,不再决定起点 + chunks: list[tuple[float, np.ndarray]], + sr: int, + wall_dur: float, + ) -> float: + """录屏式写盘:每个 chunk 落在它真实的 arrival 时间上,空隙留 0。""" + if not chunks: + n_total = max(int(wall_dur * sr), 1) + track = np.zeros(n_total, dtype=np.float32) + audio_dur = 0.0 + else: + # 计算总长 = max(最后一段尾巴, wall_dur) + end_time = max(t + pcm.size / sr for t, pcm in chunks) + n_total = max(int(max(end_time, wall_dur) * sr), 1) + track = np.zeros(n_total, dtype=np.float32) + audio_dur = 0.0 # 这里改为"有效样本"的累计,不再是 concat 长度 + prev_end_samp = 0 + for t, pcm in chunks: + i = max(0, int(t * sr)) + # 防止下一段的 arrival time 小于上一段结束——重叠时按上一段尾巴顺延 + i = max(i, prev_end_samp) + j = min(i + pcm.size, n_total) + if j > i: + track[i:j] = pcm[: j - i] + prev_end_samp = j + audio_dur += (j - i) / sr + + i16 = np.clip(track * 32768.0, -32768, 32767).astype(np.int16) + with wave.open(str(path), "wb") as w: + w.setnchannels(1); + w.setsampwidth(2); + w.setframerate(sr) + w.writeframes(i16.tobytes()) + return audio_dur + + # ------------------------------------------------------------ + # MP4 finalize + # ------------------------------------------------------------ + + def finalize_mp4(self) -> Optional[Path]: + if self.no_media: + return None + if not self.enabled or self.dir is None: + return None + if shutil.which("ffmpeg") is None: + LOGGER.warning("[LIVE] ffmpeg not found. WAV/帧已保存在 %s。", self.dir) + return None + main_out = None + + + try: + main_out = self._do_finalize_mp4() + except Exception as e: + LOGGER.warning("[LIVE] mp4 finalize failed: %s", e) + # 额外合成一份 user-only,用于 gateway 8006 视频输入测试 + try: + self._do_finalize_useronly_mp4() + except Exception as e: + LOGGER.warning("[LIVE] useronly mp4 finalize failed: %s", e) + return main_out + + def _do_finalize_useronly_mp4(self) -> Optional[Path]: + """额外合成一份只含 user 音轨的 mp4,用于给 gateway 的 8006 视频输入端做 fixture 测试。""" + d = self.dir + assert d is not None + user_wav = d / "live_user.wav" + if not user_wav.exists(): + LOGGER.info("[LIVE] no user wav, skip useronly mp4") + return None + + with wave.open(str(user_wav), "rb") as w: + u_dur = w.getnframes() / w.getframerate() + if u_dur <= 0.5: + LOGGER.info("[LIVE] user too short (%.2fs), skip useronly mp4", u_dur) + return None + + with self._lock: + frames = list(self._frames) + + concat_txt = d / "_useronly_frames.txt" + if frames: + with concat_txt.open("w", encoding="utf-8") as f: + f.write("ffconcat version 1.0\n") + first_t = frames[0][0] + if first_t > 0.05: + f.write(f"file '{frames[0][1]}'\n") + f.write(f"duration {first_t:.3f}\n") + for i, (rt, p, _idx) in enumerate(frames): + f.write(f"file '{p}'\n") + if i + 1 < len(frames): + dur = max(0.04, frames[i + 1][0] - rt) + else: + dur = max(0.5, u_dur - rt) + f.write(f"duration {dur:.3f}\n") + f.write(f"file '{frames[-1][1]}'\n") + + if not frames: + out = d / "live_useronly.m4a" + cmd = [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "warning", + "-i", str(user_wav), + "-c:a", "aac", "-b:a", "192k", + "-ac", "1", + "-t", f"{u_dur:.3f}", + str(out), + ] + else: + out = d / "live_useronly.mp4" + cmd = [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "warning", + "-f", "concat", "-safe", "0", "-i", str(concat_txt), + "-i", str(user_wav), + "-map", "0:v", "-map", "1:a", + "-c:v", "libx264", "-preset", "medium", "-crf", "18", "-pix_fmt", "yuv420p", + "-vsync", "vfr", + "-c:a", "aac", "-b:a", "192k", + "-ac", "1", + "-t", f"{u_dur:.3f}", + str(out), + ] + + LOGGER.info( + "[LIVE] assembling useronly: frames=%d u_dur=%.1fs → %s", + len(frames), u_dur, out.name, + ) + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=600) + if r.returncode != 0: + LOGGER.warning("[LIVE] useronly ffmpeg rc=%d stderr tail:\n%s", + r.returncode, r.stderr[-1200:]) + return None + if out.exists(): + LOGGER.info("[LIVE] ✅ saved %s (%.1f MB, %.1fs)", + out, out.stat().st_size / 1e6, u_dur) + return out + except subprocess.TimeoutExpired: + LOGGER.warning("[LIVE] useronly ffmpeg timeout (>10min)") + return None + finally: + try: + if concat_txt.exists(): + concat_txt.unlink() + except Exception: + pass + def _do_finalize_mp4(self) -> Optional[Path]: + d = self.dir + assert d is not None + user_wav = d / "live_user.wav" + ai_wav = d / "live_ai.wav" + if not (user_wav.exists() and ai_wav.exists()): + LOGGER.info("[LIVE] no wav files, skip mp4") + return None + + with self._lock: + frames = list(self._frames) + + with wave.open(str(user_wav), "rb") as w: + u_dur = w.getnframes() / w.getframerate() + with wave.open(str(ai_wav), "rb") as w: + a_dur = w.getnframes() / w.getframerate() + total_dur = max(u_dur, a_dur) + if total_dur <= 0.5: + LOGGER.info("[LIVE] session too short (%.2fs), skip mp4", total_dur) + return None + + concat_txt = d / "_live_frames.txt" + if frames: + with concat_txt.open("w", encoding="utf-8") as f: + f.write("ffconcat version 1.0\n") + first_t = frames[0][0] + if first_t > 0.05: + f.write(f"file '{frames[0][1]}'\n") + f.write(f"duration {first_t:.3f}\n") + for i, (rt, p, _idx) in enumerate(frames): + f.write(f"file '{p}'\n") + if i + 1 < len(frames): + dur = max(0.04, frames[i + 1][0] - rt) + else: + dur = max(0.5, total_dur - rt) + f.write(f"duration {dur:.3f}\n") + f.write(f"file '{frames[-1][1]}'\n") + + # apad + -t 让短的一轨自动补尾静音对齐到 mp4 总时长 + if not frames: + out = d / "live_session.m4a" + cmd = [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "warning", + "-i", str(user_wav), "-i", str(ai_wav), + "-filter_complex", + "[0:a]aresample=24000,aformat=channel_layouts=mono,apad[u];" + "[1:a]aformat=channel_layouts=mono,apad[a];" + "[u][a]amerge=inputs=2[aout]", + "-map", "[aout]", + "-c:a", "aac", "-b:a", "192k", + "-t", f"{total_dur:.3f}", + str(out), + ] + else: + out = d / "live_session.mp4" + cmd = [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "warning", + "-f", "concat", "-safe", "0", "-i", str(concat_txt), + "-i", str(user_wav), "-i", str(ai_wav), + "-filter_complex", + "[1:a]aresample=24000,aformat=channel_layouts=mono,apad[u];" + "[2:a]aformat=channel_layouts=mono,apad[a];" + "[u][a]amerge=inputs=2[aout]", + "-map", "0:v", "-map", "[aout]", + #"-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", + "-c:v", "libx264", "-preset", "medium", "-crf", "18", "-pix_fmt", "yuv420p", + "-vsync", "vfr", + "-c:a", "aac", "-b:a", "192k", + "-t", f"{total_dur:.3f}", + str(out), + ] + + LOGGER.info( + "[LIVE] assembling: frames=%d total=%.1fs (u=%.1fs a=%.1fs) → %s", + len(frames), total_dur, u_dur, a_dur, out.name, + ) + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=600) + if r.returncode != 0: + LOGGER.warning( + "[LIVE] ffmpeg rc=%d stderr tail:\n%s", + r.returncode, r.stderr[-1200:], + ) + return None + if out.exists(): + LOGGER.info( + "[LIVE] ✅ saved %s (%.1f MB, %.1fs)", + out, out.stat().st_size / 1e6, total_dur, + ) + return out + except subprocess.TimeoutExpired: + LOGGER.warning("[LIVE] ffmpeg timeout (>10min)") + return None + finally: + try: + if concat_txt.exists(): + concat_txt.unlink() + except Exception: + pass + + + # ------------------------------------------------------------ + # 整簇多帧 mp4（演示用）：用每轮整簇多帧拼，比 1fps best 流畅； + # 角落用 ass 字幕标注每轮 send/reject + reason，直接可视化漏斗决策。 + # 对比无漏斗/有漏斗两份 mp4，即可证明"模糊图被拒、不进模型"的正确性。 + # ------------------------------------------------------------ + def _ass_escape(self, s: str) -> str: + return (s or "").replace("\\", "\\\\").replace("{", "(").replace("}", ")") + + def _gen_reject_ass(self, rounds, total_dur: float) -> Optional[Path]: + """生成角落标注字幕：每轮时间段显示 SEND / REJECT:reason。""" + d = self.dir + if d is None or not rounds: + return None + def _fmt(t): + t = max(0.0, t) + h = int(t // 3600); m = int((t % 3600) // 60) + s = t % 60 + return f"{h:d}:{m:02d}:{s:05.2f}" + ass = d / "_reject_notes.ass" + header = ( + "[Script Info]\nScriptType: v4.00+\nPlayResX: 1280\nPlayResY: 720\n\n" + "[V4+ Styles]\n" + "Format: Name, Fontname, Fontsize, PrimaryColour, Bold, Alignment, " + "MarginL, MarginR, MarginV, BorderStyle, Outline, Shadow\n" + # Alignment 9 = 右上角；红=&H000000FF 绿=&H0000FF00 (ASS 是 &HAABBGGRR) + "Style: REJ,Arial,36,&H000000FF,1,9,20,20,20,1,2,0\n" + "Style: SND,Arial,36,&H0000FF00,1,9,20,20,20,1,2,0\n\n" + "[Events]\nFormat: Layer, Start, End, Style, Text\n" + ) + lines = [] + for i, rec in enumerate(rounds): + t0 = float(rec.get("t", i)) + t1 = float(rounds[i + 1].get("t", t0 + 1.0)) if i + 1 < len(rounds) else total_dur + if t1 <= t0: + t1 = t0 + 0.3 + send = bool(rec.get("send")) + reason = self._ass_escape(rec.get("reason", "")) + if send: + style, txt = "SND", "SEND ✓" + else: + style, txt = "REJ", f"REJECT ✗ {reason}" + lines.append(f"Dialogue: 0,{_fmt(t0)},{_fmt(t1)},{style},,{txt}") + try: + ass.write_text(header + "\n".join(lines) + "\n", encoding="utf-8") + return ass + except Exception as e: + LOGGER.warning("[LIVE] write reject ass err: %r", e) + return None + + def finalize_multiframe_mp4(self) -> Optional[Path]: + if self.no_media: + return None + """整簇多帧 mp4 + 角落 reject 标注。演示用，比 best 1fps 流畅。""" + if not self.enabled or self.dir is None: + return None + if shutil.which("ffmpeg") is None: + LOGGER.warning("[LIVE] ⚠ ffmpeg 未安装，无法生成 mp4！" + "装法: conda install -c conda-forge ffmpeg。" + "帧和 wav 已存在 %s，可手动拼。", self.dir) + return None + d = self.dir + user_wav = d / "live_user.wav" + ai_wav = d / "live_ai.wav" + if not (user_wav.exists() and ai_wav.exists()): + return None + with self._lock: + rounds = list(self._funnel_rounds) + if not rounds: + LOGGER.info("[LIVE] no funnel rounds, skip multiframe mp4") + return None + + with wave.open(str(user_wav), "rb") as w: + u_dur = w.getnframes() / w.getframerate() + with wave.open(str(ai_wav), "rb") as w: + a_dur = w.getnframes() / w.getframerate() + total_dur = max(u_dur, a_dur) + if total_dur <= 0.5: + return None + + # 整簇多帧 concat：每轮的 t 到下一轮的 t 之间，均分给该轮的 N 帧（高 fps） + concat_txt = d / "_multiframe.txt" + with concat_txt.open("w", encoding="utf-8") as f: + f.write("ffconcat version 1.0\n") + first_t = float(rounds[0].get("t", 0.0)) + if first_t > 0.05 and rounds[0].get("frames"): + f.write(f"file 'images/{Path(rounds[0]['frames'][0]).name}'\n") + f.write(f"duration {first_t:.3f}\n") + for i, rec in enumerate(rounds): + names = rec.get("frames") or [] + if not names: + continue + t0 = float(rec.get("t", i)) + t1 = float(rounds[i + 1].get("t", t0 + 1.0)) if i + 1 < len(rounds) else total_dur + round_dur = max(0.12, t1 - t0) + per = round_dur / len(names) # 整簇内多帧均分 → 高 fps + for nm in names: + f.write(f"file 'images/{Path(nm).name}'\n") + f.write(f"duration {max(0.03, per):.3f}\n") + # 末帧兜底 + last_names = rounds[-1].get("frames") or [] + if last_names: + f.write(f"file 'images/{Path(last_names[-1]).name}'\n") + + ass = self._gen_reject_ass(rounds, total_dur) + vf = "format=yuv420p" + if ass is not None: + safe = str(ass.name) # 相对 cwd 需在 dir 下运行；用绝对更稳 + abs_ass = str(ass.resolve()).replace("\\", "/").replace(":", "\\:") + vf = f"ass='{abs_ass}',format=yuv420p" + + out = d / "live_multiframe.mp4" + cmd = [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "warning", + "-f", "concat", "-safe", "0", "-i", str(concat_txt), + "-i", str(user_wav), "-i", str(ai_wav), + "-filter_complex", + "[1:a]aresample=24000,aformat=channel_layouts=mono,apad[u];" + "[2:a]aformat=channel_layouts=mono,apad[a];" + "[u][a]amerge=inputs=2[aout]", + "-map", "0:v", "-map", "[aout]", + "-vf", vf, + "-c:v", "libx264", "-preset", "medium", "-crf", "20", "-pix_fmt", "yuv420p", + "-vsync", "vfr", + "-c:a", "aac", "-b:a", "192k", + "-t", f"{total_dur:.3f}", + str(out), + ] + LOGGER.info("[LIVE] assembling multiframe mp4: rounds=%d total=%.1fs → %s", + len(rounds), total_dur, out.name) + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=600) + if r.returncode != 0: + LOGGER.warning("[LIVE] multiframe ffmpeg rc=%d stderr:\n%s", + r.returncode, r.stderr[-1200:]) + return None + if out.exists(): + LOGGER.info("[LIVE] ✅ multiframe mp4 saved %s (%.1f MB)", + out, out.stat().st_size / 1e6) + return out + except subprocess.TimeoutExpired: + LOGGER.warning("[LIVE] multiframe ffmpeg timeout") + return None + finally: + for p in (concat_txt, ass): + try: + if p is not None and p.exists(): + p.unlink() + except Exception: + pass diff --git a/extensions/assistive_harness/phase_b/requirements-phase-b.txt b/extensions/assistive_harness/phase_b/requirements-phase-b.txt index 88c2364..a247f64 100644 --- a/extensions/assistive_harness/phase_b/requirements-phase-b.txt +++ b/extensions/assistive_harness/phase_b/requirements-phase-b.txt @@ -1,3 +1,4 @@ aiohttp>=3.13 Pillow>=10 sounddevice>=0.5 + diff --git a/extensions/assistive_harness/phase_b/rokid_runtime.py b/extensions/assistive_harness/phase_b/rokid_runtime.py index 52885d8..af9f8cc 100644 --- a/extensions/assistive_harness/phase_b/rokid_runtime.py +++ b/extensions/assistive_harness/phase_b/rokid_runtime.py @@ -670,6 +670,16 @@ async def _send_loop(self, ws: aiohttp.ClientWebSocketResponse) -> None: ] last_frame_sequence = frame.sequence last_frame_sent = time.monotonic() + # 诊断：真正送进 gateway 的每个 chunk。audio=xxpps 那个统计的是 + # "推进 audio_queue 的包数"，看不出这里有没有按 1Hz 正常发出去。 + self._sent_chunks = getattr(self, "_sent_chunks", 0) + 1 + _lvl = float(np.abs(audio).mean()) if audio is not None else 0.0 + LOG.debug("[GW-SEND] #%d audio=%.2fs lvl=%.4f frame=%s hold=%s", + self._sent_chunks, + (len(audio) / SAMPLE_RATE_IN) if audio is not None else 0.0, + _lvl, + "frame_base64_list" in payload, + self.gate.speech_hold_active) await self._send_json(payload) async def inject_task(self, text: str) -> bool: @@ -709,6 +719,22 @@ async def _receive_loop(self, ws: aiohttp.ClientWebSocketResponse) -> None: message_type = payload.get("type") if message_type in {"result", "audio_only"}: await self.on_result(self, payload) + elif message_type == "hint_audio": + # 强制措施念提示：独立播放，不进 on_result 状态机 + # （不记 model_turn / 不发 model.state / 不触发 EchoGuard）， + # 避免和 duplex 主对话抢话、纠缠。 + hint_b64 = str(payload.get("audio_data") or "") + if hint_b64: + try: + pcm = np.frombuffer(base64.b64decode(hint_b64), dtype=np.float32) + # 念提示前先解除可能的 block（stop 时 block_and_flush 过）， + # 否则 enqueue 会被丢。 + await self.speaker.resume() + await self.speaker.enqueue(pcm, self.spec.generation) + LOG.info("[HINT] 念提示播放: %r (%d samples)", + payload.get("text"), int(pcm.size)) + except Exception as exc: + LOG.warning("[HINT] 念提示播放失败: %s", exc) elif message_type == "stopped": return elif message_type in {"timeout", "error"}: @@ -1081,6 +1107,18 @@ async def handle_result( self.gate.dropped_old_text += 1 if audio_b64: self.gate.dropped_old_audio += 1 + # 诊断：这条路径原本静默 return，模型的输出被整个丢掉却不留痕迹 + #（表现就是 ai(calls=0) + [MODEL] 只有孤零零一条）。打出丢弃原因。 + self._drop_n = getattr(self, "_drop_n", 0) + 1 + if self._drop_n <= 3 or self._drop_n % 100 == 0: + LOG.warning( + "[GW-DROP] #%d stale=%s (active=%s gen_sess=%s gen_gate=%s) " + "drop_until_listen=%s is_listen=%s text=%r audio=%dB", + self._drop_n, stale, + session is self.active, + session.spec.generation, self.gate.generation, + self.gate.drop_output_until_listen, is_listen, + text[:20], len(audio_b64)) return if is_listen and self.gate.drop_output_until_listen: if not self.gate.speech_hold_active: @@ -1160,6 +1198,10 @@ def __init__( self._send_lock = asyncio.Lock() self._control_tasks: set[asyncio.Task[Any]] = set() + # 外部可挂的只读观察者：收到未被上面分支处理的消息时调用（同步、异常吞掉）。 + # 不改构造签名，赋值即可：client.on_message = fn + on_message = None + async def run(self) -> None: while not self._stop.is_set(): try: @@ -1188,6 +1230,15 @@ async def run(self) -> None: ) self._control_tasks.add(task) task.add_done_callback(self._control_tasks.discard) + # 可选旁路：把其余消息（如 asr.transcript）交给外部观察者。 + # 默认 None，行为与原来完全一致。8021 的 asr.transcript 只投递给 + # "送音频进来的那个 client" 的 outbound 队列（每个 client_id 有 + # 独立 runtime），另开一条连接是收不到的，只能在这里截。 + elif self.on_message is not None: + try: + self.on_message(payload) + except Exception: + pass except asyncio.CancelledError: raise except Exception as exc: diff --git a/extensions/assistive_harness/phase_b/session_recorder.py b/extensions/assistive_harness/phase_b/session_recorder.py new file mode 100644 index 0000000..ce6bcda --- /dev/null +++ b/extensions/assistive_harness/phase_b/session_recorder.py @@ -0,0 +1,90 @@ +"""Session 录制器（对齐 -v 的 SessionRecorder 格式）。 + +沿用 pc_vlm_v5_funnel.SessionRecorder 的落盘结构，让 -o 链路录出的 session +和 -v 完全一致，可用 -v 的分析 / rerun 工具链直接读。 + + sessions// + images/q{NNN}_f{MM}.jpg 每轮一簇帧（整簇存） + queries.jsonl 每行一条，-v 原字段 + 漏斗判定字段 + meta.json + +启动标志：-v 是「ASR 出 query」，-o 里换成「链路启动」——从 start 起持续录， +每轮漏斗决策 append 一条。qid 单调递增（一轮 = 一个 qid）。 + +queries.jsonl 每行（保留 -v 原字段 qid/text/frames/ts，兼容；追加漏斗字段）： + { + "qid": 3, "ts": 1699..., # -v 原字段 + "text": "", # -v 原字段（-o 无 ASR query 文本，留空/可后填） + "frames": ["q003_f00.jpg", ...], # -v 原字段（整簇帧文件名） + "reason": "severe_shake", # 漏斗判定 + "send": false, + "hint": "晃得厉害，请拿稳一下", + "best_index": 1, + "af_triggered": true, "af_ok": true, + "timings": {...} + } +""" +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Optional + + +class SessionRecorder: + def __init__(self, sessions_root: str = "sessions", sid: Optional[str] = None): + sid = sid or time.strftime("%Y%m%d_%H%M%S") + self.dir = Path(sessions_root) / sid + self.images = self.dir / "images" + self.images.mkdir(parents=True, exist_ok=True) + self.jsonl = self.dir / "queries.jsonl" + (self.dir / "meta.json").write_text( + json.dumps({"sid": sid, "source": "esp32_runtime", "created": time.time()}, + ensure_ascii=False, indent=2), + encoding="utf-8") + self._qid = 0 + print(f"[SESSION] recording to {self.dir}") + + def record(self, decision, text: str = "") -> None: + """存一轮漏斗决策：整簇帧落盘 + queries.jsonl append 一条。 + + decision: FunnelDecision（含 frames / best / reason / hint / + best_index / af_triggered / af_ok / timings） + text: 可选，-o 无 ASR query 文本，留空；如上层有 ASR 结果可传入。 + """ + self._qid += 1 + qid = self._qid + frame_names = [] + frames = decision.frames or ([] if decision.best is None else [decision.best]) + for i, jpg in enumerate(frames): + name = f"q{qid:03d}_f{i:02d}.jpg" + try: + (self.images / name).write_bytes(jpg) + frame_names.append(name) + except Exception: + pass + + rec = { + # -v 原字段（兼容 -v 工具链） + "qid": qid, + "ts": time.time(), + "text": text, + "frames": frame_names, + # 漏斗判定字段 + "send": bool(decision.send), + "reason": decision.reason, + "hint": decision.hint, + "best_index": decision.best_index, + "af_triggered": decision.af_triggered, + "af_ok": decision.af_ok, + "timings": decision.timings, + } + try: + with self.jsonl.open("a", encoding="utf-8") as f: + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + except Exception: + pass + + def close(self) -> None: + print(f"[SESSION] recorded {self._qid} queries -> {self.dir}") diff --git a/extensions/assistive_harness/phase_b/templates/live.html b/extensions/assistive_harness/phase_b/templates/live.html new file mode 100644 index 0000000..96099f9 --- /dev/null +++ b/extensions/assistive_harness/phase_b/templates/live.html @@ -0,0 +1,620 @@ + + + + + +AI 眼镜 · 实时演示 + + + + +
+
+
+ 🕶 +
+
AI 眼镜 · 实时演示
+
Real-time multimodal assistant for the visually impaired
+
+
+
+ + 连接中… + + + 待连接 + + + 输入异常 + + 📁 历史 + +
+
+ +
+
+ +
+
📷
+
等待眼镜画面…
+
+
+ ● LIVE · 眼镜第一视角 +
+
+ 📡 前方 -- m + 🧭 朝向 --° +
+
+ 🎤 +
+ -- dB +
+
+
+
+ +
+
+ 💬 对话 + 0 轮 +
+
+
开始说话以与 AI 眼镜互动…
+
+
+
+ +
+
+ AI 静默中 +
+
+
+ + +
+
+
+ +
⚡ 用户打断了 AI
+ + + + + +
+
+

停止并保存?

+

将立即结束本次会话,并把视频 / 音频 / 字幕打包写盘。确认后无法恢复。

+
+ + +
+
+
+ + + + \ No newline at end of file diff --git a/extensions/assistive_harness/phase_b/templates/replay.html b/extensions/assistive_harness/phase_b/templates/replay.html new file mode 100644 index 0000000..458d238 --- /dev/null +++ b/extensions/assistive_harness/phase_b/templates/replay.html @@ -0,0 +1,566 @@ + + + + + + AI 眼镜 · 回放 + + + + +
+
+
+ 🎬 +
+
AI 眼镜 · 会话回放
+
—
+
+
+
+ ·探测中… + 0 轮 + ⬅ 所有会话 + 🔴 实时 +
+
+ +
+
+
+
⏳
+
加载中…
+
+
+ +
+
+ 💬 字幕时间线 + 点击跳转 +
+
+
暂无字幕
+
+
+
+ +
+ +
+
+
+ + + + \ No newline at end of file diff --git a/extensions/assistive_harness/phase_b/templates/replay_index.html b/extensions/assistive_harness/phase_b/templates/replay_index.html new file mode 100644 index 0000000..efcc60a --- /dev/null +++ b/extensions/assistive_harness/phase_b/templates/replay_index.html @@ -0,0 +1,321 @@ + + + + + + AI 眼镜 · 回访 + + + + +
+
+
+ 🕶 +
+
AI 眼镜 · 会话回访
+
Session replay & offline inspection
+
+
+
+ — 条会话 + ⬅ 返回实时 +
+
+ +
+
+
+ 点击行进入回放 · 所有会话按时间倒序 +
+
+
Loading…
+
+
+
+
+ + + + \ No newline at end of file diff --git a/extensions/assistive_harness/phase_b/timing_probe.py b/extensions/assistive_harness/phase_b/timing_probe.py new file mode 100644 index 0000000..6d921c4 --- /dev/null +++ b/extensions/assistive_harness/phase_b/timing_probe.py @@ -0,0 +1,163 @@ +"""时序探针（纯旁路，不改链路行为）。 + +只做一件事：把 ESP32 -o 链路里"取图 / 音频 / 相位"的时间点写进一个 CSV， +供事后分析漏斗该插在哪、跨几个 chunk、音图怎么抢。 + +不接对焦、不接漏斗判定 —— 这是纯时序探针。 + +列名尽量沿用 -v 的 latency CSV 风格（grab_ms / n_frames 等）， +额外加 -o 特有的 chunk / 音频相位列，方便和 -v 数据对比、也让分析工具链复用。 + +用法（在 esp32_runtime 里）： + from .timing_probe import TimingProbe + probe = TimingProbe(enabled=args.timing_probe, path=args.timing_csv, chunk_ms=1000) + # 取图： + probe.mark_grab(seq, grab_ms, jpeg_bytes, since_last_ms) + # 音频（每包调，内部按秒聚合）： + probe.mark_audio(seq, n_samples, drops, rms) + # 关闭：probe.close() +""" +from __future__ import annotations + +import csv +import time +from pathlib import Path +from typing import Optional + + +class TimingProbe: + def __init__(self, enabled: bool = True, path: Optional[str] = None, + chunk_ms: int = 1000): + self.enabled = enabled + self.chunk_ms = chunk_ms + self._t0 = time.monotonic() # 进程起点，用于算 rel_ms / chunk_id + self._f = None + self._w = None + + # ── 音频活动/静默相位判定（用于给每帧图标 phase）── + # 简单能量门限：rms > 阈值算"活动"，否则"静默"。阈值可调。 + self._voice_rms_gate = 0.01 + self._last_audio_active_rel_ms = -1e9 # 最近一次音频活动的相对时刻 + self._audio_active = False + + # ── 音频按秒聚合缓存 ── + self._agg_bucket = -1 # 当前聚合到第几秒 + self._agg_pkts = 0 + self._agg_seqgap = 0 # 本秒内 seq 跳变累计（= 真丢包数） + self._agg_rms_sum = 0.0 + self._agg_active_pkts = 0 + self._last_seq = None # 上一包 seq，用于算 gap + + if self.enabled: + p = Path(path or f"timing_{time.strftime('%Y%m%d_%H%M%S')}.csv") + self._f = open(p, "w", newline="", encoding="utf-8") + self._w = csv.writer(self._f) + # 统一表头：kind 区分 grab / audio 两类行 + self._w.writerow([ + "kind", # "grab" 或 "audio" + "rel_ms", # 相对进程启动的毫秒（对齐两类事件的时间轴） + "chunk_id", # rel_ms // chunk_ms，看事件落在第几个 chunk + "seq", + # grab 专用 + "grab_ms", # 本次取图/一轮漏斗耗时 + "jpeg_bytes", + "since_last_grab_ms", + "audio_phase", # 取这帧图时音频是 active / silent（相位） + "ms_since_voice",# 距最近一次音频活动多久（找静默缝） + # grab 专用 —— 漏斗字段 + "reason", # 漏斗决策：send/severe_shake/unstable/need_focus/orient... + "judge_ms", # run_funnel 判定耗时 + "af_ms", # 对焦触发+settle+重抓耗时（0=没触发对焦） + "n_frames", # 本轮抓帧数 + # audio 专用（按秒聚合） + "audio_pps", # 这一秒的包数 + "audio_seqgap", # 这一秒 seq 跳变累计 = 真丢包数（不是固件 reserved） + "audio_rms", # 这一秒平均 rms + "audio_active_pps", # 这一秒里"活动"包数 + ]) + print(f"[TimingProbe] writing {p}") + + def _rel_ms(self) -> float: + return (time.monotonic() - self._t0) * 1000.0 + + # ── 取图事件 ── + def mark_grab(self, seq: int, grab_ms: float, jpeg_bytes: int, + since_last_ms: float, reason: str = "", judge_ms=None, + af_ms=None, n_frames=None) -> None: + if not self.enabled: + return + rel = self._rel_ms() + phase = "active" if self._audio_active else "silent" + ms_since_voice = rel - self._last_audio_active_rel_ms + if ms_since_voice > 1e8: + ms_since_voice = -1 # 还没出现过音频活动 + self._w.writerow([ + "grab", round(rel, 1), int(rel // self.chunk_ms), seq, + round(grab_ms, 1), jpeg_bytes, round(since_last_ms, 1), + phase, round(ms_since_voice, 1), + reason, judge_ms if judge_ms is not None else "", + af_ms if af_ms is not None else "", + n_frames if n_frames is not None else "", + "", "", "", "", # audio 专用列留空 + ]) + self._f.flush() + + # ── 音频事件（每包调，内部按秒聚合写一行）── + # 注意：固件包头第4字段是 reserved(pad)，不是丢包数；真丢包用 seq 跳变推断。 + def mark_audio(self, seq: int, n_samples: int, rms: float) -> None: + if not self.enabled: + return + rel = self._rel_ms() + + # 更新相位状态（供 mark_grab 读） + self._audio_active = rms > self._voice_rms_gate + if self._audio_active: + self._last_audio_active_rel_ms = rel + + # seq 跳变 = 真丢包（PC 视角：应收 seq 连续，缺号即丢） + gap = 0 + if self._last_seq is not None: + d = seq - self._last_seq - 1 + if 0 < d < 10000: # 合理范围，排除重连/回绕 + gap = d + self._last_seq = seq + + # 按秒聚合 + bucket = int(rel // 1000) + if self._agg_bucket == -1: + self._agg_bucket = bucket + if bucket != self._agg_bucket: + self._flush_audio_bucket() + self._agg_bucket = bucket + self._agg_pkts += 1 + self._agg_seqgap += gap + self._agg_rms_sum += rms + if rms > self._voice_rms_gate: + self._agg_active_pkts += 1 + + def _flush_audio_bucket(self) -> None: + if self._agg_pkts == 0: + return + rel = self._agg_bucket * 1000.0 + avg_rms = self._agg_rms_sum / self._agg_pkts + self._w.writerow([ + "audio", round(rel, 1), int(rel // self.chunk_ms), "", + "", "", "", "", "", # grab 基础列留空 + "", "", "", "", # grab 漏斗列留空 + self._agg_pkts, self._agg_seqgap, round(avg_rms, 4), self._agg_active_pkts, + ]) + self._f.flush() + self._agg_pkts = 0 + self._agg_seqgap = 0 + self._agg_rms_sum = 0.0 + self._agg_active_pkts = 0 + + def close(self) -> None: + if not self.enabled: + return + try: + self._flush_audio_bucket() + if self._f: + self._f.close() + except Exception: + pass diff --git a/extensions/assistive_harness/prompts/README.md b/extensions/assistive_harness/prompts/README.md index b818ff4..38fb5ac 100644 --- a/extensions/assistive_harness/prompts/README.md +++ b/extensions/assistive_harness/prompts/README.md @@ -15,10 +15,14 @@ restarting the Harness service. | `describe_scene` | `describe_scene_zh.txt` | enabled | | `obstacle_avoidance` | `obstacle_avoidance_zh.txt` | enabled / experimental | -The find/read/obstacle task bodies were derived from the project's frozen -AAAI_SI C1 prompts. `find_object_zh.txt` additionally contains `{{target}}`, -because the new Session must receive the target extracted from the command -that closed the old Session. No external absolute path is required at runtime. +The find/read/obstacle task bodies were copied from the frozen AAAI_SI C1 +prompts. `find_object_zh.txt` additionally contains `{{target}}`, because the +new Session must receive the target extracted from the command that closed the +old Session. + +Source directory: + +`C:\Users\Lenovo\AI_Glasses_0618\AAAI_SI\submission_release\AAAI27_AISI_Anonymous_Code_Data\prompts` ## Editing an existing prompt diff --git a/extensions/assistive_harness/server.py b/extensions/assistive_harness/server.py index f9255e8..3d3343d 100644 --- a/extensions/assistive_harness/server.py +++ b/extensions/assistive_harness/server.py @@ -26,7 +26,7 @@ from .model_log import ModelTurnAccumulator from .registry import SkillRegistry from .router import RuleIntentRouter -from .schemas import ASREvent, ControlIntent +from .schemas import ASREvent, ControlEvent, ControlIntent from .state_machine import HarnessController from .telemetry import TelemetryWriter @@ -424,6 +424,39 @@ async def _handle_message( model="injected-test-only", device="none", ) + if message_type in ("funnel.stop", "funnel.resume"): + # 漏斗 reject/恢复:程序直接触发,不经 ASR/router 文字匹配。 + # 复用 controller.process + STOP_SPEECH/RESUME_SPEECH,和真人「停一下/恢复对话」 + # 走完全相同的下游(设备端收到 control.intent 执行 stop_speech/resume_speech)。 + # 归口 8021:telemetry 统一记录,标 source=funnel 以区分真人还是漏斗触发。 + is_stop = message_type == "funnel.stop" + intent = ControlIntent.STOP_SPEECH if is_stop else ControlIntent.RESUME_SPEECH + event = ControlEvent( + event_id=runtime.next_event_id(), + intent=intent, + utterance="[%s]" % message_type, + confidence=1.0, + asr_event_id=runtime.next_event_id(), + created_at_ms=now_ms(), + reason=str(message.get("reason") or "funnel"), + ) + decision = runtime.controller.process(event, now_ms=event.created_at_ms) + payload = { + **decision.payload, + "accepted": decision.accepted, + "action": decision.action, + "decision_reason": decision.reason, + "source": "funnel", + "client_id": runtime.client_id, + } + runtime.telemetry.write("control", payload) + print( + "[AssistiveHarness][FUNNEL] " + "type=%s accepted=%s action=%s reason=%s" + % (message_type, decision.accepted, decision.action, event.reason), + flush=True, + ) + return payload if decision.accepted else None if message_type == "model.state": event = dict(message) runtime.telemetry.write("model", event) diff --git a/extensions/assistive_harness/tests/__init__.py b/extensions/assistive_harness/tests/__init__.py index 8b13789..e69de29 100644 --- a/extensions/assistive_harness/tests/__init__.py +++ b/extensions/assistive_harness/tests/__init__.py @@ -1 +0,0 @@ - diff --git a/extensions/assistive_harness/tests/test_cv_yolo.py b/extensions/assistive_harness/tests/test_cv_yolo.py index 62c231a..59476f8 100644 --- a/extensions/assistive_harness/tests/test_cv_yolo.py +++ b/extensions/assistive_harness/tests/test_cv_yolo.py @@ -9,7 +9,13 @@ from extensions.assistive_harness.cv.yolo_onnx import YoloOnnxProvider -MODEL = Path(__file__).resolve().parents[3] / "models" / "yolo26n.onnx" +MODEL = ( + Path(__file__).resolve().parents[4] + / "OmniHarness" + / "mini_omni_harness" + / "models" + / "yolo26n.onnx" +) @unittest.skipUnless(MODEL.is_file(), "local YOLO reference weights are unavailable")