From be0f09009ce6b0d3649e22836b5176d85613b889 Mon Sep 17 00:00:00 2001 From: Aparencia Date: Fri, 31 Jul 2026 22:54:05 +0800 Subject: [PATCH 1/5] =?UTF-8?q?feat(audio):=20=E8=BF=9B=E7=A8=8B=E7=8E=AF?= =?UTF-8?q?=E5=9B=9E=E9=87=87=E9=9B=86=E5=8E=9F=E7=94=9F=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=20Phase=201=20=E5=8F=AF=E8=A1=8C=E6=80=A7=E9=AA=8C=E8=AF=81=20?= =?UTF-8?q?+=20ADR-001?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 端点环回（getDisplayMedia）在系统主音量之后截取全系统混音，导致两类 问题：跨应用杂音混入课堂转写（ASR 幻觉的可能成因之一），以及音量 0/ 设备错配时采到全零。引入 Windows 10 2004+ 进程环回作为互补音频源。 Phase 1 spike 全部硬验收通过（实测数据见 README 与 ADR）： - 杂音隔离：同时刻发声进程 RMS 0.049 / 静默进程 RMS 0.000000 - Chromium 进程树覆盖（网课成败点）：Electron 声源 RMS 0.055， 证明能采到 audio service 子进程的声音 - 系统静音下峰值与正常音量完全一致（连衰减都没有） - 16kHz mono float32 被直接接受，无需实现重采样 - Chromium 顶层窗口归属 browser process，窗口 PID 即进程树根 ADR-001 记录决策：双源互补而非替换——端点环回是设备视角（永不漏采但 脏），进程环回是应用视角（干净但可能漏采）；虚拟声卡方案因需内核驱动 签名与用户安装摩擦被否决。 本模块独立于 client 主构建（未接入 package.json / electron-builder）， 对已发布版本零影响。 --- client/native/process-audio/README.md | 79 ++ client/native/process-audio/binding.gyp | 41 + client/native/process-audio/package-lock.json | 1252 +++++++++++++++++ client/native/process-audio/package.json | 17 + client/native/process-audio/src/addon.cc | 116 ++ .../process-audio/src/loopback_capture.cc | 276 ++++ .../process-audio/src/loopback_capture.h | 59 + .../native/process-audio/src/window_finder.cc | 128 ++ .../native/process-audio/src/window_finder.h | 35 + .../test/chromium-source/index.html | 11 + .../test/chromium-source/main.js | 70 + .../test/chromium-source/package.json | 7 + .../process-audio/test/spike-capture.mjs | 75 + .../process-audio/test/spike-chromium.mjs | 97 ++ .../process-audio/test/spike-isolation.mjs | 56 + .../native/process-audio/test/spike-muted.mjs | 76 + .../process-audio/test/spike-windows.mjs | 37 + .../ADR-001-audio-capture-process-loopback.md | 139 ++ docs/adr/README.md | 22 + 19 files changed, 2593 insertions(+) create mode 100644 client/native/process-audio/README.md create mode 100644 client/native/process-audio/binding.gyp create mode 100644 client/native/process-audio/package-lock.json create mode 100644 client/native/process-audio/package.json create mode 100644 client/native/process-audio/src/addon.cc create mode 100644 client/native/process-audio/src/loopback_capture.cc create mode 100644 client/native/process-audio/src/loopback_capture.h create mode 100644 client/native/process-audio/src/window_finder.cc create mode 100644 client/native/process-audio/src/window_finder.h create mode 100644 client/native/process-audio/test/chromium-source/index.html create mode 100644 client/native/process-audio/test/chromium-source/main.js create mode 100644 client/native/process-audio/test/chromium-source/package.json create mode 100644 client/native/process-audio/test/spike-capture.mjs create mode 100644 client/native/process-audio/test/spike-chromium.mjs create mode 100644 client/native/process-audio/test/spike-isolation.mjs create mode 100644 client/native/process-audio/test/spike-muted.mjs create mode 100644 client/native/process-audio/test/spike-windows.mjs create mode 100644 docs/adr/ADR-001-audio-capture-process-loopback.md create mode 100644 docs/adr/README.md diff --git a/client/native/process-audio/README.md b/client/native/process-audio/README.md new file mode 100644 index 00000000..2c6a26ba --- /dev/null +++ b/client/native/process-audio/README.md @@ -0,0 +1,79 @@ +# process-audio — Windows 进程环回音频采集（Phase 1 spike） + +面向课堂助手音频采集的原生模块。当前处于 **Phase 1（可行性验证）已完成**状态， +尚未接入 client 主构建，对已发布版本零影响。 + +## 为什么需要它 + +现有音频采集走 Chromium 的 `getDisplayMedia({audio:'loopback'})`，即 +**端点环回**——在系统主音量之后截取最终混音，两个原理性缺陷： + +| 缺陷 | 后果 | +|---|---| +| 采全系统混音 | QQ/微信提示音、其他视频混入课堂转写（脏数据源头，**主要动因**） | +| 跟随主音量/静音 | 音量 0、每应用音量调低、默认输出设备错配 → 采到全零 | + +**进程环回**（Windows 10 2004+ 的 `AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK`） +在混音之前按进程树截取，两个缺陷同时消除。这也是 OBS 28+「应用程序音频采集」 +的实现路线，无需虚拟声卡驱动、无需管理员权限。 + +## Phase 1 验收结果（全部通过） + +| 验收项 | 结果 | +|---|---| +| 编译链路（VS 2022 + node-gyp，Python 3.14 可用） | ✅ 零错误零警告 | +| 窗口 → PID → 进程树根 | ✅ 可用。**Chromium 顶层窗口归属 browser process**，窗口 PID 直接即树根 | +| **杂音隔离**（主要收益） | ✅ 发声进程 RMS 0.049 / 同时刻静默进程 RMS 0.000000 | +| **Chromium 进程树覆盖**（网课成败点） | ✅ Electron 声源 RMS 0.055，证明能采到 audio service **子进程**的声音 | +| 系统静音下仍可采 | ✅ 静音时峰值 0.299246 与正常音量下**完全一致**（连衰减都没有） | +| 16kHz mono float32 格式协商 | ✅ 被 Windows 直接接受 → **无需实现重采样** | + +## 目录结构 + +``` +src/ + window_finder.{h,cc} 窗口枚举 + PID → 应用根进程回溯 + loopback_capture.{h,cc} WASAPI 进程环回采集 + WAV 落盘 + RMS 统计 + addon.cc N-API 绑定（listAudioWindows / resolveRootPid / captureToWav） +test/ + spike-windows.mjs 验证窗口/进程解析 + spike-capture.mjs 按窗口关键字采集并落盘（人工复核用） + spike-isolation.mjs 杂音隔离对照实验 + spike-muted.mjs 系统静音场景验收（需手动静音） + spike-chromium.mjs Chromium 进程树覆盖验证（全自动） + chromium-source/ Electron 等价声源（含播放状态自检） +``` + +## 本地构建与验证 + +```bash +cd client/native/process-audio +npm install +npx node-gyp rebuild + +node test/spike-windows.mjs # 窗口与进程解析 +node test/spike-chromium.mjs # 网课场景（全自动，会短暂响铃） +node test/spike-muted.mjs # 静音场景（按提示手动静音） +``` + +## 已知边界 + +- **仅 Windows 10 2004（build 19041）+**，低于此版本需降级到端点环回 +- **WASAPI 独占模式**播放的应用采不到（罕见） +- **漏采风险**：只采目标进程树，用户换播放器/新开应用需重新绑定——这是 + 与端点环回互补而非替代的根本原因（端点环回"永不漏采但脏"） +- macOS/Linux 无对应 API +- 混音器内**每应用音量**是否衰减尚未单独实测（主音量已确认无影响） + +## 后续阶段 + +Phase 2 集成时需把当前的同步阻塞采集改为**独立采集线程 + +`Napi::ThreadSafeFunction` 流式回调**，产出符合 `AudioChunkData` 契约的 +16kHz mono Float32 块（5s/块），下游 VAD/ASR/幻觉过滤零改动。 + +## 过程教训 + +验证 Chromium 场景时曾连续两次采到 RMS=0，一度指向"进程环回对 Chromium 无效" +的错误结论。真实原因是 **spike 声源本身没启动**（Electron 以目录方式启动需 +`package.json` 指定 main 入口），与被测对象无关。加入「声源自报播放状态」自检 +后立即定位。**测量类实验必须先自证伪声源/测量链路，再评价被测对象。** diff --git a/client/native/process-audio/binding.gyp b/client/native/process-audio/binding.gyp new file mode 100644 index 00000000..019f65da --- /dev/null +++ b/client/native/process-audio/binding.gyp @@ -0,0 +1,41 @@ +{ + "targets": [ + { + "target_name": "process_audio", + "sources": [ + "src/addon.cc", + "src/window_finder.cc", + "src/loopback_capture.cc" + ], + "include_dirs": [ + "=12" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@npmcli/agent": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/@npmcli/agent/-/agent-3.0.0.tgz", + "integrity": "sha512-S79NdEgDQd/NGCay6TCoVzXSj74skRZIKJcpJjC5lOq34SZzyI6MqtiiWoiVWoVrTcGjNeC4ipbh1VIHlpfF5Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "agent-base": "^7.1.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", + "lru-cache": "^10.0.1", + "socks-proxy-agent": "^8.0.3" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/@npmcli/fs": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/@npmcli/fs/-/fs-4.0.0.tgz", + "integrity": "sha512-/xGlezI6xfGO9NwuJlnwz/K14qD1kCSAGtacBHnGzeAIuJGazcp45KP5NuyARXoKb7cwulAGWVsbeSxdG/cb0Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmmirror.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/abbrev": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/abbrev/-/abbrev-3.0.1.tgz", + "integrity": "sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmmirror.com/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.4", + "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/cacache": { + "version": "19.0.1", + "resolved": "https://registry.npmmirror.com/cacache/-/cacache-19.0.1.tgz", + "integrity": "sha512-hdsUxulXCi5STId78vRVYEtDAjq99ICAUktLTeTYsLoTE6Z8dS0c8pWNCxwdrk9YfJeobDZc2Y186hD/5ZQgFQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "@npmcli/fs": "^4.0.0", + "fs-minipass": "^3.0.0", + "glob": "^10.2.2", + "lru-cache": "^10.0.1", + "minipass": "^7.0.3", + "minipass-collect": "^2.0.1", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "p-map": "^7.0.2", + "ssri": "^12.0.0", + "tar": "^7.4.3", + "unique-filename": "^4.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/cross-spawn/node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/cross-spawn/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmmirror.com/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "dev": true, + "license": "MIT" + }, + "node_modules/exponential-backoff": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/exponential-backoff/-/exponential-backoff-3.1.3.tgz", + "integrity": "sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmmirror.com/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fs-minipass": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-3.0.3.tgz", + "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmmirror.com/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmmirror.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmmirror.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/ip-address": { + "version": "10.4.0", + "resolved": "https://registry.npmmirror.com/ip-address/-/ip-address-10.4.0.tgz", + "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "3.1.5", + "resolved": "https://registry.npmmirror.com/isexe/-/isexe-3.1.5.tgz", + "integrity": "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmmirror.com/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/make-fetch-happen": { + "version": "14.0.3", + "resolved": "https://registry.npmmirror.com/make-fetch-happen/-/make-fetch-happen-14.0.3.tgz", + "integrity": "sha512-QMjGbFTP0blj97EeidG5hk/QhKQ3T4ICckQGLgz38QF7Vgbk6e6FTARN8KhKxyBbWn8R0HU+bnw8aSoFPD4qtQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "@npmcli/agent": "^3.0.0", + "cacache": "^19.0.1", + "http-cache-semantics": "^4.1.1", + "minipass": "^7.0.2", + "minipass-fetch": "^4.0.0", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^1.0.0", + "proc-log": "^5.0.0", + "promise-retry": "^2.0.1", + "ssri": "^12.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minipass-collect": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/minipass-collect/-/minipass-collect-2.0.1.tgz", + "integrity": "sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minipass-fetch": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/minipass-fetch/-/minipass-fetch-4.0.1.tgz", + "integrity": "sha512-j7U11C5HXigVuutxebFadoYBbd7VSdZWggSe64NVdvWNBqGAiXPL2QVCehjmw7lY1oF9gOllYbORh+hiNgfPgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "minipass": "^7.0.3", + "minipass-sized": "^1.0.3", + "minizlib": "^3.0.1" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + }, + "optionalDependencies": { + "encoding": "^0.1.13" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/minipass-flush/-/minipass-flush-1.0.7.tgz", + "integrity": "sha512-TbqTz9cUwWyHS2Dy89P3ocAGUGxKjjLuR9z8w4WUTGAVgEj17/4nhgo2Du56i0Fm3Pm30g4iA8Lcqctc76jCzA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-flush/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-flush/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "license": "ISC" + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "license": "ISC" + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "license": "ISC" + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmmirror.com/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/node-gyp": { + "version": "11.5.0", + "resolved": "https://registry.npmmirror.com/node-gyp/-/node-gyp-11.5.0.tgz", + "integrity": "sha512-ra7Kvlhxn5V9Slyus0ygMa2h+UqExPqUIkfk7Pc8QTLT956JLSy51uWFwHtIYy0vI8cB4BDhc/S03+880My/LQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.0", + "exponential-backoff": "^3.1.1", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^14.0.3", + "nopt": "^8.0.0", + "proc-log": "^5.0.0", + "semver": "^7.3.5", + "tar": "^7.4.3", + "tinyglobby": "^0.2.12", + "which": "^5.0.0" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/nopt": { + "version": "8.1.0", + "resolved": "https://registry.npmmirror.com/nopt/-/nopt-8.1.0.tgz", + "integrity": "sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==", + "dev": true, + "license": "ISC", + "dependencies": { + "abbrev": "^3.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/p-map": { + "version": "7.0.6", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-7.0.6.tgz", + "integrity": "sha512-I4Prw6ivkd6p8PiYR1tXASOAOBzIJwu0TB7fqaX0c/8c3QAehNYmX57EijyGGGBt3c/BIowGwV03RVBtXvHEVg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmmirror.com/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmmirror.com/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/proc-log": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/proc-log/-/proc-log-5.0.0.tgz", + "integrity": "sha512-Azwzvl90HaF0aCz1JrDdXQykFakSSNPaPoiZ9fm5qJIMHioDZEi7OAdRwSm6rSoPtY3Qutnm3L7ogmg3dc+wbQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmmirror.com/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.9", + "resolved": "https://registry.npmmirror.com/socks/-/socks-2.8.9.tgz", + "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==", + "dev": true, + "license": "MIT", + "dependencies": { + "ip-address": "^10.1.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmmirror.com/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ssri": { + "version": "12.0.0", + "resolved": "https://registry.npmmirror.com/ssri/-/ssri-12.0.0.tgz", + "integrity": "sha512-S7iGNosepx9RadX82oimUkvr0Ct7IjJbEbs4mJcTxst8um95J3sDYU1RBEOvdu6oL1Wek2ODI5i4MAw+dZ6cAQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmmirror.com/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmmirror.com/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/unique-filename": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/unique-filename/-/unique-filename-4.0.0.tgz", + "integrity": "sha512-XSnEewXmQ+veP7xX2dS5Q4yZAvO40cBN2MWkJ7D/6sW4Dg6wYBNwM1Vrnz1FhH5AdeLIlUXRI9e28z1YZi71NQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "unique-slug": "^5.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/unique-slug": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/unique-slug/-/unique-slug-5.0.0.tgz", + "integrity": "sha512-9OdaqO5kwqR+1kVgHAhsp5vPNU0hnxRa26rBFNfNgM7M6pNtgzeBn3s/xbyCQL3dcjzOatcef6UUHpB/6MaETg==", + "dev": true, + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/which": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/which/-/which-5.0.0.tgz", + "integrity": "sha512-JEdGzHwwkrbWoGOlIHqQ5gtprKGOenpDHpxE9zVR1bWbOtYRyPPHMe9FaP6x61CmNaTThSkb0DAJte5jD+DmzQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^3.1.1" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmmirror.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + } + } +} diff --git a/client/native/process-audio/package.json b/client/native/process-audio/package.json new file mode 100644 index 00000000..b2012999 --- /dev/null +++ b/client/native/process-audio/package.json @@ -0,0 +1,17 @@ +{ + "name": "@entropydecrease/process-audio", + "version": "0.0.0", + "private": true, + "description": "Windows 进程环回音频采集原生模块（Phase 1 spike，独立于 client 构建）", + "main": "index.js", + "scripts": { + "build": "node-gyp rebuild", + "configure": "node-gyp configure" + }, + "dependencies": { + "node-addon-api": "^8.3.0" + }, + "devDependencies": { + "node-gyp": "^11.0.0" + } +} diff --git a/client/native/process-audio/src/addon.cc b/client/native/process-audio/src/addon.cc new file mode 100644 index 00000000..47662c15 --- /dev/null +++ b/client/native/process-audio/src/addon.cc @@ -0,0 +1,116 @@ +// N-API 绑定入口 +// +// @ai-context: Phase 1 spike 第一步仅暴露窗口/进程解析，用于验证编译链路 +// 与"窗口 → PID → 应用根进程"回溯是否可靠；WASAPI 进程环回采集在验证 +// 通过后追加。 + +#include + +#include "loopback_capture.h" +#include "window_finder.h" + +namespace { + +/** std::wstring → Napi::String（UTF-16 直通，避免中文标题乱码） */ +Napi::String ToNapiString(Napi::Env env, const std::wstring& s) { + return Napi::String::New(env, std::u16string(s.begin(), s.end())); +} + +/** listAudioWindows(): 返回所有可见顶层窗口及其 pid / rootPid */ +Napi::Value ListAudioWindows(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + const auto windows = process_audio::ListAudioWindows(); + + Napi::Array arr = Napi::Array::New(env, windows.size()); + for (size_t i = 0; i < windows.size(); ++i) { + const auto& w = windows[i]; + Napi::Object obj = Napi::Object::New(env); + // hwnd 可能超过 2^32，用字符串传递避免 JS number 精度问题 + obj.Set("hwnd", Napi::String::New(env, std::to_string(w.hwnd))); + obj.Set("pid", Napi::Number::New(env, static_cast(w.pid))); + obj.Set("rootPid", Napi::Number::New(env, static_cast(w.root_pid))); + obj.Set("title", ToNapiString(env, w.title)); + obj.Set("processName", ToNapiString(env, w.process_name)); + obj.Set("rootProcessName", ToNapiString(env, w.root_process_name)); + arr.Set(i, obj); + } + return arr; +} + +/** resolveRootPid(pid): 解析某 PID 的应用根进程 */ +Napi::Value ResolveRootPid(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsNumber()) { + Napi::TypeError::New(env, "resolveRootPid(pid: number) 需要一个数字参数") + .ThrowAsJavaScriptException(); + return env.Undefined(); + } + const uint32_t pid = info[0].As().Uint32Value(); + const uint32_t root = process_audio::ResolveRootPidForPid(pid); + return Napi::Number::New(env, static_cast(root)); +} + +/** + * captureToWav({ pid, durationMs, sampleRate, channels, outPath }): + * 同步采集指定时长并可选落盘，返回含 RMS/peak 的诊断信息。 + * + * spike 阶段故意采用同步造型（会阻塞调用线程），Phase 2 集成时 + * 改为采集线程 + ThreadSafeFunction 流式回调。 + */ +Napi::Value CaptureToWav(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsObject()) { + Napi::TypeError::New(env, "captureToWav(options) 需要一个配置对象") + .ThrowAsJavaScriptException(); + return env.Undefined(); + } + Napi::Object opts = info[0].As(); + + if (!opts.Has("pid")) { + Napi::TypeError::New(env, "options.pid 为必填项").ThrowAsJavaScriptException(); + return env.Undefined(); + } + const uint32_t pid = opts.Get("pid").As().Uint32Value(); + const uint32_t duration_ms = opts.Has("durationMs") + ? opts.Get("durationMs").As().Uint32Value() : 5000; + const uint32_t sample_rate = opts.Has("sampleRate") + ? opts.Get("sampleRate").As().Uint32Value() : 16000; + const uint32_t channels = opts.Has("channels") + ? opts.Get("channels").As().Uint32Value() : 1; + + const auto result = process_audio::CaptureProcessAudio( + pid, duration_ms, sample_rate, channels); + + Napi::Object out = Napi::Object::New(env); + out.Set("ok", Napi::Boolean::New(env, result.ok)); + out.Set("error", Napi::String::New(env, result.error)); + out.Set("sampleRate", Napi::Number::New(env, result.sample_rate)); + out.Set("channels", Napi::Number::New(env, result.channels)); + out.Set("sampleCount", Napi::Number::New(env, + static_cast(result.samples.size()))); + out.Set("packetCount", Napi::Number::New(env, result.packet_count)); + out.Set("silentPacketCount", + Napi::Number::New(env, result.silent_packet_count)); + out.Set("rms", Napi::Number::New(env, result.rms)); + out.Set("peak", Napi::Number::New(env, result.peak)); + + if (result.ok && opts.Has("outPath")) { + const std::string path = opts.Get("outPath").As().Utf8Value(); + const bool written = process_audio::WriteWavFloat32( + path, result.samples, result.sample_rate, result.channels); + out.Set("wavWritten", Napi::Boolean::New(env, written)); + out.Set("outPath", Napi::String::New(env, path)); + } + return out; +} + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports.Set("listAudioWindows", Napi::Function::New(env, ListAudioWindows)); + exports.Set("resolveRootPid", Napi::Function::New(env, ResolveRootPid)); + exports.Set("captureToWav", Napi::Function::New(env, CaptureToWav)); + return exports; +} + +} // namespace + +NODE_API_MODULE(process_audio, Init) diff --git a/client/native/process-audio/src/loopback_capture.cc b/client/native/process-audio/src/loopback_capture.cc new file mode 100644 index 00000000..fccfdcb2 --- /dev/null +++ b/client/native/process-audio/src/loopback_capture.cc @@ -0,0 +1,276 @@ +// WASAPI 进程环回采集（实现） +// +// @ai-context: 三个易踩的坑：①process loopback 不支持 GetMixFormat， +// 必须显式指定 WAVEFORMATEX，被拒时需回退；②必须用 +// PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE，否则采不到 +// Chromium audio service（browser process 的子进程）播放的声音； +// ③目标进程无音频输出时不产生数据包，需靠超时而非"等够包数"退出。 + +#include "loopback_capture.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace process_audio { +namespace { + +using Microsoft::WRL::ComPtr; + +/** 事件驱动采集的缓冲时长（100ns 单位，20ms） */ +constexpr REFERENCE_TIME kBufferDuration = 200000; + +/** ActivateAudioInterfaceAsync 的完成回调（异步激活转同步等待） */ +class ActivationHandler + : public Microsoft::WRL::RuntimeClass< + Microsoft::WRL::RuntimeClassFlags, + Microsoft::WRL::FtmBase, + IActivateAudioInterfaceCompletionHandler> { + public: + HANDLE done_event = nullptr; + HRESULT activate_hr = E_FAIL; + ComPtr client; + + STDMETHODIMP ActivateCompleted( + IActivateAudioInterfaceAsyncOperation* operation) override { + HRESULT inner_hr = S_OK; + ComPtr unknown; + const HRESULT hr = operation->GetActivateResult(&inner_hr, &unknown); + if (SUCCEEDED(hr) && SUCCEEDED(inner_hr) && unknown) { + activate_hr = unknown.As(&client); + } else { + activate_hr = FAILED(hr) ? hr : inner_hr; + } + if (done_event != nullptr) SetEvent(done_event); + return S_OK; + } +}; + +/** 构造 Float32 交错格式描述 */ +WAVEFORMATEX MakeFloatFormat(uint32_t sample_rate, uint32_t channels) { + WAVEFORMATEX wfx = {}; + wfx.wFormatTag = WAVE_FORMAT_IEEE_FLOAT; + wfx.nChannels = static_cast(channels); + wfx.nSamplesPerSec = sample_rate; + wfx.wBitsPerSample = 32; + wfx.nBlockAlign = static_cast(channels * 4); + wfx.nAvgBytesPerSec = sample_rate * wfx.nBlockAlign; + wfx.cbSize = 0; + return wfx; +} + +std::string HrToString(const char* stage, HRESULT hr) { + char buf[128]; + std::snprintf(buf, sizeof(buf), "%s 失败 (hr=0x%08lX)", stage, + static_cast(hr)); + return std::string(buf); +} + +/** 激活目标进程树的 IAudioClient */ +HRESULT ActivateProcessLoopbackClient(uint32_t root_pid, + ComPtr* out_client) { + AUDIOCLIENT_ACTIVATION_PARAMS params = {}; + params.ActivationType = AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK; + params.ProcessLoopbackParams.TargetProcessId = static_cast(root_pid); + // 关键：包含整棵进程树，才能覆盖 Chromium audio service 等发声子进程 + params.ProcessLoopbackParams.ProcessLoopbackMode = + PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE; + + PROPVARIANT activate_params = {}; + activate_params.vt = VT_BLOB; + activate_params.blob.cbSize = sizeof(params); + activate_params.blob.pBlobData = reinterpret_cast(¶ms); + + auto handler = Microsoft::WRL::Make(); + if (!handler) return E_OUTOFMEMORY; + handler->done_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (handler->done_event == nullptr) return HRESULT_FROM_WIN32(GetLastError()); + + ComPtr operation; + HRESULT hr = ActivateAudioInterfaceAsync( + VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, __uuidof(IAudioClient), + &activate_params, handler.Get(), &operation); + + if (SUCCEEDED(hr)) { + // 激活为异步流程，等待 handler 回调（含超时兜底避免永久阻塞） + if (WaitForSingleObject(handler->done_event, 3000) != WAIT_OBJECT_0) { + hr = HRESULT_FROM_WIN32(WAIT_TIMEOUT); + } else { + hr = handler->activate_hr; + if (SUCCEEDED(hr)) *out_client = handler->client; + } + } + CloseHandle(handler->done_event); + handler->done_event = nullptr; + return hr; +} + +} // namespace + +CaptureResult CaptureProcessAudio(uint32_t root_pid, + uint32_t duration_ms, + uint32_t preferred_sample_rate, + uint32_t preferred_channels) { + CaptureResult result; + + const HRESULT co_hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + const bool need_uninit = SUCCEEDED(co_hr); + + ComPtr client; + HRESULT hr = ActivateProcessLoopbackClient(root_pid, &client); + if (FAILED(hr) || !client) { + result.error = HrToString("进程环回激活", hr); + if (need_uninit) CoUninitialize(); + return result; + } + + // process loopback 不支持 GetMixFormat，必须显式指定格式； + // 优先请求下游所需的 16kHz mono，被拒时回退 48kHz stereo 由调用方重采样 + WAVEFORMATEX wfx = MakeFloatFormat(preferred_sample_rate, preferred_channels); + const DWORD stream_flags = + AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_EVENTCALLBACK; + hr = client->Initialize(AUDCLNT_SHAREMODE_SHARED, stream_flags, + kBufferDuration, 0, &wfx, nullptr); + if (FAILED(hr)) { + // 回退格式重试（需重新激活：Initialize 失败后的 client 不可复用） + client.Reset(); + hr = ActivateProcessLoopbackClient(root_pid, &client); + if (SUCCEEDED(hr) && client) { + wfx = MakeFloatFormat(48000, 2); + hr = client->Initialize(AUDCLNT_SHAREMODE_SHARED, stream_flags, + kBufferDuration, 0, &wfx, nullptr); + } + if (FAILED(hr)) { + result.error = HrToString("IAudioClient::Initialize", hr); + if (need_uninit) CoUninitialize(); + return result; + } + } + result.sample_rate = wfx.nSamplesPerSec; + result.channels = wfx.nChannels; + + HANDLE sample_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (sample_event == nullptr) { + result.error = "创建采集事件失败"; + if (need_uninit) CoUninitialize(); + return result; + } + hr = client->SetEventHandle(sample_event); + if (FAILED(hr)) { + result.error = HrToString("SetEventHandle", hr); + CloseHandle(sample_event); + if (need_uninit) CoUninitialize(); + return result; + } + + ComPtr capture; + hr = client->GetService(__uuidof(IAudioCaptureClient), + reinterpret_cast(capture.GetAddressOf())); + if (FAILED(hr)) { + result.error = HrToString("GetService(IAudioCaptureClient)", hr); + CloseHandle(sample_event); + if (need_uninit) CoUninitialize(); + return result; + } + + hr = client->Start(); + if (FAILED(hr)) { + result.error = HrToString("IAudioClient::Start", hr); + CloseHandle(sample_event); + if (need_uninit) CoUninitialize(); + return result; + } + + const DWORD deadline = GetTickCount() + duration_ms; + double square_sum = 0.0; + while (GetTickCount() < deadline) { + const DWORD remain = deadline - GetTickCount(); + // 目标进程静默时不产生事件，故等待上限取剩余时长与 200ms 的较小值， + // 保证到点即退出而非无限等待 + const DWORD wait_ms = remain < 200 ? remain : 200; + WaitForSingleObject(sample_event, wait_ms); + + UINT32 packet_frames = 0; + while (SUCCEEDED(capture->GetNextPacketSize(&packet_frames)) && + packet_frames > 0) { + BYTE* data = nullptr; + UINT32 frames = 0; + DWORD flags = 0; + hr = capture->GetBuffer(&data, &frames, &flags, nullptr, nullptr); + if (FAILED(hr)) break; + + result.packet_count++; + const size_t sample_count = + static_cast(frames) * result.channels; + if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0) { + result.silent_packet_count++; + result.samples.insert(result.samples.end(), sample_count, 0.0f); + } else { + const auto* floats = reinterpret_cast(data); + result.samples.insert(result.samples.end(), floats, + floats + sample_count); + for (size_t i = 0; i < sample_count; ++i) { + const double v = static_cast(floats[i]); + square_sum += v * v; + const double a = std::fabs(v); + if (a > result.peak) result.peak = a; + } + } + capture->ReleaseBuffer(frames); + } + } + + client->Stop(); + CloseHandle(sample_event); + if (need_uninit) CoUninitialize(); + + if (!result.samples.empty()) { + result.rms = std::sqrt(square_sum / static_cast(result.samples.size())); + } + result.ok = true; + return result; +} + +bool WriteWavFloat32(const std::string& path, + const std::vector& samples, + uint32_t sample_rate, + uint32_t channels) { + FILE* fp = nullptr; + if (fopen_s(&fp, path.c_str(), "wb") != 0 || fp == nullptr) return false; + + const uint32_t data_bytes = static_cast(samples.size() * sizeof(float)); + const uint32_t block_align = channels * 4; + const uint32_t byte_rate = sample_rate * block_align; + const uint32_t riff_size = 36 + data_bytes; + const uint16_t format_tag = 3; // WAVE_FORMAT_IEEE_FLOAT + const uint16_t bits = 32; + const uint32_t fmt_size = 16; + const uint16_t ch = static_cast(channels); + + auto put = [fp](const void* p, size_t n) { std::fwrite(p, 1, n, fp); }; + put("RIFF", 4); + put(&riff_size, 4); + put("WAVE", 4); + put("fmt ", 4); + put(&fmt_size, 4); + put(&format_tag, 2); + put(&ch, 2); + put(&sample_rate, 4); + put(&byte_rate, 4); + put(reinterpret_cast(&block_align), 2); + put(&bits, 2); + put("data", 4); + put(&data_bytes, 4); + if (!samples.empty()) put(samples.data(), data_bytes); + + std::fclose(fp); + return true; +} + +} // namespace process_audio diff --git a/client/native/process-audio/src/loopback_capture.h b/client/native/process-audio/src/loopback_capture.h new file mode 100644 index 00000000..e8aa899c --- /dev/null +++ b/client/native/process-audio/src/loopback_capture.h @@ -0,0 +1,59 @@ +// WASAPI 进程环回采集（声明） +// +// @ai-context: Windows 10 2004+ 的 AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK +// 在系统混音之前按进程树截取音频，故不受系统主音量/静音影响，且天然隔离 +// 其他应用声音——这是相对 endpoint loopback（getDisplayMedia）的两项核心优势。 + +#ifndef PROCESS_AUDIO_LOOPBACK_CAPTURE_H_ +#define PROCESS_AUDIO_LOOPBACK_CAPTURE_H_ + +#include +#include +#include + +namespace process_audio { + +/** 一次采集的结果与诊断信息 */ +struct CaptureResult { + bool ok = false; + /** 失败原因（ok=false 时有效） */ + std::string error; + /** 实际生效的采样率与声道数（Windows 可能拒绝请求格式而需回退） */ + uint32_t sample_rate = 0; + uint32_t channels = 0; + /** 采集到的 Float32 交错样本 */ + std::vector samples; + /** 收到的数据包总数 */ + uint32_t packet_count = 0; + /** 其中被标记为 AUDCLNT_BUFFERFLAGS_SILENT 的包数 */ + uint32_t silent_packet_count = 0; + /** 全程 RMS（判断"是否真的采到声音"的客观依据） */ + double rms = 0.0; + /** 样本绝对值峰值 */ + double peak = 0.0; +}; + +/** + * 对指定进程树采集音频。 + * + * @param root_pid 目标进程树根 PID（浏览器场景传 browser process， + * 由 window_finder 解析；采集覆盖其全部子进程， + * 因此能拿到 Chromium audio service 播放的声音） + * @param duration_ms 采集时长 + * @param preferred_sample_rate 优先请求的采样率（失败自动回退 48000） + * @param preferred_channels 优先请求的声道数（失败自动回退 2） + */ +CaptureResult CaptureProcessAudio(uint32_t root_pid, + uint32_t duration_ms, + uint32_t preferred_sample_rate, + uint32_t preferred_channels); + +/** 将 Float32 交错样本写为 IEEE float WAV 文件；失败返回 false */ +bool WriteWavFloat32(const std::string& path, + const std::vector& samples, + uint32_t sample_rate, + uint32_t channels); + +} // namespace process_audio + +#endif // PROCESS_AUDIO_LOOPBACK_CAPTURE_H_ diff --git a/client/native/process-audio/src/window_finder.cc b/client/native/process-audio/src/window_finder.cc new file mode 100644 index 00000000..703235c2 --- /dev/null +++ b/client/native/process-audio/src/window_finder.cc @@ -0,0 +1,128 @@ +// 窗口枚举与目标进程解析 +// +// @ai-context: 进程环回采集必须拿到"能覆盖真实发声进程"的 PID。浏览器 +// （Chrome/Edge）的音频由 browser process 派生的 audio service utility +// 进程播放，而窗口 HWND 归属 renderer 进程——直接对窗口 PID 采集会静默 +// 采到空。故需向上回溯到同名祖先进程（即 browser process），再配合 +// PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE 覆盖整棵进程树。 + +#include "window_finder.h" + +#include +#include +#include + +#include +#include +#include + +namespace process_audio { +namespace { + +/** 取进程可执行文件名（小写，不含路径）；失败返回空串 */ +std::wstring GetProcessImageName(DWORD pid) { + HANDLE h = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); + if (h == nullptr) return L""; + wchar_t buf[MAX_PATH] = {0}; + DWORD size = MAX_PATH; + std::wstring name; + if (QueryFullProcessImageNameW(h, 0, buf, &size) != 0) { + std::wstring full(buf, size); + size_t pos = full.find_last_of(L'\\'); + name = (pos == std::wstring::npos) ? full : full.substr(pos + 1); + std::transform(name.begin(), name.end(), name.begin(), ::towlower); + } + CloseHandle(h); + return name; +} + +/** 构建 pid -> parentPid 映射（一次快照，避免逐进程重复枚举） */ +std::map BuildParentMap() { + std::map parents; + HANDLE snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (snap == INVALID_HANDLE_VALUE) return parents; + PROCESSENTRY32W entry; + entry.dwSize = sizeof(entry); + if (Process32FirstW(snap, &entry)) { + do { + parents[entry.th32ProcessID] = entry.th32ParentProcessID; + } while (Process32NextW(snap, &entry)); + } + CloseHandle(snap); + return parents; +} + +/** + * 向上回溯到同名祖先进程（应用根进程）。 + * + * Chrome/Edge 的 renderer、utility（含 audio service）与 browser process + * 同为 chrome.exe/msedge.exe，且都以 browser process 为祖先。逐级上溯， + * 只要父进程与自身同名就继续，直到父进程名不同——此时当前 PID 即根进程。 + * 单进程应用（如多数播放器）回溯结果就是自身。 + */ +DWORD ResolveRootPid(DWORD pid, const std::map& parents) { + const std::wstring target = GetProcessImageName(pid); + if (target.empty()) return pid; + + DWORD current = pid; + // 防御上限：避免异常的父子环导致死循环 + for (int depth = 0; depth < 16; ++depth) { + auto it = parents.find(current); + if (it == parents.end()) break; + const DWORD parent = it->second; + if (parent == 0 || parent == current) break; + if (GetProcessImageName(parent) != target) break; + current = parent; + } + return current; +} + +struct EnumContext { + std::vector* out; + std::map* parents; +}; + +BOOL CALLBACK EnumProc(HWND hwnd, LPARAM lparam) { + auto* ctx = reinterpret_cast(lparam); + + if (IsWindowVisible(hwnd) == 0) return TRUE; + // 过滤工具窗口与无标题窗口（任务栏不可见的辅助窗口） + if ((GetWindowLongW(hwnd, GWL_EXSTYLE) & WS_EX_TOOLWINDOW) != 0) return TRUE; + const int len = GetWindowTextLengthW(hwnd); + if (len <= 0) return TRUE; + + std::wstring title(static_cast(len) + 1, L'\0'); + GetWindowTextW(hwnd, title.data(), len + 1); + title.resize(static_cast(len)); + + DWORD pid = 0; + GetWindowThreadProcessId(hwnd, &pid); + if (pid == 0) return TRUE; + + WindowInfo info; + info.hwnd = reinterpret_cast(hwnd); + info.pid = pid; + info.root_pid = ResolveRootPid(pid, *ctx->parents); + info.title = title; + info.process_name = GetProcessImageName(pid); + info.root_process_name = GetProcessImageName(info.root_pid); + ctx->out->push_back(std::move(info)); + return TRUE; +} + +} // namespace + +std::vector ListAudioWindows() { + std::vector result; + auto parents = BuildParentMap(); + EnumContext ctx{&result, &parents}; + EnumWindows(EnumProc, reinterpret_cast(&ctx)); + return result; +} + +uint32_t ResolveRootPidForPid(uint32_t pid) { + auto parents = BuildParentMap(); + return ResolveRootPid(static_cast(pid), parents); +} + +} // namespace process_audio diff --git a/client/native/process-audio/src/window_finder.h b/client/native/process-audio/src/window_finder.h new file mode 100644 index 00000000..dce1e559 --- /dev/null +++ b/client/native/process-audio/src/window_finder.h @@ -0,0 +1,35 @@ +// 窗口枚举与目标进程解析（声明） +// +// @ai-context: 见 window_finder.cc 头部说明——root_pid 是进程环回采集 +// 真正应传入的目标，pid（窗口所属进程）在浏览器场景下不发声。 + +#ifndef PROCESS_AUDIO_WINDOW_FINDER_H_ +#define PROCESS_AUDIO_WINDOW_FINDER_H_ + +#include +#include +#include + +namespace process_audio { + +/** 单个可见顶层窗口的信息 */ +struct WindowInfo { + uint64_t hwnd = 0; + /** 窗口所属进程（浏览器场景下为 renderer，不发声） */ + uint32_t pid = 0; + /** 回溯得到的应用根进程（浏览器场景下为 browser process，进程树覆盖发声进程） */ + uint32_t root_pid = 0; + std::wstring title; + std::wstring process_name; + std::wstring root_process_name; +}; + +/** 枚举所有可见且有标题的顶层窗口 */ +std::vector ListAudioWindows(); + +/** 单独解析某个 PID 的应用根进程 */ +uint32_t ResolveRootPidForPid(uint32_t pid); + +} // namespace process_audio + +#endif // PROCESS_AUDIO_WINDOW_FINDER_H_ diff --git a/client/native/process-audio/test/chromium-source/index.html b/client/native/process-audio/test/chromium-source/index.html new file mode 100644 index 00000000..976ac6fa --- /dev/null +++ b/client/native/process-audio/test/chromium-source/index.html @@ -0,0 +1,11 @@ + + + + + spike 声源 + + + + + + diff --git a/client/native/process-audio/test/chromium-source/main.js b/client/native/process-audio/test/chromium-source/main.js new file mode 100644 index 00000000..bb93ff5f --- /dev/null +++ b/client/native/process-audio/test/chromium-source/main.js @@ -0,0 +1,70 @@ +// Chromium 架构声源（Phase 1 spike 验证用） +// +// @ai-context: Electron 与 Chrome/Edge 同为 Chromium 多进程架构——音频由 +// browser process 派生的 audio service utility 进程播放。用它当声源即可 +// 等价验证"进程树模式能否覆盖浏览器的发声子进程"这一网课场景成败点。 +// @ai-context: 必须自报播放状态——否则"采到 0"无法区分是采集失败还是 +// 声源没响（实验设计自证伪优先于怀疑被测对象）。 + +const { app, BrowserWindow } = require('electron'); +const path = require('path'); +const fs = require('fs'); + +// Windows 上 Electron 的 console.log 不会传到父进程 stdout，故状态写文件 +const STATE_FILE = path.join(__dirname, '..', '.source-state.json'); +const writeState = (obj) => { + try { + fs.writeFileSync(STATE_FILE, JSON.stringify({ at: Date.now(), ...obj }), 'utf-8'); + } catch { /* 忽略写入失败 */ } +}; + +// 无用户手势场景下强制允许自动播放（否则 Chromium autoplay 策略会拦截） +app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required'); + +app.whenReady().then(async () => { + writeState({ stage: 'ready' }); + + // show: true 避免后台窗口被 Chromium 节流导致音频暂停 + const win = new BrowserWindow({ + width: 320, + height: 200, + show: true, + webPreferences: { backgroundThrottling: false }, + }); + + win.webContents.on('did-fail-load', (_e, code, desc) => { + writeState({ stage: 'load-failed', code, desc }); + }); + + try { + await win.loadFile(path.join(__dirname, 'index.html')); + writeState({ stage: 'loaded' }); + } catch (err) { + writeState({ stage: 'load-throw', message: String(err) }); + } + + // 每秒自报播放状态，供 spike 脚本判断声源是否有效 + setInterval(async () => { + try { + const state = await win.webContents.executeJavaScript(` + (() => { + const a = document.querySelector('audio'); + if (!a) return { found: false }; + return { + found: true, + paused: a.paused, + currentTime: Number(a.currentTime.toFixed(2)), + readyState: a.readyState, + error: a.error ? a.error.code : null, + muted: a.muted, + volume: a.volume, + src: a.currentSrc, + }; + })() + `); + writeState({ stage: 'playing-check', ...state }); + } catch (err) { + writeState({ stage: 'check-failed', message: String(err) }); + } + }, 1000); +}); diff --git a/client/native/process-audio/test/chromium-source/package.json b/client/native/process-audio/test/chromium-source/package.json new file mode 100644 index 00000000..6fb2e711 --- /dev/null +++ b/client/native/process-audio/test/chromium-source/package.json @@ -0,0 +1,7 @@ +{ + "name": "spike-chromium-source", + "version": "0.0.0", + "private": true, + "description": "Electron 需要 package.json 指定入口才能以目录方式启动（spike 声源）", + "main": "main.js" +} diff --git a/client/native/process-audio/test/spike-capture.mjs b/client/native/process-audio/test/spike-capture.mjs new file mode 100644 index 00000000..c8d7981e --- /dev/null +++ b/client/native/process-audio/test/spike-capture.mjs @@ -0,0 +1,75 @@ +// Phase 1 spike 验证脚本 B：进程环回采集两条硬验收 +// +// 用法：node test/spike-capture.mjs [窗口标题关键字] [采集秒数] +// 例：node test/spike-capture.mjs bilibili 6 +// +// 验收点 1（消除静默失败）：把系统主音量调到 0/静音后重跑，rms 仍应 > 0 +// 验收点 2（杂音隔离，主要收益）：采集浏览器时让微信/QQ 发出提示音， +// 波形与 rms 不应受其影响；对比 endpoint 环回会把提示音一并采入 +// +// 判据用 rms 客观数值，不依赖人耳；同时落盘 WAV 供人工复核。 + +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const require = createRequire(import.meta.url); +const addon = require('../build/Release/process_audio.node'); +const here = path.dirname(fileURLToPath(import.meta.url)); + +const keyword = (process.argv[2] ?? 'edge').toLowerCase(); +const seconds = Number(process.argv[3] ?? 5); + +const windows = addon.listAudioWindows(); +const target = windows.find( + (w) => + w.title.toLowerCase().includes(keyword) || + w.processName.toLowerCase().includes(keyword), +); + +if (!target) { + console.error(`未找到匹配 "${keyword}" 的窗口。当前窗口：`); + for (const w of windows) console.error(` ${w.processName} ${w.title.slice(0, 50)}`); + process.exit(1); +} + +console.log('采集目标：'); +console.log(` 标题 ${target.title}`); +console.log(` 窗口进程 pid=${target.pid} (${target.processName})`); +console.log(` 进程树根 rootPid=${target.rootPid} (${target.rootProcessName})`); +console.log(`\n开始采集 ${seconds}s —— 请确保目标窗口正在播放声音...\n`); + +const outPath = path.join(here, `spike-${target.rootPid}-${Date.now()}.wav`); +const result = addon.captureToWav({ + pid: target.rootPid, + durationMs: seconds * 1000, + sampleRate: 16000, + channels: 1, + outPath, +}); + +if (!result.ok) { + console.error(`采集失败：${result.error}`); + process.exit(1); +} + +const requested = result.sampleRate === 16000 && result.channels === 1; +console.log('采集结果：'); +console.log(` 生效格式 ${result.sampleRate}Hz / ${result.channels}ch ` + + `${requested ? '（16k mono 请求被接受，无需重采样 ✔）' : '（已回退，需在原生侧重采样）'}`); +console.log(` 样本数 ${result.sampleCount}`); +console.log(` 数据包 ${result.packetCount}（其中静音包 ${result.silentPacketCount}）`); +console.log(` RMS ${result.rms.toFixed(6)}`); +console.log(` 峰值 ${result.peak.toFixed(6)}`); +console.log(` WAV ${result.wavWritten ? outPath : '(未写入)'}`); + +// 与 asrFilters 的静音门控阈值对齐，直接判断这段音频能否进入 ASR +const SILENCE_RMS_THRESHOLD = 0.008; +console.log('\n判定：'); +if (result.packetCount === 0) { + console.log(' ✖ 未收到任何数据包 —— 目标进程树可能未播放音频，或进程树未覆盖发声进程'); +} else if (result.rms >= SILENCE_RMS_THRESHOLD) { + console.log(` ✔ 采到有效音频（RMS ${result.rms.toFixed(6)} ≥ 门控阈值 ${SILENCE_RMS_THRESHOLD}）`); +} else { + console.log(` ⚠ 采到数据但能量低于门控阈值（RMS ${result.rms.toFixed(6)} < ${SILENCE_RMS_THRESHOLD}）—— 会被 asrFilters 判为静音`); +} diff --git a/client/native/process-audio/test/spike-chromium.mjs b/client/native/process-audio/test/spike-chromium.mjs new file mode 100644 index 00000000..605b7301 --- /dev/null +++ b/client/native/process-audio/test/spike-chromium.mjs @@ -0,0 +1,97 @@ +// Phase 1 spike 验证脚本 E：Chromium 多进程架构下的进程树覆盖 +// +// 用法：node test/spike-chromium.mjs +// +// 这是网课场景的成败验证点：Chrome/Edge/Electron 的音频均由 browser +// process 派生的 audio service utility 子进程播放。本脚本用 Electron +// 当等价声源，验证 PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE +// 能否采到子进程播放的声音。 +// +// 对照实验设计： +// A) 采集 Electron browser process 的进程树 → 应采到声音（进程树覆盖生效） +// B) 采集本 Node 进程自身 → 应为 0（证明不是把系统混音误当结果） + +import { createRequire } from 'node:module'; +import { spawn } from 'node:child_process'; +import { readFileSync, existsSync, rmSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const require = createRequire(import.meta.url); +const addon = require('../build/Release/process_audio.node'); +const here = path.dirname(fileURLToPath(import.meta.url)); + +const SILENCE_RMS_THRESHOLD = 0.008; +// 复用 client 项目已安装的 Electron（本 spike 目录不单独装，避免二进制重复下载） +const clientRequire = createRequire(path.join(here, '..', '..', '..', 'package.json')); +const electronBin = clientRequire('electron'); // 返回可执行文件绝对路径 +const sourceDir = path.join(here, 'chromium-source'); + +console.log('=== Chromium 进程树覆盖验证（网课场景成败点）===\n'); + +const child = spawn(electronBin, [sourceDir], { stdio: 'inherit', windowsHide: false }); +const stop = () => { if (!child.killed) child.kill(); }; +process.on('exit', stop); +process.on('SIGINT', () => { stop(); process.exit(1); }); + +// 等待 Electron 启动、加载页面并开始播放 +await new Promise((r) => setTimeout(r, 5000)); + +// 先确认声源有效（自证伪优先）——否则“采到 0”无法区分失败原因 +const stateFile = path.join(here, '.source-state.json'); +let sourceState = null; +if (existsSync(stateFile)) { + try { + sourceState = JSON.parse(readFileSync(stateFile, 'utf-8')); + } catch { /* 忽略解析失败 */ } +} +console.log(`声源自报状态: ${sourceState ? JSON.stringify(sourceState) : '(无状态文件)'}`); +const sourcePlaying = + sourceState?.found === true && sourceState.paused === false && sourceState.currentTime > 0; +if (!sourcePlaying) { + console.log('⚠ 声源未在播放，本次采集结果不具备判定意义（先修声源再评采集）\n'); +} else { + console.log('✔ 声源确认正在播放\n'); +} + +console.log(`Electron browser process PID=${child.pid}`); +const resolvedRoot = addon.resolveRootPid(child.pid); +console.log(`回溯得到的进程树根 = ${resolvedRoot}（应与上一行相同）\n`); + +function capture(pid, label) { + const r = addon.captureToWav({ pid, durationMs: 4000, sampleRate: 16000, channels: 1 }); + if (!r.ok) { + console.log(` ${label}：采集失败 —— ${r.error}`); + return null; + } + console.log( + ` ${label} (pid=${pid})：RMS=${r.rms.toFixed(6)} 峰值=${r.peak.toFixed(6)} 包数=${r.packetCount}`, + ); + return r; +} + +console.log('各采集 4s：\n'); +const chromium = capture(child.pid, 'Electron 进程树'); +const selfNode = capture(process.pid, '本 Node 进程'); + +stop(); +rmSync(stateFile, { force: true }); + +console.log('\n判定：'); +if (!chromium || !selfNode) { + console.log(' ✖ 采集异常，无法判定'); + process.exit(1); +} +if (!sourcePlaying) { + console.log(' ⚠ 声源无效，不作结论——请先修正声源（见上方自报状态）'); + process.exit(2); +} +const covered = chromium.rms >= SILENCE_RMS_THRESHOLD; +const notLeaking = selfNode.rms < SILENCE_RMS_THRESHOLD; +console.log(` ${covered ? '✔' : '✖'} 进程树覆盖到 audio service 子进程（RMS ${chromium.rms.toFixed(6)}）`); +console.log(` ${notLeaking ? '✔' : '✖'} 无声进程未串入系统混音（RMS ${selfNode.rms.toFixed(6)}）`); +console.log( + covered && notLeaking + ? '\n ✅ 网课场景成立：浏览器/Electron 类多进程应用可被正确采集' + : '\n ❌ 未通过 —— 需改用 addon 自行枚举发声进程或调整进程树模式', +); diff --git a/client/native/process-audio/test/spike-isolation.mjs b/client/native/process-audio/test/spike-isolation.mjs new file mode 100644 index 00000000..408aa402 --- /dev/null +++ b/client/native/process-audio/test/spike-isolation.mjs @@ -0,0 +1,56 @@ +// Phase 1 spike 验证脚本 C：杂音隔离对照实验（主要收益的严格证明） +// +// 用法：node test/spike-isolation.mjs <发声进程PID> <静默进程PID> [秒数] +// +// 设计：同一时刻系统内只有"发声进程"在播放音频。 +// - 采集发声进程 → RMS 应显著 > 0 +// - 采集静默进程 → RMS 应 ≈ 0（若为端点环回，此处会采到发声进程的声音） +// 后者为 0 即证明进程环回实现了声音隔离。 + +import { createRequire } from 'node:module'; + +const require = createRequire(import.meta.url); +const addon = require('../build/Release/process_audio.node'); + +const loudPid = Number(process.argv[2]); +const quietPid = Number(process.argv[3]); +const seconds = Number(process.argv[4] ?? 4); + +if (!loudPid || !quietPid) { + console.error('用法: node test/spike-isolation.mjs <发声进程PID> <静默进程PID> [秒数]'); + process.exit(1); +} + +function capture(pid, label) { + const r = addon.captureToWav({ pid, durationMs: seconds * 1000, sampleRate: 16000, channels: 1 }); + if (!r.ok) { + console.log(` ${label} (pid=${pid})：采集失败 —— ${r.error}`); + return null; + } + console.log( + ` ${label} (pid=${pid})：RMS=${r.rms.toFixed(6)} 峰值=${r.peak.toFixed(6)} ` + + `包数=${r.packetCount} 样本=${r.sampleCount}`, + ); + return r; +} + +console.log(`各采集 ${seconds}s（串行，同一声源持续播放中）：\n`); +const loud = capture(loudPid, '发声进程'); +const quiet = capture(quietPid, '静默进程'); + +const THRESHOLD = 0.008; // 与 asrFilters 静音门控一致 +console.log('\n判定：'); +if (!loud || !quiet) { + console.log(' ✖ 采集异常，无法判定'); + process.exit(1); +} +const loudOk = loud.rms >= THRESHOLD; +const isolated = quiet.rms < THRESHOLD; +console.log(` ${loudOk ? '✔' : '✖'} 发声进程采到有效音频（RMS ${loud.rms.toFixed(6)} ${loudOk ? '≥' : '<'} ${THRESHOLD}）`); +console.log(` ${isolated ? '✔' : '✖'} 静默进程未采到其他应用的声音（RMS ${quiet.rms.toFixed(6)} ${isolated ? '<' : '≥'} ${THRESHOLD}）`); +if (loudOk && isolated) { + const ratio = quiet.rms > 0 ? (loud.rms / quiet.rms).toFixed(0) : '∞'; + console.log(`\n ✅ 杂音隔离成立：两者能量比 ${ratio}× —— 端点环回在此场景下两次都会采到同样的声音`); +} else { + console.log('\n ❌ 隔离未成立或声源无效，需复查'); +} diff --git a/client/native/process-audio/test/spike-muted.mjs b/client/native/process-audio/test/spike-muted.mjs new file mode 100644 index 00000000..5a3fe617 --- /dev/null +++ b/client/native/process-audio/test/spike-muted.mjs @@ -0,0 +1,76 @@ +// Phase 1 spike 验证脚本 D：系统音量为 0 / 静音时是否仍能采集 +// +// 用法（在 client/native/process-audio 目录下）： +// node test/spike-muted.mjs +// +// 脚本会自动：启动循环声源 → 采集 4s → 停止声源，并打印 RMS 判定。 +// 请按提示在采集期间把系统音量调到 0（或点静音）。 +// +// 验收标准：静音状态下 RMS 仍 ≥ 0.008（asrFilters 门控阈值）即通过—— +// 证明进程环回在系统混音之前截取，不受主音量影响。 +// 对照：端点环回（getDisplayMedia）在此场景下必然得到 RMS = 0。 + +import { createRequire } from 'node:module'; +import { spawn } from 'node:child_process'; + +const require = createRequire(import.meta.url); +const addon = require('../build/Release/process_audio.node'); + +const SILENCE_RMS_THRESHOLD = 0.008; +const WAV = 'C:\\Windows\\Media\\Alarm01.wav'; +const CAPTURE_SECONDS = 4; + +console.log('=== 进程环回：系统静音场景验收 ===\n'); +console.log('步骤：脚本将启动一个循环播放的声源进程，随后采集 4 秒。'); +console.log('请在看到「开始采集」后，立刻把系统音量拖到 0 或点击静音。\n'); + +// 启动独立的声源进程（PowerShell + SoundPlayer 循环播放） +const player = spawn( + 'powershell', + [ + '-NoProfile', + '-Command', + `$p = New-Object System.Media.SoundPlayer '${WAV}'; $p.PlayLooping(); Start-Sleep -Seconds 60`, + ], + { windowsHide: true, stdio: 'ignore' }, +); + +const stopPlayer = () => { + if (!player.killed) player.kill(); +}; +process.on('exit', stopPlayer); +process.on('SIGINT', () => { stopPlayer(); process.exit(1); }); + +await new Promise((r) => setTimeout(r, 2500)); +console.log(`声源进程 PID=${player.pid}，已开始播放`); +console.log('\n>>> 开始采集，请现在静音系统 <<<\n'); + +const result = addon.captureToWav({ + pid: player.pid, + durationMs: CAPTURE_SECONDS * 1000, + sampleRate: 16000, + channels: 1, +}); + +stopPlayer(); + +if (!result.ok) { + console.error(`采集失败：${result.error}`); + process.exit(1); +} + +console.log('采集结果：'); +console.log(` 生效格式 ${result.sampleRate}Hz / ${result.channels}ch`); +console.log(` 包数 ${result.packetCount}（静音包 ${result.silentPacketCount}）`); +console.log(` RMS ${result.rms.toFixed(6)}`); +console.log(` 峰值 ${result.peak.toFixed(6)}`); + +console.log('\n判定：'); +if (result.rms >= SILENCE_RMS_THRESHOLD) { + console.log(` ✅ 通过 —— 静音状态下仍采到有效音频（RMS ${result.rms.toFixed(6)} ≥ ${SILENCE_RMS_THRESHOLD}）`); + console.log(' 若采集期间确实处于静音，则证明进程环回不受系统主音量影响。'); +} else { + console.log(` ❌ 未通过 —— RMS ${result.rms.toFixed(6)} < ${SILENCE_RMS_THRESHOLD}`); + console.log(' 可能原因：采集期间未真正静音 / 声源未播放 / 进程环回仍受主音量影响。'); +} +console.log('\n声源进程已停止。'); diff --git a/client/native/process-audio/test/spike-windows.mjs b/client/native/process-audio/test/spike-windows.mjs new file mode 100644 index 00000000..2f297338 --- /dev/null +++ b/client/native/process-audio/test/spike-windows.mjs @@ -0,0 +1,37 @@ +// Phase 1 spike 验证脚本 A：窗口 → PID → 应用根进程回溯 +// +// 用法：node test/spike-windows.mjs +// 验收点：浏览器窗口的 pid 与 rootPid 应不同（窗口属 renderer， +// rootPid 为 browser process），且 rootProcessName 与 processName 同名。 + +import { createRequire } from 'node:module'; + +const require = createRequire(import.meta.url); +const addon = require('../build/Release/process_audio.node'); + +const windows = addon.listAudioWindows(); +console.log(`枚举到 ${windows.length} 个可见顶层窗口\n`); + +/** 关注的多进程应用（音频由子进程播放，必须靠 rootPid + 进程树覆盖） */ +const MULTI_PROCESS = /^(chrome|msedge|firefox|brave|qqbrowser|360se|electron|entropy)/i; + +const interesting = windows.filter((w) => MULTI_PROCESS.test(w.processName)); +if (interesting.length === 0) { + console.log('未发现浏览器类窗口，请打开浏览器后重试'); +} else { + console.log('多进程应用窗口（进程环回的关键场景）：'); + for (const w of interesting) { + const differs = w.pid !== w.rootPid; + console.log( + ` ${differs ? '✔' : '·'} "${w.title.slice(0, 40)}"\n` + + ` 窗口进程 pid=${w.pid} (${w.processName})\n` + + ` 根进程 rootPid=${w.rootPid} (${w.rootProcessName})` + + `${differs ? ' ← 回溯生效' : ' ← 窗口进程即根进程'}`, + ); + } +} + +console.log('\n全部窗口概览（前 15 条）：'); +for (const w of windows.slice(0, 15)) { + console.log(` pid=${String(w.pid).padStart(6)} root=${String(w.rootPid).padStart(6)} ${w.processName.padEnd(20)} ${w.title.slice(0, 36)}`); +} diff --git a/docs/adr/ADR-001-audio-capture-process-loopback.md b/docs/adr/ADR-001-audio-capture-process-loopback.md new file mode 100644 index 00000000..c8a850b3 --- /dev/null +++ b/docs/adr/ADR-001-audio-capture-process-loopback.md @@ -0,0 +1,139 @@ +# ADR-001: 音频采集采用进程环回与端点环回双源互补 + +## 状态 + +已接受（Phase 1 可行性验证已通过，Phase 0/2 待实施） + +## 日期 + +2026-07-31 + +## 背景 + +课堂助手的音频采集当前使用 Chromium 的 `getDisplayMedia({audio:'loopback'})`， +即 **端点环回**（endpoint loopback）——在系统主音量之后截取音频设备的最终混音。 +内测暴露出两类问题： + +1. **脏数据（主要动因）**：采集的是全系统混音，QQ/微信提示音、其他标签页视频、 + 系统音效全部混入课堂转写。内测出现 ASR 幻觉输出脏话（见知识卡片 + `2026-07-classroom-capture-asr-hallucination-json-leak.md`），混入声音是可能成因之一。 +2. **静默失败**：主音量为 0、混音器内单独调低应用音量、声音输出到非默认设备 + （HDMI/蓝牙）时，采集结果为全零 PCM。现有 `useAudioRecovery` 的两条恢复逻辑 + （静音诊断、设备变更重启）都是在为此打补丁。 + +约束条件： + +- 产品为 Windows-only 桌面应用（`electron-builder.yml` 仅定义 win target） +- 面向学生用户，**不可要求安装驱动或授予管理员权限** +- 项目约定 AI/采集能力必须支持降级（本地优先原则） +- 后续规划「线下课堂助手」，其音频来自麦克风，与系统音频链路无关 + +## 决策 + +我们将在采集层引入 `AudioSourceProvider` 抽象，使**进程环回**与**端点环回** +作为两个一等公民音频源并存，按场景智能选源，而非用前者替换后者： + +- **进程环回**（新增）：Windows 10 2004+ 的 + `AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK`，经 C++ N-API 原生模块 + （`client/native/process-audio/`）实现，在混音前按**进程树**截取 +- **端点环回**（保留）：现有 Chromium 路径，作为不支持场景的降级与 + "需要跨应用采集"场景的首选 +- **麦克风**（预留）：线下课堂场景，不进入上述选源链 + +选源策略：用户已选定具体窗口 → 进程环回；选"整个屏幕"或未指定 → 端点环回； +进程环回启动失败或 15s 无数据 → 自动降级端点环回。设置页提供 +`自动 / 强制进程环回 / 强制端点环回` 覆盖项。 + +## 备选方案 + +### 方案 A：维持端点环回现状 + 体验补丁 +- 优点：零开发成本；不引入原生代码与编译链 +- 缺点：脏数据与静默失败**原理上无法解决**，只能靠提示缓解 +- 适用场景：若进程环回验证失败，这是唯一退路 + +### 方案 B：虚拟声卡驱动（VB-Cable 类） +- 优点：兼容所有 Windows 版本；可绕过主音量 +- 缺点：需安装**内核驱动**——签名成本、杀软误报、安装摩擦、信任成本； + 对学生用户群体不可接受 +- 适用场景：专业音频工作流（VoiceMeeter 等），非本产品 + +### 方案 C：进程环回替换端点环回（单源） +- 优点：实现路径单一，维护面小 +- 缺点：进程环回**只采目标进程树**，用户换播放器/新开应用即漏采； + 且 Win10 2004 以下无法运行 +- 适用场景：能强约束用户只用一个播放器的场景，不符合真实使用 + +### 方案 D（选定）：双源互补 + 智能选源 +- 优点：兼得"干净"（进程环回）与"不漏采"（端点环回）；老系统自然降级 +- 缺点：需维护抽象层与两套实现，选源策略本身成为需要解释的复杂度 +- 适用场景：本产品——用户环境不可控，两种视角都有不可替代性 + +## 选择理由 + +关键洞察：两种环回是**不同视角而非优劣关系**——端点环回是"设备视角" +（响什么采什么，永不漏采但脏），进程环回是"应用视角"（干净但可能漏采）。 +OBS 28+ 同时提供两个音频源，正是因为它们互补。若强行二选一，无论选哪个 +都会在真实使用中产生新的失败模式。 + +否决虚拟声卡（方案 B）的决定性因素不是技术能力而是**用户信任成本**：面向 +学生的轻量学习工具要求用户安装内核驱动，转化损失远大于功能收益。 + +Phase 1 spike 已用实测数据验证进程环回的核心收益成立（见下方合规性验证）， +其中"能采到 Chromium audio service 子进程声音"是网课场景的成败点。 + +## 影响 + +### 正面影响 +- 消除课堂转写中的跨应用杂音（主要收益，直接改善 ASR 质量） +- 消除主音量 0 / 每应用音量 / 默认设备错配三类静默失败 +- 与输出设备解耦，`useAudioRecovery` 的 devicechange 重启逻辑在进程源下可退化 +- `AudioSourceProvider` 抽象同时为线下课堂的麦克风源铺路 +- 实测确认 16kHz mono 格式被 Windows 直接接受，**无需实现重采样** + +### 负面影响 / 代价 +- 引入 C++ 原生模块：本地与 CI 都需 MSVC 编译环节，构建时间增加 +- 采集侧从"渲染进程主导"变为"主进程主导"（进程源），需重构编排逻辑 +- 选源策略与降级链本身是需要向用户解释、且需在会话元数据中记录的复杂度 +- Windows-only；未来支持 macOS/Linux 需另写实现 + +### 风险 +- **addon 崩溃可能拖垮主进程** → 缓解：addon 内全异常捕获、失败即降级、 + 不在渲染进程加载 +- **漏采**（用户换播放器/新开应用） → 缓解：复用 `useWindowWatcher` 检测 + 目标窗口消失并提示/重绑；必要时引导切回端点源 +- **CI Windows 编译失败中断发布** → 缓解：Phase 1 即在 CI 跑通编译， + 不留到集成阶段 +- WASAPI 独占模式应用仍采不到 → 已知限制，文档说明 +- 混音器内**每应用音量**是否衰减进程环回尚未单独实测 + +## 合规性验证 + +Phase 1 spike 实测结果（`client/native/process-audio/`，全部通过）： + +| 验收项 | 判据与结果 | +|---|---| +| 杂音隔离（主要收益） | 同一时刻发声进程 RMS=0.049 / 静默进程 RMS=**0.000000** | +| Chromium 进程树覆盖（网课成败点） | Electron 声源 RMS=0.055，证明覆盖到 audio service 子进程 | +| 系统静音下可采 | 静音时峰值 0.299246 与正常音量**完全一致**（无衰减） | +| 格式协商 | 16kHz mono float32 被直接接受 | +| 窗口 → 进程树根 | Chromium 顶层窗口归属 browser process，窗口 PID 即树根 | + +后续阶段的验证要求： + +- 单测覆盖选源策略与降级决策、`useAudioRecovery` 按源分支文案 +- 手动回归必须包含：其他应用放提示音不进转写、系统音量 0、切换输出设备、 + 目标窗口关闭、采集中降级触发、采集线程 30 分钟长跑 +- 不回归项：client 与 ai-gateway 现有测试全绿 + +## 相关决策 + +- 暂无前置 ADR（本项目首个 ADR） + +## 参考 + +- 实施说明与验收数据：`client/native/process-audio/README.md` +- 相关知识卡片：`docs/knowledge/bugs/2026-07-classroom-capture-asr-hallucination-json-leak.md` +- 现有采集链路：`client/electron/audioCapture.ts`、`client/electron/displayMediaHandler.ts`、 + `client/src/features/classroom/hooks/useClassroomAudio.ts` +- 下游契约：`client/src/lib/capture/captureTypes.ts` 的 `AudioChunkData` +- 同路线的成熟实现：OBS 28+「应用程序音频采集」 diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 00000000..5823dc54 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,22 @@ +# 架构决策记录（ADR）索引 + +> 记录重要技术决策的背景、备选方案与权衡。规范见 [ADR 标准](../standards/adr.md)， +> 模板见 [ADR 模板](../templates/adr-template.md)。 + +| 编号 | 标题 | 状态 | 日期 | +|------|------|------|------| +| [ADR-001](./ADR-001-audio-capture-process-loopback.md) | 音频采集采用进程环回与端点环回双源互补 | 已接受 | 2026-07-31 | + +## 编号规则 + +按创建顺序递增，三位数字（ADR-001、ADR-002…），编号一经分配不再复用。 +文件名格式：`ADR-XXX-kebab-case-title.md`。 + +## 状态说明 + +| 状态 | 含义 | +|------|------| +| 提议 | 尚在讨论，未开始实施 | +| 已接受 | 决策生效，代码/配置按此实施 | +| 已废弃 | 不再适用，但保留供历史追溯 | +| 已取代 | 被更新的 ADR 替代，需注明取代者编号 | From d41773d5fba0622fc7845442fc6a5c03167e8ff6 Mon Sep 17 00:00:00 2001 From: Aparencia Date: Fri, 31 Jul 2026 23:01:38 +0800 Subject: [PATCH 2/5] =?UTF-8?q?refactor(audio):=20=E5=BC=95=E5=85=A5=20Aud?= =?UTF-8?q?ioSourceProvider=20=E6=8A=BD=E8=B1=A1=E5=B1=82=EF=BC=88Phase=20?= =?UTF-8?q?0=EF=BC=8C=E9=9B=B6=E8=A1=8C=E4=B8=BA=E5=8F=98=E6=9B=B4?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 ADR-001 的双源互补铺路：audioCapture 从'直接驱动渲染进程采集'改为 '按策略选源 + 降级 + 统一补时间戳'的编排器，采集实现下沉到 Provider。 - src/lib/capture/audioSourceStrategy.ts：源类型与选源策略纯函数 （主/渲染进程共用——useAudioRecovery 后续需按源分支提示文案），12 例单测 - electron/audio/audioSourceProvider.ts：Provider 接口与音频块契约 - electron/audio/endpointLoopbackProvider.ts：现有端点环回逻辑原样迁入 - audioCapture.ts：编排器，公开 API（start/stop/dispose/handleRendererChunk /isCapturing/config）保持不变，mediaCaptureHandlers 零改动 Phase 0 阶段能力探测恒为'进程环回不可用'，选源结果必为端点环回， 行为与重构前完全一致。 --- client/electron/audio/audioSourceProvider.ts | 75 +++++ .../audio/endpointLoopbackProvider.ts | 119 ++++++++ client/electron/audioCapture.ts | 258 +++++++++--------- client/electron/tsconfig.json | 3 +- .../lib/capture/audioSourceStrategy.test.ts | 112 ++++++++ client/src/lib/capture/audioSourceStrategy.ts | 132 +++++++++ 6 files changed, 569 insertions(+), 130 deletions(-) create mode 100644 client/electron/audio/audioSourceProvider.ts create mode 100644 client/electron/audio/endpointLoopbackProvider.ts create mode 100644 client/src/lib/capture/audioSourceStrategy.test.ts create mode 100644 client/src/lib/capture/audioSourceStrategy.ts diff --git a/client/electron/audio/audioSourceProvider.ts b/client/electron/audio/audioSourceProvider.ts new file mode 100644 index 00000000..e05c2392 --- /dev/null +++ b/client/electron/audio/audioSourceProvider.ts @@ -0,0 +1,75 @@ +/** + * 音频源 Provider 接口（主进程侧） + * + * @ai-context: 见 ADR-001。采集层抽象的目的是让"如何拿到 PCM"可替换， + * 而下游（VAD / ASR / 幻觉过滤 / 交叉融合）只依赖 AudioChunk 契约不受影响。 + * 两类 Provider 的数据流方向相反：端点环回由渲染进程采集后回传主进程 + * （需 handleRendererChunk），进程环回在主进程原生采集后直接产出。 + */ + +import type { BrowserWindow } from 'electron'; +import type { AudioSourceKind } from '../../src/lib/capture/audioSourceStrategy.js'; + +/** 音频采集配置 */ +export interface AudioCaptureOptions { + /** 音频块时长(ms)，默认 5000 */ + chunkDurationMs: number; + /** 采样率，默认 16000 */ + sampleRate: number; + /** 声道数，默认 1（单声道） */ + channels: number; +} + +/** 音频块（Provider → 消费者） */ +export interface AudioChunk { + /** PCM Float32 数据 */ + audioBuffer: ArrayBuffer; + sampleRate: number; + channels: number; + durationMs: number; + /** 单调递增时间戳 (ms) */ + timestamp: number; +} + +/** 渲染进程上报的原始音频块（无时间戳，由编排器统一补） */ +export interface RendererAudioChunk { + audioBuffer: ArrayBuffer; + sampleRate: number; + channels: number; + durationMs: number; +} + +/** Provider 启动上下文 */ +export interface AudioProviderStartContext { + /** 绑定的窗口（端点环回需向其下发采集指令） */ + window: BrowserWindow; + /** 用户选定的采集源 ID（desktopCapturer 格式），null 表示自动 */ + sourceId: string | null; + options: AudioCaptureOptions; +} + +/** + * 音频源 Provider。 + * + * 实现约定： + * - start 失败必须抛错，由编排器决定是否降级 + * - stop / dispose 必须幂等 + * - 产出的 chunk 不带时间戳，统一由编排器补（保证跨源时间基准一致） + */ +export interface AudioSourceProvider { + readonly kind: AudioSourceKind; + /** 启动采集；失败抛错 */ + start(ctx: AudioProviderStartContext): Promise; + /** 停止采集（幂等） */ + stop(): void; + /** 释放资源（幂等） */ + dispose(): void; + /** + * 接收渲染进程回传的音频块。 + * 仅渲染进程侧采集的 Provider（端点环回 / 麦克风）实现此方法。 + */ + handleRendererChunk?(data: RendererAudioChunk): void; +} + +/** Provider 产出音频块的回调 */ +export type AudioChunkSink = (chunk: RendererAudioChunk) => void; diff --git a/client/electron/audio/endpointLoopbackProvider.ts b/client/electron/audio/endpointLoopbackProvider.ts new file mode 100644 index 00000000..07fdf8fa --- /dev/null +++ b/client/electron/audio/endpointLoopbackProvider.ts @@ -0,0 +1,119 @@ +/** + * 端点环回 Provider（Chromium getDisplayMedia 路径） + * + * @ai-context: 从 audioCapture.ts 原样迁入（Phase 0 纯重构，行为不变）。 + * Windows 下系统音频只能在渲染进程取到：主进程登记期望源 → 通知渲染进程 + * 调 getDisplayMedia（由 displayMediaHandler 附加 audio:'loopback'）→ + * 渲染进程切片后经 IPC 回传，故本 Provider 实现 handleRendererChunk。 + * @ai-context: 采到的是设备最终混音（含其他应用声音、受主音量影响）， + * 这是它与进程环回的本质差异，见 ADR-001。 + */ + +import { desktopCapturer, type DesktopCapturerSource } from 'electron'; +import { logger } from '../logger.js'; +import { setPreferredDisplaySource } from '../displayMediaHandler.js'; +import type { + AudioChunkSink, + AudioProviderStartContext, + AudioSourceProvider, + RendererAudioChunk, +} from './audioSourceProvider.js'; +import type { AudioSourceKind } from '../../src/lib/capture/audioSourceStrategy.js'; + +/** 音频源信息 */ +export interface AudioSourceInfo { + id: string; + name: string; +} + +/** + * 列出所有可用的系统音频源 + * + * Electron 中系统音频环回（WASAPI Loopback）通过桌面捕获源实现： + * 任意 screen/window 源均可配合 getDisplayMedia 捕获系统音频， + * 因此这里枚举 screen 类型源作为音频采集候选。 + */ +export async function listAudioSources(): Promise { + const sources: DesktopCapturerSource[] = await desktopCapturer.getSources({ + types: ['screen'], + thumbnailSize: { width: 1, height: 1 }, // 枚举不需要缩略图 + }); + + return sources.map((src) => ({ + id: src.id, + name: `系统音频 - ${src.name}`, + })); +} + +export class EndpointLoopbackProvider implements AudioSourceProvider { + readonly kind: AudioSourceKind = 'endpoint_loopback'; + + private readonly sink: AudioChunkSink; + private capturing = false; + private disposed = false; + private boundWindow: AudioProviderStartContext['window'] | null = null; + + constructor(sink: AudioChunkSink) { + this.sink = sink; + } + + async start(ctx: AudioProviderStartContext): Promise { + if (this.capturing || this.disposed) return; + + // 解析音频源：未指定时自动取首个屏幕源 + let resolvedSourceId = ctx.sourceId; + if (!resolvedSourceId) { + const sources = await listAudioSources(); + if (sources.length === 0) { + throw new Error('No audio source available'); + } + resolvedSourceId = sources[0].id; + logger.info(`[EndpointLoopback] 自动选择音频源: ${sources[0].name} (${resolvedSourceId})`); + } + + // 登记期望源：渲染进程随后调用 getDisplayMedia，主进程 handler 据此授权 + // 并附加 audio: 'loopback' 才能拿到真实系统音频（详见 displayMediaHandler.ts） + setPreferredDisplaySource(resolvedSourceId); + + this.capturing = true; + this.boundWindow = ctx.window; + + logger.info( + `[EndpointLoopback] 开始捕获, sourceId=${resolvedSourceId}, ` + + `chunkDurationMs=${ctx.options.chunkDurationMs}, ` + + `sampleRate=${ctx.options.sampleRate}, channels=${ctx.options.channels}`, + ); + + // 通知渲染进程开始音频采集 + if (!ctx.window.isDestroyed()) { + ctx.window.webContents.send('audio_capture_do_start', { + sourceId: resolvedSourceId, + options: ctx.options, + }); + } + } + + stop(): void { + if (!this.capturing) return; + + this.capturing = false; + logger.info('[EndpointLoopback] 停止捕获'); + + // 通知渲染进程停止音频采集 + if (this.boundWindow && !this.boundWindow.isDestroyed()) { + this.boundWindow.webContents.send('audio_capture_do_stop'); + } + this.boundWindow = null; + } + + /** 接收渲染进程回传的音频块，转交编排器补时间戳后分发 */ + handleRendererChunk(data: RendererAudioChunk): void { + if (!this.capturing || this.disposed) return; + this.sink(data); + } + + dispose(): void { + this.stop(); + this.disposed = true; + } +} diff --git a/client/electron/audioCapture.ts b/client/electron/audioCapture.ts index b57a3128..6521447d 100644 --- a/client/electron/audioCapture.ts +++ b/client/electron/audioCapture.ts @@ -1,58 +1,38 @@ /** - * Electron 主进程系统音频捕获模块 + * 音频采集编排器 * - * 架构： - * 1. 主进程使用 desktopCapturer.getSources({ types: ['screen'] }) 枚举屏幕源作为音频环回候选 - * 2. 将音频 sourceId 传递给渲染进程 - * 3. 渲染进程通过 getUserMedia + chromeMediaSource: 'desktop' 获取 MediaStream - * 4. 渲染进程使用 Web Audio API (AudioContext + ScriptProcessor) 切片 - * 5. PCM 数据块通过 IPC 回传主进程，添加单调时间戳后推送给消费者 + * @ai-context: Phase 0 重构（见 ADR-001）——本文件由"直接驱动渲染进程采集" + * 改为「按策略选源 + 降级 + 统一补时间戳」的编排器，具体采集实现下沉到 + * audio/*Provider。对外 API（AudioCapture 的 start/stop/dispose/ + * handleRendererChunk/isCapturing/config）保持不变，故 mediaCaptureHandlers + * 无需改动，行为与重构前一致。 + * @ai-context: 时间戳统一在编排器补，保证不同源切换（含降级）后时间基准连续。 * - * @ai-context: 系统音频捕获：渲染进程 getDisplayMedia 采集、主进程聚合分块。 - * - * TODO(现场课程): 当前仅支持系统音频环回（捕获电脑播放的声音，适配网课场景）。 - * 后续「现场课程」需扩展麦克风输入源：listAudioSources 增加枚举 - * navigator.mediaDevices 的 audioinput 设备，getUserMedia 直接以 deviceId - * 采集麦克风，并与环回源并列供用户选择（或双路混合）。 + * TODO(现场课程): 麦克风 Provider（MicrophoneProvider）待补，届时经 + * selectAudioSource 的 microphone 分支进入，VADMarker 以 + * sourceType:'microphone' 构造启用背景噪声校准。 */ -import { desktopCapturer, DesktopCapturerSource, BrowserWindow } from 'electron'; -import { logger } from './logger'; -import { setPreferredDisplaySource } from './displayMediaHandler'; - -// ================================================================ -// 类型定义 -// ================================================================ - -/** 音频源信息 */ -export interface AudioSourceInfo { - id: string; - name: string; -} - -/** 音频捕获配置 */ -export interface AudioCaptureOptions { - chunkDurationMs: number; // 音频块时长(ms)，默认 5000 - sampleRate: number; // 采样率，默认 16000 - channels: number; // 声道数，默认 1（单声道） -} - -/** 音频块数据（主进程 → 渲染进程） */ -export interface AudioChunk { - audioBuffer: ArrayBuffer; // PCM Float32 数据 - sampleRate: number; - channels: number; - durationMs: number; - timestamp: number; // 单调递增时间戳 (ms) -} - -/** 渲染进程上报的原始音频块 */ -interface RendererAudioChunk { - audioBuffer: ArrayBuffer; - sampleRate: number; - channels: number; - durationMs: number; -} +import type { BrowserWindow } from 'electron'; +import { logger } from './logger.js'; +import { EndpointLoopbackProvider, listAudioSources } from './audio/endpointLoopbackProvider.js'; +import type { + AudioCaptureOptions, + AudioChunk, + AudioSourceProvider, + RendererAudioChunk, +} from './audio/audioSourceProvider.js'; +import { + selectAudioSource, + type AudioSourceDecision, + type AudioSourceKind, + type AudioSourcePreference, +} from '../src/lib/capture/audioSourceStrategy.js'; + +// 保持既有导出路径不变（mediaCaptureHandlers 等调用方无需改动） +export { listAudioSources }; +export type { AudioCaptureOptions, AudioChunk } from './audio/audioSourceProvider.js'; +export type { AudioSourceInfo } from './audio/endpointLoopbackProvider.js'; // ================================================================ // 默认配置 @@ -72,31 +52,16 @@ function monotonicTimestamp(): number { return lastTimestamp; } -// ================================================================ -// 音频源枚举 -// ================================================================ - -/** - * 列出所有可用的系统音频源 - * - * Electron 中系统音频环回（WASAPI Loopback）通过桌面捕获源实现： - * 任意 screen/window 源均可配合 getUserMedia({ chromeMediaSource: 'desktop' }) - * 捕获系统音频。因此这里枚举 screen 类型源作为音频采集候选。 - */ -export async function listAudioSources(): Promise { - const sources: DesktopCapturerSource[] = await desktopCapturer.getSources({ - types: ['screen'], - thumbnailSize: { width: 1, height: 1 }, // 枚举不需要缩略图 - }); - - return sources.map((src) => ({ - id: src.id, - name: `系统音频 - ${src.name}`, - })); +/** 额外的启动参数（选源相关，均可选以保持向后兼容） */ +export interface AudioCaptureStartExtras { + /** 设置页的源偏好，默认 auto */ + preference?: AudioSourcePreference; + /** 线下课堂（麦克风）场景 */ + microphone?: boolean; } // ================================================================ -// 音频捕获管理器 +// 音频采集编排器 // ================================================================ export class AudioCapture { @@ -104,7 +69,11 @@ export class AudioCapture { private readonly onChunk: (chunk: AudioChunk) => void; private capturing = false; private disposed = false; - private boundWin: BrowserWindow | null = null; + + /** 当前活跃的 Provider */ + private provider: AudioSourceProvider | null = null; + /** 本次采集的选源决策（供日志/会话元数据归因） */ + private decision: AudioSourceDecision | null = null; constructor( options: Partial, @@ -124,90 +93,121 @@ export class AudioCapture { return this.options; } + /** 本次生效的音频源类型（未启动时为 null） */ + get activeSourceKind(): AudioSourceKind | null { + return this.provider?.kind ?? null; + } + + /** 本次选源决策（含理由，供会话元数据记录） */ + get sourceDecision(): AudioSourceDecision | null { + return this.decision; + } + /** - * 开始音频捕获 + * 开始音频捕获。 * - * 1. 如果未指定 sourceId，自动选择第一个可用的系统音频源 - * 2. 将期望源登记到 displayMedia handler（渲染进程 getDisplayMedia 时生效） - * 3. 向渲染进程发送启动指令（含 sourceId + 配置） - * 4. 渲染进程负责 getDisplayMedia 与音频切片 + * 按 selectAudioSource 的决策创建 Provider；若首选源启动失败且存在降级目标， + * 自动降级重试一次（降级事实通过 decision.reason 与日志暴露）。 */ - async start(win: BrowserWindow, sourceId?: string): Promise { + async start( + win: BrowserWindow, + sourceId?: string, + extras?: AudioCaptureStartExtras, + ): Promise { if (this.capturing || this.disposed) return; - // 解析音频源 - let resolvedSourceId = sourceId ?? null; - if (!resolvedSourceId) { - const sources = await listAudioSources(); - if (sources.length === 0) { - console.warn('[AudioCapture] 未找到可用的系统音频源'); - throw new Error('No audio source available'); - } - resolvedSourceId = sources[0].id; - logger.info(`[AudioCapture] 自动选择音频源: ${sources[0].name} (${resolvedSourceId})`); - } - - // 登记期望源：渲染进程随后调用 getDisplayMedia，主进程 handler 据此授权 - // 并附加 audio: 'loopback' 才能拿到真实系统音频（详见 displayMediaHandler.ts） - setPreferredDisplaySource(resolvedSourceId); - - this.capturing = true; - this.boundWin = win; + const resolvedSourceId = sourceId ?? null; + this.decision = selectAudioSource({ + // Phase 0：进程环回尚未接入，能力探测恒为不可用（行为与重构前一致） + capabilities: { processLoopbackAvailable: false }, + sourceId: resolvedSourceId, + preference: extras?.preference, + microphone: extras?.microphone, + }); logger.info( - `[AudioCapture] 开始捕获, sourceId=${resolvedSourceId}, ` + - `chunkDurationMs=${this.options.chunkDurationMs}, ` + - `sampleRate=${this.options.sampleRate}, channels=${this.options.channels}`, + `[AudioCapture] 选源: ${this.decision.kind}（${this.decision.reason}）` + + `${this.decision.fallback ? `, 降级目标=${this.decision.fallback}` : ''}`, ); - // 通知渲染进程开始音频采集 - if (!win.isDestroyed()) { - win.webContents.send('audio_capture_do_start', { - sourceId: resolvedSourceId, - options: this.options, - }); + try { + await this.startWithKind(this.decision.kind, win, resolvedSourceId); + } catch (err) { + const fallback = this.decision.fallback; + if (!fallback) throw err; + + const message = err instanceof Error ? err.message : String(err); + logger.warn(`[AudioCapture] ${this.decision.kind} 启动失败（${message}），降级到 ${fallback}`); + this.decision = { + kind: fallback, + reason: `${this.decision.kind} 启动失败后降级：${message}`, + fallback: null, + }; + await this.startWithKind(fallback, win, resolvedSourceId); } - } - /** - * 停止音频捕获 - */ - stop(): void { - if (!this.capturing) return; + this.capturing = true; + } - this.capturing = false; - logger.info('[AudioCapture] 停止捕获'); + /** 按源类型创建并启动 Provider */ + private async startWithKind( + kind: AudioSourceKind, + win: BrowserWindow, + sourceId: string | null, + ): Promise { + this.provider?.dispose(); + this.provider = this.createProvider(kind); + await this.provider.start({ window: win, sourceId, options: this.options }); + } - // 通知渲染进程停止音频采集 - if (this.boundWin && !this.boundWin.isDestroyed()) { - this.boundWin.webContents.send('audio_capture_do_stop'); + /** Provider 工厂 */ + private createProvider(kind: AudioSourceKind): AudioSourceProvider { + const sink = (data: RendererAudioChunk) => this.emitChunk(data); + switch (kind) { + case 'endpoint_loopback': + return new EndpointLoopbackProvider(sink); + case 'process_loopback': + // Phase 2 接入；Phase 0 阶段选源不会产生该分支 + throw new Error('进程环回 Provider 尚未接入'); + case 'microphone': + // TODO(现场课程): MicrophoneProvider 待实现 + throw new Error('麦克风 Provider 尚未实现'); } - this.boundWin = null; } - /** - * 接收来自渲染进程的音频块数据 - * 由 IPC handler 调用，添加单调时间戳后通过回调发出 - */ - handleRendererChunk(data: RendererAudioChunk): void { - if (!this.capturing || this.disposed) return; - - const chunk: AudioChunk = { + /** 统一补时间戳后向消费者分发 */ + private emitChunk(data: RendererAudioChunk): void { + if (this.disposed) return; + this.onChunk({ audioBuffer: data.audioBuffer, sampleRate: data.sampleRate, channels: data.channels, durationMs: data.durationMs, timestamp: monotonicTimestamp(), - }; + }); + } - this.onChunk(chunk); + /** 停止音频捕获 */ + stop(): void { + if (!this.capturing) return; + this.capturing = false; + this.provider?.stop(); } /** - * 销毁实例，释放所有资源 + * 接收来自渲染进程的音频块数据（由 IPC handler 调用）。 + * 仅当前 Provider 为渲染进程侧采集时有效。 */ + handleRendererChunk(data: RendererAudioChunk): void { + if (!this.capturing || this.disposed) return; + this.provider?.handleRendererChunk?.(data); + } + + /** 销毁实例，释放所有资源 */ dispose(): void { this.stop(); + this.provider?.dispose(); + this.provider = null; this.disposed = true; logger.info('[AudioCapture] 已销毁'); } diff --git a/client/electron/tsconfig.json b/client/electron/tsconfig.json index cc86beb8..46e90606 100644 --- a/client/electron/tsconfig.json +++ b/client/electron/tsconfig.json @@ -19,6 +19,7 @@ "include": [ "./**/*.ts", "../src/lib/storage/interfaces.ts", - "../src/types/models.ts" + "../src/types/models.ts", + "../src/lib/capture/audioSourceStrategy.ts" ] } diff --git a/client/src/lib/capture/audioSourceStrategy.test.ts b/client/src/lib/capture/audioSourceStrategy.test.ts new file mode 100644 index 00000000..b02f43b3 --- /dev/null +++ b/client/src/lib/capture/audioSourceStrategy.test.ts @@ -0,0 +1,112 @@ +/** + * @ai-context: 选源策略单测。覆盖 ADR-001 定义的全部决策分支—— + * 麦克风独立链、强制偏好（含不支持时的降级）、auto 下"锁定窗口→进程环回 / + * 整屏→端点环回"的互补取舍。 + */ +import { describe, it, expect } from 'vitest'; +import { + selectAudioSource, + isWindowSource, + describeAudioSource, + type AudioSourceSelectionInput, +} from './audioSourceStrategy'; + +const supported = { processLoopbackAvailable: true }; +const unsupported = { processLoopbackAvailable: false }; + +function decide(overrides: Partial = {}) { + return selectAudioSource({ + capabilities: supported, + sourceId: 'window:12345:0', + ...overrides, + }); +} + +describe('isWindowSource', () => { + it('仅 window: 前缀视为锁定了具体窗口', () => { + expect(isWindowSource('window:12345:0')).toBe(true); + expect(isWindowSource('screen:0:0')).toBe(false); + expect(isWindowSource(null)).toBe(false); + expect(isWindowSource('')).toBe(false); + }); +}); + +describe('selectAudioSource — 麦克风场景', () => { + it('麦克风场景独立成链，不参与环回选源且无降级', () => { + const d = decide({ microphone: true, capabilities: unsupported, sourceId: null }); + expect(d.kind).toBe('microphone'); + expect(d.fallback).toBeNull(); + }); + + it('麦克风优先级高于强制环回偏好', () => { + const d = decide({ microphone: true, preference: 'force_process' }); + expect(d.kind).toBe('microphone'); + }); +}); + +describe('selectAudioSource — 用户强制偏好', () => { + it('强制端点环回时始终用端点环回', () => { + const d = decide({ preference: 'force_endpoint' }); + expect(d.kind).toBe('endpoint_loopback'); + expect(d.fallback).toBeNull(); + }); + + it('强制进程环回且环境支持时用进程环回，并保留端点降级', () => { + const d = decide({ preference: 'force_process', sourceId: 'screen:0:0' }); + expect(d.kind).toBe('process_loopback'); + expect(d.fallback).toBe('endpoint_loopback'); + }); + + it('强制进程环回但环境不支持时降级端点环回，理由说明原因', () => { + const d = decide({ preference: 'force_process', capabilities: unsupported }); + expect(d.kind).toBe('endpoint_loopback'); + expect(d.reason).toContain('不支持'); + }); +}); + +describe('selectAudioSource — auto 策略', () => { + it('锁定具体窗口时选进程环回（隔离杂音），降级为端点环回', () => { + const d = decide(); + expect(d.kind).toBe('process_loopback'); + expect(d.fallback).toBe('endpoint_loopback'); + }); + + it('整屏采集时选端点环回（避免漏采跨应用声音）', () => { + const d = decide({ sourceId: 'screen:0:0' }); + expect(d.kind).toBe('endpoint_loopback'); + }); + + it('未指定源时选端点环回', () => { + const d = decide({ sourceId: null }); + expect(d.kind).toBe('endpoint_loopback'); + }); + + it('环境不支持进程环回时选端点环回，理由标明环境限制', () => { + const d = decide({ capabilities: unsupported }); + expect(d.kind).toBe('endpoint_loopback'); + expect(d.reason).toContain('Windows 10 2004+'); + }); + + it('所有分支都给出非空决策理由（供会话元数据归因）', () => { + const cases: Partial[] = [ + {}, + { sourceId: null }, + { capabilities: unsupported }, + { preference: 'force_endpoint' }, + { preference: 'force_process' }, + { preference: 'force_process', capabilities: unsupported }, + { microphone: true }, + ]; + for (const c of cases) { + expect(decide(c).reason.length).toBeGreaterThan(0); + } + }); +}); + +describe('describeAudioSource', () => { + it('三种源都有用户可读名称', () => { + expect(describeAudioSource('process_loopback')).toContain('目标窗口'); + expect(describeAudioSource('endpoint_loopback')).toContain('全部声音'); + expect(describeAudioSource('microphone')).toBe('麦克风'); + }); +}); diff --git a/client/src/lib/capture/audioSourceStrategy.ts b/client/src/lib/capture/audioSourceStrategy.ts new file mode 100644 index 00000000..94fa37b7 --- /dev/null +++ b/client/src/lib/capture/audioSourceStrategy.ts @@ -0,0 +1,132 @@ +/** + * 音频源类型与选源策略（主进程 / 渲染进程共用） + * + * @ai-context: 见 ADR-001。端点环回是"设备视角"（永不漏采但采全系统混音）， + * 进程环回是"应用视角"（只采目标进程树，干净但可能漏采）——两者互补而非 + * 替代关系，故此处是「选源」而非「降级」策略：默认按用户是否锁定了具体 + * 窗口来选，仅在进程环回不可用/启动失败时才降级。 + * @ai-context: 本文件必须保持纯函数无副作用——主进程（audioCapture 编排器） + * 与渲染进程（useAudioRecovery 按源分支提示文案）共同依赖，且需被单测覆盖。 + */ + +/** 音频采集源类型 */ +export type AudioSourceKind = + /** 进程环回：Win10 2004+ 原生 API，混音前按进程树截取 */ + | 'process_loopback' + /** 端点环回：Chromium getDisplayMedia，截取设备最终混音 */ + | 'endpoint_loopback' + /** 麦克风：线下课堂场景 */ + | 'microphone'; + +/** 用户偏好覆盖（设置页） */ +export type AudioSourcePreference = 'auto' | 'force_process' | 'force_endpoint'; + +/** 运行环境能力探测结果 */ +export interface AudioSourceCapabilities { + /** 进程环回是否可用（Windows 10 2004+ 且原生模块加载成功） */ + processLoopbackAvailable: boolean; +} + +/** 选源输入 */ +export interface AudioSourceSelectionInput { + capabilities: AudioSourceCapabilities; + /** + * 用户选定的采集源 ID（desktopCapturer 格式）。 + * `window:*` 表示锁定了具体窗口；`screen:*` 或 null 表示整屏/未指定。 + */ + sourceId: string | null; + /** 设置页偏好，默认 auto */ + preference?: AudioSourcePreference; + /** 线下课堂（麦克风）场景，不参与环回选源 */ + microphone?: boolean; +} + +/** 选源结果 */ +export interface AudioSourceDecision { + kind: AudioSourceKind; + /** 决策理由，写入会话元数据供内测问题归因 */ + reason: string; + /** 启动失败时的降级目标；null 表示无降级可用 */ + fallback: AudioSourceKind | null; +} + +/** 判断是否锁定了具体窗口（而非整个屏幕） */ +export function isWindowSource(sourceId: string | null): boolean { + return typeof sourceId === 'string' && sourceId.startsWith('window:'); +} + +/** + * 按场景选择音频源。 + * + * 规则优先级： + * 1. 麦克风场景独立成链，不涉及环回 + * 2. 用户强制偏好优先于自动策略（强制进程环回但环境不支持时仍需降级） + * 3. auto：锁定具体窗口 → 进程环回（干净）；整屏/未指定 → 端点环回（不漏采） + */ +export function selectAudioSource(input: AudioSourceSelectionInput): AudioSourceDecision { + const { capabilities, sourceId, preference = 'auto', microphone = false } = input; + + if (microphone) { + return { + kind: 'microphone', + reason: '现场课程场景，采集麦克风输入', + fallback: null, + }; + } + + if (preference === 'force_endpoint') { + return { + kind: 'endpoint_loopback', + reason: '用户设置为强制端点环回', + fallback: null, + }; + } + + if (preference === 'force_process') { + return capabilities.processLoopbackAvailable + ? { + kind: 'process_loopback', + reason: '用户设置为强制进程环回', + fallback: 'endpoint_loopback', + } + : { + kind: 'endpoint_loopback', + reason: '用户设置为强制进程环回，但当前环境不支持（需 Windows 10 2004+），已降级', + fallback: null, + }; + } + + if (capabilities.processLoopbackAvailable && isWindowSource(sourceId)) { + return { + kind: 'process_loopback', + reason: '已锁定目标窗口，采用进程环回以隔离其他应用声音', + fallback: 'endpoint_loopback', + }; + } + + if (!capabilities.processLoopbackAvailable) { + return { + kind: 'endpoint_loopback', + reason: '当前环境不支持进程环回（需 Windows 10 2004+）', + fallback: null, + }; + } + + return { + kind: 'endpoint_loopback', + reason: '未锁定具体窗口，采用端点环回以避免漏采跨应用声音', + fallback: null, + }; +} + +/** 源类型的用户可读名称（UI 展示与日志用） */ +export function describeAudioSource(kind: AudioSourceKind): string { + switch (kind) { + case 'process_loopback': + return '进程音频（仅目标窗口）'; + case 'endpoint_loopback': + return '系统音频（全部声音）'; + case 'microphone': + return '麦克风'; + } +} From b368bf873daf8ac70e517650ffb9dbe65143de72 Mon Sep 17 00:00:00 2001 From: Aparencia Date: Fri, 31 Jul 2026 23:11:50 +0800 Subject: [PATCH 3/5] =?UTF-8?q?feat(audio):=20=E5=8E=9F=E7=94=9F=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E6=94=AF=E6=8C=81=E6=B5=81=E5=BC=8F=E9=87=87=E9=9B=86?= =?UTF-8?q?=EF=BC=88=E9=87=87=E9=9B=86=E7=BA=BF=E7=A8=8B=20+=20ThreadSafeF?= =?UTF-8?q?unction=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 前置：同步阻塞采集无法用于真实链路，新增生产采集入口。 - capture_session：WASAPI 激活/初始化/读包原语（pimpl 隐藏 COM），供同步与流式 两路复用；含 isProcessLoopbackSupported 能力探测（真实尝试激活而非查版本号， 避免兼容性设置伪造版本号导致误判） - streaming_capture：采集线程按 chunkDurationMs 聚块回调，Stop 阻塞等待线程退出 以保证无回调泄漏 - addon：新增 startCapture/stopCapture/isProcessLoopbackSupported，块经 ThreadSafeFunction 投递到 JS 线程并拷贝为 ArrayBuffer - loopback_capture 精简为复用 CaptureSession（去重 WASAPI 样板代码） 实测 spike-streaming：5 个 1s 块周期准确、样本数 16000 一致、元数据正确、 stopCapture 后零泄漏。 --- client/native/process-audio/binding.gyp | 4 +- client/native/process-audio/src/addon.cc | 139 ++++++++++- .../process-audio/src/capture_session.cc | 234 ++++++++++++++++++ .../process-audio/src/capture_session.h | 82 ++++++ .../process-audio/src/loopback_capture.cc | 233 +++-------------- .../process-audio/src/streaming_capture.cc | 82 ++++++ .../process-audio/src/streaming_capture.h | 84 +++++++ .../process-audio/test/spike-streaming.mjs | 102 ++++++++ 8 files changed, 753 insertions(+), 207 deletions(-) create mode 100644 client/native/process-audio/src/capture_session.cc create mode 100644 client/native/process-audio/src/capture_session.h create mode 100644 client/native/process-audio/src/streaming_capture.cc create mode 100644 client/native/process-audio/src/streaming_capture.h create mode 100644 client/native/process-audio/test/spike-streaming.mjs diff --git a/client/native/process-audio/binding.gyp b/client/native/process-audio/binding.gyp index 019f65da..14f8bc93 100644 --- a/client/native/process-audio/binding.gyp +++ b/client/native/process-audio/binding.gyp @@ -5,7 +5,9 @@ "sources": [ "src/addon.cc", "src/window_finder.cc", - "src/loopback_capture.cc" + "src/capture_session.cc", + "src/loopback_capture.cc", + "src/streaming_capture.cc" ], "include_dirs": [ " +#include +#include + +#include "capture_session.h" #include "loopback_capture.h" +#include "streaming_capture.h" #include "window_finder.h" namespace { @@ -104,10 +111,136 @@ Napi::Value CaptureToWav(const Napi::CallbackInfo& info) { return out; } +/** isProcessLoopbackSupported(): 能力探测（真实尝试激活一次） */ +Napi::Value IsSupported(const Napi::CallbackInfo& info) { + return Napi::Boolean::New(info.Env(), process_audio::IsProcessLoopbackSupported()); +} + +// ================================================================ +// 流式采集（生产入口） +// ================================================================ + +/** 投递给 JS 线程的负载：一个音频块或一次错误 */ +struct StreamEvent { + bool is_error = false; + std::string error; + process_audio::StreamingChunk chunk; +}; + +/** 全局单例采集器与其线程安全回调句柄 */ +std::unique_ptr g_capture; +Napi::ThreadSafeFunction g_tsfn; + +/** 在 JS 线程消费采集线程投递的事件 */ +void DispatchStreamEvent(Napi::Env env, Napi::Function callback, StreamEvent* event) { + if (env != nullptr && callback != nullptr) { + Napi::Object payload = Napi::Object::New(env); + if (event->is_error) { + payload.Set("error", Napi::String::New(env, event->error)); + } else { + const size_t bytes = event->chunk.samples.size() * sizeof(float); + // 拷贝一份给 JS：采集线程的 vector 回调后即释放，不能共享内存 + Napi::ArrayBuffer buffer = Napi::ArrayBuffer::New(env, bytes); + if (bytes > 0) { + std::memcpy(buffer.Data(), event->chunk.samples.data(), bytes); + } + payload.Set("audioBuffer", buffer); + payload.Set("sampleRate", Napi::Number::New(env, event->chunk.sample_rate)); + payload.Set("channels", Napi::Number::New(env, event->chunk.channels)); + payload.Set("durationMs", Napi::Number::New(env, event->chunk.duration_ms)); + } + callback.Call({payload}); + } + delete event; +} + +/** + * startCapture({ pid, sampleRate, channels, chunkDurationMs }, cb): + * 启动采集线程，每聚成一块就回调 cb({audioBuffer, sampleRate, channels, + * durationMs})；致命错误回调 cb({error})。 + * + * 会话打开在采集线程内完成，故"目标不可采"这类失败以 error 回调形式 + * 异步上报，调用方应据此触发降级。 + */ +Napi::Value StartCapture(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 2 || !info[0].IsObject() || !info[1].IsFunction()) { + Napi::TypeError::New(env, "startCapture(options, callback) 参数不完整") + .ThrowAsJavaScriptException(); + return env.Undefined(); + } + if (g_capture && g_capture->running()) { + Napi::Error::New(env, "采集已在进行中，请先 stopCapture") + .ThrowAsJavaScriptException(); + return env.Undefined(); + } + + Napi::Object opts = info[0].As(); + process_audio::StreamingOptions options; + options.root_pid = opts.Has("pid") + ? opts.Get("pid").As().Uint32Value() : 0; + options.sample_rate = opts.Has("sampleRate") + ? opts.Get("sampleRate").As().Uint32Value() : 16000; + options.channels = opts.Has("channels") + ? opts.Get("channels").As().Uint32Value() : 1; + options.chunk_duration_ms = opts.Has("chunkDurationMs") + ? opts.Get("chunkDurationMs").As().Uint32Value() : 5000; + + g_tsfn = Napi::ThreadSafeFunction::New( + env, info[1].As(), "process-audio-capture", 0, 1); + + g_capture = std::make_unique(); + const std::string err = g_capture->Start( + options, + [](process_audio::StreamingChunk&& chunk) { + auto* event = new StreamEvent(); + event->chunk = std::move(chunk); + // 采集线程不能直接碰 JS，统一经 ThreadSafeFunction 投递 + if (g_tsfn.BlockingCall(event, DispatchStreamEvent) != napi_ok) { + delete event; + } + }, + [](const std::string& message) { + auto* event = new StreamEvent(); + event->is_error = true; + event->error = message; + if (g_tsfn.BlockingCall(event, DispatchStreamEvent) != napi_ok) { + delete event; + } + }); + + Napi::Object result = Napi::Object::New(env); + if (!err.empty()) { + g_tsfn.Release(); + g_capture.reset(); + result.Set("ok", Napi::Boolean::New(env, false)); + result.Set("error", Napi::String::New(env, err)); + return result; + } + result.Set("ok", Napi::Boolean::New(env, true)); + result.Set("error", Napi::String::New(env, "")); + return result; +} + +/** stopCapture(): 停止采集并等待线程退出（幂等） */ +Napi::Value StopCapture(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (g_capture) { + g_capture->Stop(); + g_capture.reset(); + g_tsfn.Release(); + } + return Napi::Boolean::New(env, true); +} + Napi::Object Init(Napi::Env env, Napi::Object exports) { exports.Set("listAudioWindows", Napi::Function::New(env, ListAudioWindows)); exports.Set("resolveRootPid", Napi::Function::New(env, ResolveRootPid)); exports.Set("captureToWav", Napi::Function::New(env, CaptureToWav)); + exports.Set("isProcessLoopbackSupported", + Napi::Function::New(env, IsSupported)); + exports.Set("startCapture", Napi::Function::New(env, StartCapture)); + exports.Set("stopCapture", Napi::Function::New(env, StopCapture)); return exports; } diff --git a/client/native/process-audio/src/capture_session.cc b/client/native/process-audio/src/capture_session.cc new file mode 100644 index 00000000..becc0334 --- /dev/null +++ b/client/native/process-audio/src/capture_session.cc @@ -0,0 +1,234 @@ +// WASAPI 进程环回会话（实现） +// +// @ai-context: 三个易踩的坑：①process loopback 不支持 GetMixFormat，必须 +// 显式指定 WAVEFORMATEX，被拒时需重新激活后再用回退格式（Initialize 失败 +// 的 client 不可复用）；②必须用 PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE， +// 否则采不到 Chromium audio service（browser process 的子进程）播放的声音； +// ③目标进程静默时不产生事件包，等待必须带超时否则永久阻塞。 + +#include "capture_session.h" + +#include +#include +#include +#include +#include + +#include + +namespace process_audio { +namespace { + +using Microsoft::WRL::ComPtr; + +/** 事件驱动采集的缓冲时长（100ns 单位，20ms） */ +constexpr REFERENCE_TIME kBufferDuration = 200000; + +/** ActivateAudioInterfaceAsync 的完成回调（异步激活转同步等待） */ +class ActivationHandler + : public Microsoft::WRL::RuntimeClass< + Microsoft::WRL::RuntimeClassFlags, + Microsoft::WRL::FtmBase, + IActivateAudioInterfaceCompletionHandler> { + public: + HANDLE done_event = nullptr; + HRESULT activate_hr = E_FAIL; + ComPtr client; + + STDMETHODIMP ActivateCompleted( + IActivateAudioInterfaceAsyncOperation* operation) override { + HRESULT inner_hr = S_OK; + ComPtr unknown; + const HRESULT hr = operation->GetActivateResult(&inner_hr, &unknown); + if (SUCCEEDED(hr) && SUCCEEDED(inner_hr) && unknown) { + activate_hr = unknown.As(&client); + } else { + activate_hr = FAILED(hr) ? hr : inner_hr; + } + if (done_event != nullptr) SetEvent(done_event); + return S_OK; + } +}; + +WAVEFORMATEX MakeFloatFormat(uint32_t sample_rate, uint32_t channels) { + WAVEFORMATEX wfx = {}; + wfx.wFormatTag = WAVE_FORMAT_IEEE_FLOAT; + wfx.nChannels = static_cast(channels); + wfx.nSamplesPerSec = sample_rate; + wfx.wBitsPerSample = 32; + wfx.nBlockAlign = static_cast(channels * 4); + wfx.nAvgBytesPerSec = sample_rate * wfx.nBlockAlign; + wfx.cbSize = 0; + return wfx; +} + +std::string HrToString(const char* stage, HRESULT hr) { + char buf[128]; + std::snprintf(buf, sizeof(buf), "%s 失败 (hr=0x%08lX)", stage, + static_cast(hr)); + return std::string(buf); +} + +/** 激活目标进程树的 IAudioClient */ +HRESULT ActivateClient(uint32_t root_pid, ComPtr* out_client) { + AUDIOCLIENT_ACTIVATION_PARAMS params = {}; + params.ActivationType = AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK; + params.ProcessLoopbackParams.TargetProcessId = static_cast(root_pid); + // 关键：包含整棵进程树，才能覆盖 Chromium audio service 等发声子进程 + params.ProcessLoopbackParams.ProcessLoopbackMode = + PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE; + + PROPVARIANT activate_params = {}; + activate_params.vt = VT_BLOB; + activate_params.blob.cbSize = sizeof(params); + activate_params.blob.pBlobData = reinterpret_cast(¶ms); + + auto handler = Microsoft::WRL::Make(); + if (!handler) return E_OUTOFMEMORY; + handler->done_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (handler->done_event == nullptr) return HRESULT_FROM_WIN32(GetLastError()); + + ComPtr operation; + HRESULT hr = ActivateAudioInterfaceAsync( + VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, __uuidof(IAudioClient), + &activate_params, handler.Get(), &operation); + + if (SUCCEEDED(hr)) { + // 激活为异步流程，等待 handler 回调（含超时兜底避免永久阻塞） + if (WaitForSingleObject(handler->done_event, 3000) != WAIT_OBJECT_0) { + hr = HRESULT_FROM_WIN32(WAIT_TIMEOUT); + } else { + hr = handler->activate_hr; + if (SUCCEEDED(hr)) *out_client = handler->client; + } + } + CloseHandle(handler->done_event); + handler->done_event = nullptr; + return hr; +} + +} // namespace + +struct CaptureSession::Impl { + ComPtr client; + ComPtr capture; + HANDLE sample_event = nullptr; + WAVEFORMATEX wfx = {}; + bool com_initialized = false; + bool started = false; + + ~Impl() { + if (started && client) client->Stop(); + capture.Reset(); + client.Reset(); + if (sample_event != nullptr) CloseHandle(sample_event); + if (com_initialized) CoUninitialize(); + } +}; + +CaptureSession::CaptureSession() : impl_(std::make_unique()) {} +CaptureSession::~CaptureSession() = default; + +std::string CaptureSession::Open(uint32_t root_pid, + uint32_t sample_rate, + uint32_t channels) { + const HRESULT co_hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + impl_->com_initialized = SUCCEEDED(co_hr); + + HRESULT hr = ActivateClient(root_pid, &impl_->client); + if (FAILED(hr) || !impl_->client) return HrToString("进程环回激活", hr); + + const DWORD stream_flags = + AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_EVENTCALLBACK; + impl_->wfx = MakeFloatFormat(sample_rate, channels); + hr = impl_->client->Initialize(AUDCLNT_SHAREMODE_SHARED, stream_flags, + kBufferDuration, 0, &impl_->wfx, nullptr); + if (FAILED(hr)) { + // 回退格式重试（Initialize 失败后的 client 不可复用，需重新激活） + impl_->client.Reset(); + hr = ActivateClient(root_pid, &impl_->client); + if (SUCCEEDED(hr) && impl_->client) { + impl_->wfx = MakeFloatFormat(48000, 2); + hr = impl_->client->Initialize(AUDCLNT_SHAREMODE_SHARED, stream_flags, + kBufferDuration, 0, &impl_->wfx, nullptr); + } + if (FAILED(hr)) return HrToString("IAudioClient::Initialize", hr); + } + + impl_->sample_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (impl_->sample_event == nullptr) return "创建采集事件失败"; + hr = impl_->client->SetEventHandle(impl_->sample_event); + if (FAILED(hr)) return HrToString("SetEventHandle", hr); + + hr = impl_->client->GetService( + __uuidof(IAudioCaptureClient), + reinterpret_cast(impl_->capture.GetAddressOf())); + if (FAILED(hr)) return HrToString("GetService(IAudioCaptureClient)", hr); + + return ""; +} + +std::string CaptureSession::Start() { + if (!impl_->client) return "会话未初始化"; + const HRESULT hr = impl_->client->Start(); + if (FAILED(hr)) return HrToString("IAudioClient::Start", hr); + impl_->started = true; + return ""; +} + +bool CaptureSession::ReadPackets(std::vector* out, + ReadStats* stats, + uint32_t wait_ms) { + if (!impl_->capture) return false; + WaitForSingleObject(impl_->sample_event, wait_ms); + + const uint32_t channels = impl_->wfx.nChannels; + UINT32 packet_frames = 0; + while (SUCCEEDED(impl_->capture->GetNextPacketSize(&packet_frames)) && + packet_frames > 0) { + BYTE* data = nullptr; + UINT32 frames = 0; + DWORD flags = 0; + const HRESULT hr = + impl_->capture->GetBuffer(&data, &frames, &flags, nullptr, nullptr); + if (FAILED(hr)) return false; + + const size_t sample_count = static_cast(frames) * channels; + if (stats != nullptr) stats->packets++; + if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0) { + if (stats != nullptr) stats->silent_packets++; + out->insert(out->end(), sample_count, 0.0f); + } else { + const auto* floats = reinterpret_cast(data); + out->insert(out->end(), floats, floats + sample_count); + } + impl_->capture->ReleaseBuffer(frames); + } + return true; +} + +void CaptureSession::Stop() { + if (impl_->started && impl_->client) { + impl_->client->Stop(); + impl_->started = false; + } +} + +uint32_t CaptureSession::sample_rate() const { return impl_->wfx.nSamplesPerSec; } +uint32_t CaptureSession::channels() const { return impl_->wfx.nChannels; } + +bool IsProcessLoopbackSupported() { + const HRESULT co_hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + const bool need_uninit = SUCCEEDED(co_hr); + + ComPtr client; + // 以当前进程为目标试探激活：只验证 API 可用性，不 Initialize、不采集 + const HRESULT hr = ActivateClient(GetCurrentProcessId(), &client); + const bool supported = SUCCEEDED(hr) && client; + + client.Reset(); + if (need_uninit) CoUninitialize(); + return supported; +} + +} // namespace process_audio diff --git a/client/native/process-audio/src/capture_session.h b/client/native/process-audio/src/capture_session.h new file mode 100644 index 00000000..9354198a --- /dev/null +++ b/client/native/process-audio/src/capture_session.h @@ -0,0 +1,82 @@ +// WASAPI 进程环回会话（声明） +// +// @ai-context: 把"激活 + 初始化 + 读包"的 COM 细节收在 pimpl 内，供同步 +// 采集（spike）与流式采集线程共用，避免两处重复 WASAPI 样板代码。 +// @ai-context: 同一个 CaptureSession 实例的全部方法必须在同一线程调用—— +// COM 单元（CoInitializeEx）按线程绑定，跨线程使用会失败。 + +#ifndef PROCESS_AUDIO_CAPTURE_SESSION_H_ +#define PROCESS_AUDIO_CAPTURE_SESSION_H_ + +#include +#include +#include +#include + +namespace process_audio { + +/** 单次读包的统计 */ +struct ReadStats { + uint32_t packets = 0; + uint32_t silent_packets = 0; +}; + +/** + * 进程环回采集会话。 + * + * 生命周期：Open → Start → ReadPackets（循环）→ Stop。 + * 失败的方法返回非空错误描述字符串。 + */ +class CaptureSession { + public: + CaptureSession(); + ~CaptureSession(); + + CaptureSession(const CaptureSession&) = delete; + CaptureSession& operator=(const CaptureSession&) = delete; + + /** + * 激活目标进程树的音频客户端并初始化格式。 + * + * @param root_pid 目标进程树根 PID（浏览器场景为 browser process） + * @param sample_rate 期望采样率（被拒时自动回退 48000） + * @param channels 期望声道数（被拒时自动回退 2） + * @return 空字符串表示成功 + */ + std::string Open(uint32_t root_pid, uint32_t sample_rate, uint32_t channels); + + /** 开始采集流；Open 成功后调用 */ + std::string Start(); + + /** + * 等待并取出当前可用的所有样本（追加到 out）。 + * + * @param wait_ms 事件等待上限；目标进程静默时不产生事件，靠超时返回 + * @return false 表示发生不可恢复错误，调用方应终止采集 + */ + bool ReadPackets(std::vector* out, ReadStats* stats, uint32_t wait_ms); + + /** 停止采集流（幂等） */ + void Stop(); + + /** 实际生效的采样率（Open 成功后有效） */ + uint32_t sample_rate() const; + /** 实际生效的声道数（Open 成功后有效） */ + uint32_t channels() const; + + private: + struct Impl; + std::unique_ptr impl_; +}; + +/** + * 探测当前环境是否支持进程环回。 + * + * 实现方式为真实尝试激活一次（不 Initialize），比查 Windows 版本号可靠—— + * 版本号可被兼容性设置伪造，而 API 可用性是最终判据。 + */ +bool IsProcessLoopbackSupported(); + +} // namespace process_audio + +#endif // PROCESS_AUDIO_CAPTURE_SESSION_H_ diff --git a/client/native/process-audio/src/loopback_capture.cc b/client/native/process-audio/src/loopback_capture.cc index fccfdcb2..a4c5973a 100644 --- a/client/native/process-audio/src/loopback_capture.cc +++ b/client/native/process-audio/src/loopback_capture.cc @@ -1,117 +1,19 @@ -// WASAPI 进程环回采集（实现） +// 同步采集与 WAV 落盘（spike 验证用，实现） // -// @ai-context: 三个易踩的坑：①process loopback 不支持 GetMixFormat， -// 必须显式指定 WAVEFORMATEX，被拒时需回退；②必须用 -// PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE，否则采不到 -// Chromium audio service（browser process 的子进程）播放的声音； -// ③目标进程无音频输出时不产生数据包，需靠超时而非"等够包数"退出。 +// @ai-context: 阻塞调用线程，仅供 spike 脚本做可行性验证与人工复核； +// 生产采集走 streaming_capture 的采集线程 + 流式回调。 +// WASAPI 细节已下沉到 CaptureSession，本文件只做时长控制与统计。 #include "loopback_capture.h" #include -#include -#include -#include -#include #include #include -#include -namespace process_audio { -namespace { - -using Microsoft::WRL::ComPtr; - -/** 事件驱动采集的缓冲时长（100ns 单位，20ms） */ -constexpr REFERENCE_TIME kBufferDuration = 200000; - -/** ActivateAudioInterfaceAsync 的完成回调（异步激活转同步等待） */ -class ActivationHandler - : public Microsoft::WRL::RuntimeClass< - Microsoft::WRL::RuntimeClassFlags, - Microsoft::WRL::FtmBase, - IActivateAudioInterfaceCompletionHandler> { - public: - HANDLE done_event = nullptr; - HRESULT activate_hr = E_FAIL; - ComPtr client; - - STDMETHODIMP ActivateCompleted( - IActivateAudioInterfaceAsyncOperation* operation) override { - HRESULT inner_hr = S_OK; - ComPtr unknown; - const HRESULT hr = operation->GetActivateResult(&inner_hr, &unknown); - if (SUCCEEDED(hr) && SUCCEEDED(inner_hr) && unknown) { - activate_hr = unknown.As(&client); - } else { - activate_hr = FAILED(hr) ? hr : inner_hr; - } - if (done_event != nullptr) SetEvent(done_event); - return S_OK; - } -}; - -/** 构造 Float32 交错格式描述 */ -WAVEFORMATEX MakeFloatFormat(uint32_t sample_rate, uint32_t channels) { - WAVEFORMATEX wfx = {}; - wfx.wFormatTag = WAVE_FORMAT_IEEE_FLOAT; - wfx.nChannels = static_cast(channels); - wfx.nSamplesPerSec = sample_rate; - wfx.wBitsPerSample = 32; - wfx.nBlockAlign = static_cast(channels * 4); - wfx.nAvgBytesPerSec = sample_rate * wfx.nBlockAlign; - wfx.cbSize = 0; - return wfx; -} - -std::string HrToString(const char* stage, HRESULT hr) { - char buf[128]; - std::snprintf(buf, sizeof(buf), "%s 失败 (hr=0x%08lX)", stage, - static_cast(hr)); - return std::string(buf); -} - -/** 激活目标进程树的 IAudioClient */ -HRESULT ActivateProcessLoopbackClient(uint32_t root_pid, - ComPtr* out_client) { - AUDIOCLIENT_ACTIVATION_PARAMS params = {}; - params.ActivationType = AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK; - params.ProcessLoopbackParams.TargetProcessId = static_cast(root_pid); - // 关键：包含整棵进程树，才能覆盖 Chromium audio service 等发声子进程 - params.ProcessLoopbackParams.ProcessLoopbackMode = - PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE; - - PROPVARIANT activate_params = {}; - activate_params.vt = VT_BLOB; - activate_params.blob.cbSize = sizeof(params); - activate_params.blob.pBlobData = reinterpret_cast(¶ms); - - auto handler = Microsoft::WRL::Make(); - if (!handler) return E_OUTOFMEMORY; - handler->done_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); - if (handler->done_event == nullptr) return HRESULT_FROM_WIN32(GetLastError()); - - ComPtr operation; - HRESULT hr = ActivateAudioInterfaceAsync( - VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, __uuidof(IAudioClient), - &activate_params, handler.Get(), &operation); - - if (SUCCEEDED(hr)) { - // 激活为异步流程，等待 handler 回调（含超时兜底避免永久阻塞） - if (WaitForSingleObject(handler->done_event, 3000) != WAIT_OBJECT_0) { - hr = HRESULT_FROM_WIN32(WAIT_TIMEOUT); - } else { - hr = handler->activate_hr; - if (SUCCEEDED(hr)) *out_client = handler->client; - } - } - CloseHandle(handler->done_event); - handler->done_event = nullptr; - return hr; -} +#include "capture_session.h" -} // namespace +namespace process_audio { CaptureResult CaptureProcessAudio(uint32_t root_pid, uint32_t duration_ms, @@ -119,117 +21,41 @@ CaptureResult CaptureProcessAudio(uint32_t root_pid, uint32_t preferred_channels) { CaptureResult result; - const HRESULT co_hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); - const bool need_uninit = SUCCEEDED(co_hr); - - ComPtr client; - HRESULT hr = ActivateProcessLoopbackClient(root_pid, &client); - if (FAILED(hr) || !client) { - result.error = HrToString("进程环回激活", hr); - if (need_uninit) CoUninitialize(); - return result; - } - - // process loopback 不支持 GetMixFormat，必须显式指定格式； - // 优先请求下游所需的 16kHz mono，被拒时回退 48kHz stereo 由调用方重采样 - WAVEFORMATEX wfx = MakeFloatFormat(preferred_sample_rate, preferred_channels); - const DWORD stream_flags = - AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_EVENTCALLBACK; - hr = client->Initialize(AUDCLNT_SHAREMODE_SHARED, stream_flags, - kBufferDuration, 0, &wfx, nullptr); - if (FAILED(hr)) { - // 回退格式重试（需重新激活：Initialize 失败后的 client 不可复用） - client.Reset(); - hr = ActivateProcessLoopbackClient(root_pid, &client); - if (SUCCEEDED(hr) && client) { - wfx = MakeFloatFormat(48000, 2); - hr = client->Initialize(AUDCLNT_SHAREMODE_SHARED, stream_flags, - kBufferDuration, 0, &wfx, nullptr); - } - if (FAILED(hr)) { - result.error = HrToString("IAudioClient::Initialize", hr); - if (need_uninit) CoUninitialize(); - return result; - } - } - result.sample_rate = wfx.nSamplesPerSec; - result.channels = wfx.nChannels; - - HANDLE sample_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); - if (sample_event == nullptr) { - result.error = "创建采集事件失败"; - if (need_uninit) CoUninitialize(); - return result; - } - hr = client->SetEventHandle(sample_event); - if (FAILED(hr)) { - result.error = HrToString("SetEventHandle", hr); - CloseHandle(sample_event); - if (need_uninit) CoUninitialize(); + CaptureSession session; + std::string err = session.Open(root_pid, preferred_sample_rate, preferred_channels); + if (!err.empty()) { + result.error = err; return result; } + result.sample_rate = session.sample_rate(); + result.channels = session.channels(); - ComPtr capture; - hr = client->GetService(__uuidof(IAudioCaptureClient), - reinterpret_cast(capture.GetAddressOf())); - if (FAILED(hr)) { - result.error = HrToString("GetService(IAudioCaptureClient)", hr); - CloseHandle(sample_event); - if (need_uninit) CoUninitialize(); - return result; - } - - hr = client->Start(); - if (FAILED(hr)) { - result.error = HrToString("IAudioClient::Start", hr); - CloseHandle(sample_event); - if (need_uninit) CoUninitialize(); + err = session.Start(); + if (!err.empty()) { + result.error = err; return result; } + ReadStats stats; const DWORD deadline = GetTickCount() + duration_ms; - double square_sum = 0.0; while (GetTickCount() < deadline) { const DWORD remain = deadline - GetTickCount(); - // 目标进程静默时不产生事件，故等待上限取剩余时长与 200ms 的较小值， - // 保证到点即退出而非无限等待 + // 目标进程静默时不产生事件，等待上限取剩余时长与 200ms 的较小值 const DWORD wait_ms = remain < 200 ? remain : 200; - WaitForSingleObject(sample_event, wait_ms); - - UINT32 packet_frames = 0; - while (SUCCEEDED(capture->GetNextPacketSize(&packet_frames)) && - packet_frames > 0) { - BYTE* data = nullptr; - UINT32 frames = 0; - DWORD flags = 0; - hr = capture->GetBuffer(&data, &frames, &flags, nullptr, nullptr); - if (FAILED(hr)) break; - - result.packet_count++; - const size_t sample_count = - static_cast(frames) * result.channels; - if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0) { - result.silent_packet_count++; - result.samples.insert(result.samples.end(), sample_count, 0.0f); - } else { - const auto* floats = reinterpret_cast(data); - result.samples.insert(result.samples.end(), floats, - floats + sample_count); - for (size_t i = 0; i < sample_count; ++i) { - const double v = static_cast(floats[i]); - square_sum += v * v; - const double a = std::fabs(v); - if (a > result.peak) result.peak = a; - } - } - capture->ReleaseBuffer(frames); - } + if (!session.ReadPackets(&result.samples, &stats, wait_ms)) break; } + session.Stop(); - client->Stop(); - CloseHandle(sample_event); - if (need_uninit) CoUninitialize(); + result.packet_count = stats.packets; + result.silent_packet_count = stats.silent_packets; + double square_sum = 0.0; + for (const float v : result.samples) { + const double d = static_cast(v); + square_sum += d * d; + const double a = std::fabs(d); + if (a > result.peak) result.peak = a; + } if (!result.samples.empty()) { result.rms = std::sqrt(square_sum / static_cast(result.samples.size())); } @@ -252,6 +78,7 @@ bool WriteWavFloat32(const std::string& path, const uint16_t bits = 32; const uint32_t fmt_size = 16; const uint16_t ch = static_cast(channels); + const uint16_t align16 = static_cast(block_align); auto put = [fp](const void* p, size_t n) { std::fwrite(p, 1, n, fp); }; put("RIFF", 4); @@ -263,7 +90,7 @@ bool WriteWavFloat32(const std::string& path, put(&ch, 2); put(&sample_rate, 4); put(&byte_rate, 4); - put(reinterpret_cast(&block_align), 2); + put(&align16, 2); put(&bits, 2); put("data", 4); put(&data_bytes, 4); diff --git a/client/native/process-audio/src/streaming_capture.cc b/client/native/process-audio/src/streaming_capture.cc new file mode 100644 index 00000000..6b8960b5 --- /dev/null +++ b/client/native/process-audio/src/streaming_capture.cc @@ -0,0 +1,82 @@ +// 流式进程环回采集（实现） +// +// @ai-context: 聚合策略——WASAPI 每 ~10ms 给一包，直接回调会让下游承受 +// 高频 IPC；故在采集线程内累积到 chunkDurationMs 再整块回调，与端点环回 +// 路径的分块粒度保持一致（下游 VAD/ASR 对块大小有预期）。 + +#include "streaming_capture.h" + +#include "capture_session.h" + +namespace process_audio { + +StreamingCapture::StreamingCapture() = default; + +StreamingCapture::~StreamingCapture() { Stop(); } + +std::string StreamingCapture::Start(const StreamingOptions& options, + ChunkHandler on_chunk, + ErrorHandler on_error) { + if (running_.load()) return "采集已在进行中"; + if (options.root_pid == 0) return "目标 PID 无效"; + + on_chunk_ = std::move(on_chunk); + on_error_ = std::move(on_error); + stop_flag_.store(false); + running_.store(true); + thread_ = std::thread(&StreamingCapture::ThreadMain, this, options); + return ""; +} + +void StreamingCapture::Stop() { + stop_flag_.store(true); + if (thread_.joinable()) thread_.join(); + running_.store(false); +} + +bool StreamingCapture::running() const { return running_.load(); } + +void StreamingCapture::ThreadMain(StreamingOptions options) { + CaptureSession session; + + std::string err = session.Open(options.root_pid, options.sample_rate, options.channels); + if (err.empty()) err = session.Start(); + if (!err.empty()) { + running_.store(false); + if (on_error_) on_error_(err); + return; + } + + const uint32_t rate = session.sample_rate(); + const uint32_t channels = session.channels(); + // 一个完整块的交错样本数 + const size_t chunk_samples = static_cast( + static_cast(rate) * options.chunk_duration_ms / 1000 * channels); + + std::vector pending; + pending.reserve(chunk_samples * 2); + + while (!stop_flag_.load()) { + ReadStats stats; + if (!session.ReadPackets(&pending, &stats, 100)) { + if (on_error_) on_error_("采集流读取失败，可能目标进程已退出"); + break; + } + + while (pending.size() >= chunk_samples && chunk_samples > 0) { + StreamingChunk chunk; + chunk.samples.assign(pending.begin(), + pending.begin() + static_cast(chunk_samples)); + chunk.sample_rate = rate; + chunk.channels = channels; + chunk.duration_ms = options.chunk_duration_ms; + pending.erase(pending.begin(), pending.begin() + static_cast(chunk_samples)); + if (on_chunk_) on_chunk_(std::move(chunk)); + } + } + + session.Stop(); + running_.store(false); +} + +} // namespace process_audio diff --git a/client/native/process-audio/src/streaming_capture.h b/client/native/process-audio/src/streaming_capture.h new file mode 100644 index 00000000..898ca0b0 --- /dev/null +++ b/client/native/process-audio/src/streaming_capture.h @@ -0,0 +1,84 @@ +// 流式进程环回采集（采集线程 + 回调，声明） +// +// @ai-context: 生产采集入口。独立线程跑 WASAPI 事件循环，按 chunkDurationMs +// 聚合成块后回调，块格式与下游 AudioChunkData 契约一致（Float32 交错 PCM）。 +// @ai-context: 采集线程内自建 CaptureSession——COM 单元按线程绑定， +// 不可在主线程 Open 后交给采集线程使用。 + +#ifndef PROCESS_AUDIO_STREAMING_CAPTURE_H_ +#define PROCESS_AUDIO_STREAMING_CAPTURE_H_ + +#include +#include +#include +#include +#include +#include +#include + +namespace process_audio { + +/** 流式采集配置 */ +struct StreamingOptions { + uint32_t root_pid = 0; + uint32_t sample_rate = 16000; + uint32_t channels = 1; + uint32_t chunk_duration_ms = 5000; +}; + +/** 一个聚合完成的音频块 */ +struct StreamingChunk { + std::vector samples; + uint32_t sample_rate = 0; + uint32_t channels = 0; + uint32_t duration_ms = 0; +}; + +/** 块回调（在采集线程调用，实现方需自行转投到目标线程） */ +using ChunkHandler = std::function; +/** 致命错误回调（采集线程终止前调用一次） */ +using ErrorHandler = std::function; + +/** + * 流式采集器。 + * + * Start 立即返回（采集在后台线程）；Stop 会阻塞等待线程退出，保证回调 + * 不会在 Stop 返回后继续触发。两者均幂等。 + */ +class StreamingCapture { + public: + StreamingCapture(); + ~StreamingCapture(); + + StreamingCapture(const StreamingCapture&) = delete; + StreamingCapture& operator=(const StreamingCapture&) = delete; + + /** + * 启动采集线程。 + * + * 会话打开在采集线程内完成，因此启动期的失败通过 on_error 异步上报， + * 而非返回值——调用方应据此触发降级。 + * @return 空字符串表示线程已启动 + */ + std::string Start(const StreamingOptions& options, + ChunkHandler on_chunk, + ErrorHandler on_error); + + /** 停止采集并等待线程退出（幂等） */ + void Stop(); + + bool running() const; + + private: + void ThreadMain(StreamingOptions options); + + std::atomic stop_flag_{false}; + std::atomic running_{false}; + std::thread thread_; + ChunkHandler on_chunk_; + ErrorHandler on_error_; +}; + +} // namespace process_audio + +#endif // PROCESS_AUDIO_STREAMING_CAPTURE_H_ diff --git a/client/native/process-audio/test/spike-streaming.mjs b/client/native/process-audio/test/spike-streaming.mjs new file mode 100644 index 00000000..186ff639 --- /dev/null +++ b/client/native/process-audio/test/spike-streaming.mjs @@ -0,0 +1,102 @@ +// Phase 2 验证脚本：流式采集（采集线程 + ThreadSafeFunction 回调） +// +// 用法：node test/spike-streaming.mjs +// +// 验收点： +// 1. isProcessLoopbackSupported() 正确返回 true（本机 Win11） +// 2. 启动后按 chunkDurationMs 周期性收到块，块大小 = rate*ms/1000*channels +// 3. 块内容非零（声源在播放），RMS ≥ 0.008 +// 4. stopCapture 后不再有回调（无泄漏） +// 5. 目标进程退出时通过 error 回调上报（供降级） + +import { createRequire } from 'node:module'; +import { spawn } from 'node:child_process'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const require = createRequire(import.meta.url); +const addon = require('../build/Release/process_audio.node'); +const here = path.dirname(fileURLToPath(import.meta.url)); + +const CHUNK_MS = 1000; // 用 1s 块加快验证节奏 +const SAMPLE_RATE = 16000; +const CHANNELS = 1; +const EXPECTED_SAMPLES = (SAMPLE_RATE * CHUNK_MS) / 1000 * CHANNELS; + +console.log('=== 流式采集验证 ===\n'); +console.log(`isProcessLoopbackSupported() = ${addon.isProcessLoopbackSupported()}\n`); + +// 用 Electron 当 Chromium 声源（等价浏览器场景） +const clientRequire = createRequire(path.join(here, '..', '..', '..', 'package.json')); +const electronBin = clientRequire('electron'); +const child = spawn(electronBin, [path.join(here, 'chromium-source')], { + stdio: 'ignore', + windowsHide: false, +}); +const stopSource = () => { if (!child.killed) child.kill(); }; +process.on('exit', stopSource); +process.on('SIGINT', () => { stopSource(); process.exit(1); }); + +await new Promise((r) => setTimeout(r, 5000)); +console.log(`声源 PID=${child.pid}，开始流式采集...\n`); + +const chunks = []; +let errorMessage = null; + +const started = addon.startCapture( + { pid: child.pid, sampleRate: SAMPLE_RATE, channels: CHANNELS, chunkDurationMs: CHUNK_MS }, + (payload) => { + if (payload.error) { + errorMessage = payload.error; + console.log(` [error 回调] ${payload.error}`); + return; + } + const samples = new Float32Array(payload.audioBuffer); + let sum = 0; + for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i]; + const rms = Math.sqrt(sum / (samples.length || 1)); + chunks.push({ n: samples.length, rms, rate: payload.sampleRate, ch: payload.channels, ms: payload.durationMs }); + console.log( + ` [块 ${chunks.length}] 样本=${samples.length} ${payload.sampleRate}Hz/${payload.channels}ch ` + + `${payload.durationMs}ms RMS=${rms.toFixed(6)}`, + ); + }, +); + +if (!started.ok) { + console.error(`startCapture 失败：${started.error}`); + stopSource(); + process.exit(1); +} + +// 采集 5 秒 → 预期约 5 个 1s 块 +await new Promise((r) => setTimeout(r, 5200)); +addon.stopCapture(); +console.log('\nstopCapture 已调用，等待 1.5s 观察是否仍有回调...'); +const countAtStop = chunks.length; +await new Promise((r) => setTimeout(r, 1500)); +const leaked = chunks.length - countAtStop; + +stopSource(); + +// ---- 判定 ---- +const THRESHOLD = 0.008; +const sizeOk = chunks.every((c) => c.n === EXPECTED_SAMPLES); +const formatOk = chunks.every((c) => c.rate === SAMPLE_RATE && c.ch === CHANNELS && c.ms === CHUNK_MS); +// 真实音频开头/结尾常有淡入淡出，首尾块能量偏低属正常（也正是 asrFilters +// 静音门控存在的意义），故判据取"多数块非静音"而非"全部非静音" +const loudChunks = chunks.filter((c) => c.rms >= THRESHOLD).length; +const loudOk = chunks.length > 0 && loudChunks / chunks.length >= 0.6; +const countOk = chunks.length >= 4 && chunks.length <= 6; + +console.log('\n判定：'); +console.log(` ${countOk ? '✔' : '✖'} 块数量符合周期（收到 ${chunks.length} 块，预期 4~6）`); +console.log(` ${sizeOk ? '✔' : '✖'} 每块样本数均为 ${EXPECTED_SAMPLES}`); +console.log(` ${formatOk ? '✔' : '✖'} 块元数据（采样率/声道/时长）正确`); +console.log(` ${loudOk ? '✔' : '✖'} 多数块非静音（${loudChunks}/${chunks.length} 块 RMS ≥ ${THRESHOLD}）`); +console.log(` ${leaked === 0 ? '✔' : '✖'} stopCapture 后无额外回调（泄漏 ${leaked} 块）`); +if (errorMessage) console.log(` ⓘ 期间收到 error 回调：${errorMessage}`); + +const pass = countOk && sizeOk && formatOk && loudOk && leaked === 0; +console.log(pass ? '\n ✅ 流式采集可用于生产链路' : '\n ❌ 未通过，需修正'); +process.exit(pass ? 0 : 1); From bc37a960edbbde50b3fde509429304426f193b8f Mon Sep 17 00:00:00 2001 From: Aparencia Date: Fri, 31 Jul 2026 23:21:19 +0800 Subject: [PATCH 4/5] =?UTF-8?q?feat(audio):=20=E6=8E=A5=E5=85=A5=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E7=8E=AF=E5=9B=9E=20Provider=20=E4=B8=8E=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E9=93=BE=E8=B7=AF=EF=BC=88Phase=202=EF=BC=8Cflag=20?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=85=B3=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-001 Phase 2：原生进程环回接入编排器，特性开关默认关闭，行为与 Phase 0 一致；开启方式为环境变量 ENTROPY_PROCESS_LOOPBACK=1。 - processAudioNative：原生模块加载器。addon 视为可选依赖，未编译/非 Windows/ 版本过低均优雅降级为'能力不可用'，加载失败不冒泡到启动路径；能力探测与 加载结果缓存到进程生命周期 - processLoopbackProvider：窗口源 ID 解析 HWND → 匹配原生枚举取进程树根 PID； 采集在主进程完成故不实现 handleRendererChunk - audioCapture：接入能力探测与 flag；新增运行时降级——采集开始后才发生的 进程环回故障（如目标进程退出）无缝切回端点环回 - 构建链路：native:install / native:build 脚本（electron-rebuild 针对 Electron ABI 编译）、electron-builder files 纳入 .node 产物（已有 asarUnpack 解包）、CI 增加编译步骤（continue-on-error + 产物检查告警， 编译失败不阻断发布） 验证：client 473 tests、lint 0 errors、渲染与主进程 tsc 均 0； electron-rebuild 针对 Electron ABI 重建通过。 未纳入本阶段：useAudioRecovery 按源分支提示文案（flag 关闭时现有文案正确）， 留待 Phase 3 灰度开启时一并处理。 --- .github/workflows/release.yml | 18 +++ client/electron-builder.yml | 4 + client/electron/audio/processAudioNative.ts | 121 +++++++++++++++++ .../electron/audio/processLoopbackProvider.ts | 125 ++++++++++++++++++ client/electron/audioCapture.ts | 53 +++++++- client/package.json | 2 + 6 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 client/electron/audio/processAudioNative.ts create mode 100644 client/electron/audio/processLoopbackProvider.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3df68d96..8e4040b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,6 +59,24 @@ jobs: cache: 'npm' cache-dependency-path: client/package-lock.json - run: cd client && npm ci + # 进程环回原生模块（ADR-001）：windows-latest 自带 MSVC，针对 Electron ABI 编译。 + # continue-on-error：它是可选增强（运行时未加载到则降级为端点环回）， + # 不应因其编译失败而阻断整条发布流水线 + - name: Build native process-audio module + continue-on-error: true + run: | + cd client + npm run native:install + npm run native:build + - name: Check native module artifact + shell: bash + run: | + cd client + if [ -f native/process-audio/build/Release/process_audio.node ]; then + echo "进程环回原生模块已就绪" + else + echo "::warning::进程环回原生模块未构建成功，本包仅支持端点环回采集" + fi # --publish never：发布由下方 release job 统一负责，避免 electron-builder # 因检测到 tag 而自行尝试发布（github provider 无 GH_TOKEN 会报错） # diff --git a/client/electron-builder.yml b/client/electron-builder.yml index 8f7e8d27..f304942a 100644 --- a/client/electron-builder.yml +++ b/client/electron-builder.yml @@ -25,6 +25,10 @@ files: - dist/**/* - dist-electron/**/* - app-icon.png + # 进程环回原生模块（ADR-001）：仅打包编译产物，不带 src/node_modules。 + # 经上方 asarUnpack 解到 app.asar.unpacked/，processAudioNative 按此路径加载； + # 未编译时此匹配为空，应用仍可正常启动（降级为端点环回） + - native/process-audio/build/Release/*.node # 图标说明： # app-icon.png 为统一源文件（1024x1024 方形 PNG）。 # electron-builder 会自动将其转换为各平台所需格式： diff --git a/client/electron/audio/processAudioNative.ts b/client/electron/audio/processAudioNative.ts new file mode 100644 index 00000000..030c52a3 --- /dev/null +++ b/client/electron/audio/processAudioNative.ts @@ -0,0 +1,121 @@ +/** + * 进程环回原生模块加载器 + * + * @ai-context: 原生 addon 是可选依赖——未编译、非 Windows、或 Windows 版本 + * 过低时必须优雅降级为"能力不可用"，绝不能让加载失败冒泡到应用启动路径 + * （AGENTS.md 高风险区约定）。故此处用 try/catch 包裹 require 并缓存结果。 + * @ai-context: 能力探测（isProcessLoopbackSupported）会真实尝试激活一次 + * WASAPI 接口，有几毫秒开销，因此结果缓存到进程生命周期。 + */ + +import * as path from 'path'; +import { app } from 'electron'; +import { logger } from '../logger.js'; + +/** 原生模块导出的窗口信息 */ +export interface NativeWindowInfo { + hwnd: string; + pid: number; + rootPid: number; + title: string; + processName: string; + rootProcessName: string; +} + +/** 原生模块流式回调的负载：音频块或错误 */ +export interface NativeStreamPayload { + audioBuffer?: ArrayBuffer; + sampleRate?: number; + channels?: number; + durationMs?: number; + error?: string; +} + +/** 原生模块导出接口 */ +export interface ProcessAudioNative { + listAudioWindows(): NativeWindowInfo[]; + resolveRootPid(pid: number): number; + isProcessLoopbackSupported(): boolean; + startCapture( + options: { pid: number; sampleRate: number; channels: number; chunkDurationMs: number }, + callback: (payload: NativeStreamPayload) => void, + ): { ok: boolean; error: string }; + stopCapture(): boolean; +} + +/** 加载状态缓存（undefined 表示尚未尝试加载） */ +let cached: ProcessAudioNative | null | undefined; +let supportedCache: boolean | undefined; + +/** 候选路径：开发态走源码目录构建产物，打包后走 asarUnpack 解出的目录 */ +function candidatePaths(): string[] { + const fileName = 'process_audio.node'; + const appPath = app.getAppPath(); + return [ + // 打包后：asarUnpack 将 .node 解到 app.asar.unpacked 下 + path.join(appPath.replace('app.asar', 'app.asar.unpacked'), 'native', 'process-audio', 'build', 'Release', fileName), + // 开发态：client/native/process-audio/build/Release + path.join(appPath, 'native', 'process-audio', 'build', 'Release', fileName), + ]; +} + +/** + * 加载原生模块；不可用时返回 null（不抛错）。 + */ +export function loadProcessAudioNative(): ProcessAudioNative | null { + if (cached !== undefined) return cached; + + if (process.platform !== 'win32') { + logger.info('[ProcessAudio] 非 Windows 平台，进程环回不可用'); + cached = null; + return cached; + } + + // 主进程编译目标为 CommonJS，直接用全局 require 动态加载 .node + for (const candidate of candidatePaths()) { + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const native = require(candidate) as ProcessAudioNative; + if (typeof native.startCapture === 'function') { + logger.info(`[ProcessAudio] 原生模块已加载: ${candidate}`); + cached = native; + return cached; + } + } catch { + // 继续尝试下一个候选路径 + } + } + + logger.info('[ProcessAudio] 原生模块未找到（未编译或未随包分发），进程环回不可用'); + cached = null; + return cached; +} + +/** + * 进程环回是否可用（模块已加载 + 系统 API 可用）。 + * 结果缓存到进程生命周期。 + */ +export function isProcessLoopbackAvailable(): boolean { + if (supportedCache !== undefined) return supportedCache; + + const native = loadProcessAudioNative(); + if (!native) { + supportedCache = false; + return supportedCache; + } + try { + supportedCache = native.isProcessLoopbackSupported(); + logger.info(`[ProcessAudio] 系统能力探测: ${supportedCache ? '支持' : '不支持'}（需 Windows 10 2004+）`); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + logger.warn(`[ProcessAudio] 能力探测异常，视为不可用: ${message}`); + supportedCache = false; + } + return supportedCache; +} + +/** 仅测试用：重置缓存 */ +export function resetProcessAudioCacheForTest(): void { + cached = undefined; + supportedCache = undefined; +} diff --git a/client/electron/audio/processLoopbackProvider.ts b/client/electron/audio/processLoopbackProvider.ts new file mode 100644 index 00000000..56ad355f --- /dev/null +++ b/client/electron/audio/processLoopbackProvider.ts @@ -0,0 +1,125 @@ +/** + * 进程环回 Provider（Windows 原生 WASAPI 路径） + * + * @ai-context: 见 ADR-001。与端点环回的关键差异：采集完全在主进程完成 + * （原生采集线程 → ThreadSafeFunction → 主进程 JS），不经渲染进程， + * 故不实现 handleRendererChunk；产出的块直接交 sink。 + * @ai-context: 目标 PID 解析——desktopCapturer 的 window id 形如 + * `window::0`，取 HWND 匹配原生枚举结果得到进程树根。Chromium 顶层 + * 窗口本身即归属 browser process，故 rootPid 通常等于 pid；仍走 rootPid + * 以覆盖窗口归属子进程的应用。 + * @ai-context: 启动期失败（目标不可采/激活失败）经 onFatal 异步上报， + * 由编排器触发降级到端点环回。 + */ + +import { logger } from '../logger.js'; +import { loadProcessAudioNative, type NativeWindowInfo } from './processAudioNative.js'; +import type { + AudioChunkSink, + AudioProviderStartContext, + AudioSourceProvider, +} from './audioSourceProvider.js'; +import type { AudioSourceKind } from '../../src/lib/capture/audioSourceStrategy.js'; + +/** 从 desktopCapturer 的 window id 中解析 HWND；失败返回 null */ +export function parseHwndFromSourceId(sourceId: string | null): string | null { + if (!sourceId || !sourceId.startsWith('window:')) return null; + const parts = sourceId.split(':'); + if (parts.length < 2 || !parts[1]) return null; + // id 形如 window:395794:0，中间段为十进制 HWND + return parts[1]; +} + +export class ProcessLoopbackProvider implements AudioSourceProvider { + readonly kind: AudioSourceKind = 'process_loopback'; + + private readonly sink: AudioChunkSink; + /** 致命错误上报（编排器据此降级） */ + private readonly onFatal: (message: string) => void; + private capturing = false; + private disposed = false; + + constructor(sink: AudioChunkSink, onFatal: (message: string) => void) { + this.sink = sink; + this.onFatal = onFatal; + } + + async start(ctx: AudioProviderStartContext): Promise { + if (this.capturing || this.disposed) return; + + const native = loadProcessAudioNative(); + if (!native) throw new Error('进程环回原生模块不可用'); + + const targetPid = this.resolveTargetPid(native.listAudioWindows(), ctx.sourceId); + if (targetPid === null) { + throw new Error('无法解析目标窗口所属进程，请改用系统音频采集'); + } + + const result = native.startCapture( + { + pid: targetPid, + sampleRate: ctx.options.sampleRate, + channels: ctx.options.channels, + chunkDurationMs: ctx.options.chunkDurationMs, + }, + (payload) => { + if (this.disposed) return; + if (payload.error) { + logger.warn(`[ProcessLoopback] 采集错误: ${payload.error}`); + this.onFatal(payload.error); + return; + } + if (!payload.audioBuffer) return; + this.sink({ + audioBuffer: payload.audioBuffer, + sampleRate: payload.sampleRate ?? ctx.options.sampleRate, + channels: payload.channels ?? ctx.options.channels, + durationMs: payload.durationMs ?? ctx.options.chunkDurationMs, + }); + }, + ); + + if (!result.ok) throw new Error(result.error || '进程环回启动失败'); + + this.capturing = true; + logger.info( + `[ProcessLoopback] 开始捕获, targetPid=${targetPid}, ` + + `chunkDurationMs=${ctx.options.chunkDurationMs}, ` + + `sampleRate=${ctx.options.sampleRate}, channels=${ctx.options.channels}`, + ); + } + + /** 由窗口源 ID 定位进程树根 PID */ + private resolveTargetPid(windows: NativeWindowInfo[], sourceId: string | null): number | null { + const hwnd = parseHwndFromSourceId(sourceId); + if (!hwnd) return null; + const matched = windows.find((w) => w.hwnd === hwnd); + if (!matched) { + logger.warn(`[ProcessLoopback] 未在窗口列表中找到 HWND=${hwnd}`); + return null; + } + logger.info( + `[ProcessLoopback] 目标窗口="${matched.title}" pid=${matched.pid} ` + + `rootPid=${matched.rootPid} (${matched.rootProcessName})`, + ); + return matched.rootPid; + } + + stop(): void { + if (!this.capturing) return; + this.capturing = false; + const native = loadProcessAudioNative(); + try { + native?.stopCapture(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + logger.warn(`[ProcessLoopback] stopCapture 异常: ${message}`); + } + logger.info('[ProcessLoopback] 停止捕获'); + } + + dispose(): void { + this.stop(); + this.disposed = true; + } +} diff --git a/client/electron/audioCapture.ts b/client/electron/audioCapture.ts index 6521447d..b1ff2653 100644 --- a/client/electron/audioCapture.ts +++ b/client/electron/audioCapture.ts @@ -16,6 +16,8 @@ import type { BrowserWindow } from 'electron'; import { logger } from './logger.js'; import { EndpointLoopbackProvider, listAudioSources } from './audio/endpointLoopbackProvider.js'; +import { ProcessLoopbackProvider } from './audio/processLoopbackProvider.js'; +import { isProcessLoopbackAvailable } from './audio/processAudioNative.js'; import type { AudioCaptureOptions, AudioChunk, @@ -29,6 +31,16 @@ import { type AudioSourcePreference, } from '../src/lib/capture/audioSourceStrategy.js'; +/** + * 进程环回特性开关（Phase 2：默认关）。 + * + * 开启方式：环境变量 ENTROPY_PROCESS_LOOPBACK=1。 + * Phase 3 经内测灰度后改为默认开并改由设置页控制。 + */ +function isProcessLoopbackEnabled(): boolean { + return process.env.ENTROPY_PROCESS_LOOPBACK === '1'; +} + // 保持既有导出路径不变（mediaCaptureHandlers 等调用方无需改动） export { listAudioSources }; export type { AudioCaptureOptions, AudioChunk } from './audio/audioSourceProvider.js'; @@ -74,6 +86,9 @@ export class AudioCapture { private provider: AudioSourceProvider | null = null; /** 本次采集的选源决策（供日志/会话元数据归因） */ private decision: AudioSourceDecision | null = null; + /** 绑定的窗口与源 ID（运行时降级需重建 Provider） */ + private boundWindow: BrowserWindow | null = null; + private boundSourceId: string | null = null; constructor( options: Partial, @@ -117,9 +132,10 @@ export class AudioCapture { if (this.capturing || this.disposed) return; const resolvedSourceId = sourceId ?? null; + // 特性开关关闭时能力恒为不可用，选源必为端点环回（行为与 Phase 0 一致） + const processAvailable = isProcessLoopbackEnabled() && isProcessLoopbackAvailable(); this.decision = selectAudioSource({ - // Phase 0：进程环回尚未接入，能力探测恒为不可用（行为与重构前一致） - capabilities: { processLoopbackAvailable: false }, + capabilities: { processLoopbackAvailable: processAvailable }, sourceId: resolvedSourceId, preference: extras?.preference, microphone: extras?.microphone, @@ -155,6 +171,8 @@ export class AudioCapture { win: BrowserWindow, sourceId: string | null, ): Promise { + this.boundWindow = win; + this.boundSourceId = sourceId; this.provider?.dispose(); this.provider = this.createProvider(kind); await this.provider.start({ window: win, sourceId, options: this.options }); @@ -167,14 +185,39 @@ export class AudioCapture { case 'endpoint_loopback': return new EndpointLoopbackProvider(sink); case 'process_loopback': - // Phase 2 接入；Phase 0 阶段选源不会产生该分支 - throw new Error('进程环回 Provider 尚未接入'); + // 采集中发生的致命错误（如目标进程退出）触发运行时降级 + return new ProcessLoopbackProvider(sink, (message) => { + void this.degradeToEndpoint(message); + }); case 'microphone': // TODO(现场课程): MicrophoneProvider 待实现 throw new Error('麦克风 Provider 尚未实现'); } } + /** + * 运行时降级：采集已开始后才发生的进程环回故障，无缝切到端点环回。 + * 失败时仅记日志：此时已脱离 start 调用栈，抛错无人接收， + * 且上层 watchdog（useClassroomAudio）会在 15s 内提示用户。 + */ + private async degradeToEndpoint(reason: string): Promise { + if (this.disposed || !this.capturing) return; + if (this.provider?.kind !== 'process_loopback') return; + if (!this.boundWindow || this.boundWindow.isDestroyed()) return; + + logger.warn(`[AudioCapture] 进程环回运行中故障（${reason}），切换到端点环回`); + this.decision = { + kind: 'endpoint_loopback', + reason: `进程环回运行中故障后降级：${reason}`, + fallback: null, + }; + try { + await this.startWithKind('endpoint_loopback', this.boundWindow, this.boundSourceId); + } catch (err) { + logger.error('[AudioCapture] 降级到端点环回失败', err); + } + } + /** 统一补时间戳后向消费者分发 */ private emitChunk(data: RendererAudioChunk): void { if (this.disposed) return; @@ -208,6 +251,8 @@ export class AudioCapture { this.stop(); this.provider?.dispose(); this.provider = null; + this.boundWindow = null; + this.boundSourceId = null; this.disposed = true; logger.info('[AudioCapture] 已销毁'); } diff --git a/client/package.json b/client/package.json index ce61d47a..83986d35 100644 --- a/client/package.json +++ b/client/package.json @@ -14,6 +14,8 @@ "preview": "vite preview", "electron:dev": "concurrently -k -n vite,electron \"cross-env ELECTRON_BUILD=1 vite --mode test\" \"wait-on http://localhost:5173 && tsc -p electron/tsconfig.json && cross-env ELECTRON_BUILD=1 NODE_ENV=development VITE_AI_GATEWAY_URL=http://101.37.70.235:8000 electron .\"", "electron:build": "cross-env ELECTRON_BUILD=1 vite build && tsc -p electron/tsconfig.json && electron-builder", + "native:build": "electron-rebuild --module-dir native/process-audio --only @entropydecrease/process-audio", + "native:install": "npm --prefix native/process-audio install --no-audit --no-fund", "test": "vitest run", "test:watch": "vitest", "postinstall": "electron-rebuild -f -w better-sqlite3", From 4006a929a7f5c1cefa3a50dfe12e2ba5990e1418 Mon Sep 17 00:00:00 2001 From: Aparencia Date: Fri, 31 Jul 2026 23:33:15 +0800 Subject: [PATCH 5/5] =?UTF-8?q?feat(audio):=20=E8=BF=9B=E7=A8=8B=E7=8E=AF?= =?UTF-8?q?=E5=9B=9E=E9=BB=98=E8=AE=A4=E5=90=AF=E7=94=A8=20+=20=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E9=A1=B5=E9=9F=B3=E9=A2=91=E6=BA=90=E9=80=89=E6=8B=A9?= =?UTF-8?q?=EF=BC=88Phase=203=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-001 Phase 3：进程环回从灰度转默认启用，并补齐用户可控与可归因链路。 - flag 反转为默认开启，ENTROPY_PROCESS_LOOPBACK=0 保留为应急总闸 （现场排障可不重新发版即关闭） - audioSourcePreference：偏好持久化（localStorage），读写全程静默降级， 绝不因偏好读取失败阻断采集启动；6 例单测覆盖非法值与存储异常 - 设置页新增「课堂音频采集」区块：自动 / 仅目标窗口声音 / 系统全部声音， 各选项标注取舍（干净但可能漏采 vs 不漏采但含杂音） - audio_capture_start 双向打通：入参接收 preference（主进程读不到 localStorage），返回值回传 sourceKind + sourceReason - useAudioRecovery 按源分支：对进程环回不再提示'检查系统默认输出设备' （该源不受输出设备与系统音量影响，属误导），改为提示确认目标窗口在播放； devicechange 自动重启仅端点环回保留 - 生效源经 useClassroomCapture 暴露，供 UI 展示与内测归因 验证：client 479 tests、lint 0 errors、渲染与主进程 tsc 均 0。 --- client/electron/audioCapture.ts | 10 +-- client/electron/mediaCaptureHandlers.ts | 22 ++++- .../classroom/hooks/useAudioRecovery.ts | 40 ++++++--- .../classroom/hooks/useClassroomCapture.ts | 12 ++- .../classroom/hooks/useSessionControl.ts | 24 +++++- .../lib/capture/audioSourcePreference.test.ts | 60 +++++++++++++ .../src/lib/capture/audioSourcePreference.ts | 53 ++++++++++++ client/src/pages/SettingsPage.tsx | 10 ++- .../pages/settings/AudioCaptureSettings.tsx | 86 +++++++++++++++++++ .../ADR-001-audio-capture-process-loopback.md | 22 ++++- 10 files changed, 311 insertions(+), 28 deletions(-) create mode 100644 client/src/lib/capture/audioSourcePreference.test.ts create mode 100644 client/src/lib/capture/audioSourcePreference.ts create mode 100644 client/src/pages/settings/AudioCaptureSettings.tsx diff --git a/client/electron/audioCapture.ts b/client/electron/audioCapture.ts index b1ff2653..1fddc502 100644 --- a/client/electron/audioCapture.ts +++ b/client/electron/audioCapture.ts @@ -32,13 +32,13 @@ import { } from '../src/lib/capture/audioSourceStrategy.js'; /** - * 进程环回特性开关（Phase 2：默认关）。 + * 进程环回特性开关（Phase 3：默认开）。 * - * 开启方式：环境变量 ENTROPY_PROCESS_LOOPBACK=1。 - * Phase 3 经内测灰度后改为默认开并改由设置页控制。 + * 默认开启后用户仍可在设置页选“系统全部声音”强制用端点环回； + * ENTROPY_PROCESS_LOOPBACK=0 作为应急总闸（现场排障时无需重新发版即可关闭）。 */ function isProcessLoopbackEnabled(): boolean { - return process.env.ENTROPY_PROCESS_LOOPBACK === '1'; + return process.env.ENTROPY_PROCESS_LOOPBACK !== '0'; } // 保持既有导出路径不变（mediaCaptureHandlers 等调用方无需改动） @@ -132,7 +132,7 @@ export class AudioCapture { if (this.capturing || this.disposed) return; const resolvedSourceId = sourceId ?? null; - // 特性开关关闭时能力恒为不可用，选源必为端点环回（行为与 Phase 0 一致） + // 应急总闸关闭时能力恒为不可用，选源退回端点环回 const processAvailable = isProcessLoopbackEnabled() && isProcessLoopbackAvailable(); this.decision = selectAudioSource({ capabilities: { processLoopbackAvailable: processAvailable }, diff --git a/client/electron/mediaCaptureHandlers.ts b/client/electron/mediaCaptureHandlers.ts index 1282ebe4..7212c215 100644 --- a/client/electron/mediaCaptureHandlers.ts +++ b/client/electron/mediaCaptureHandlers.ts @@ -10,6 +10,7 @@ import { BrowserWindow, ipcMain } from 'electron'; import { AudioCapture, listAudioSources } from './audioCapture.js'; import type { AudioCaptureOptions, AudioChunk } from './audioCapture.js'; +import type { AudioSourcePreference } from '../src/lib/capture/audioSourceStrategy.js'; import { VideoRecorder } from './videoRecorder.js'; import type { VideoRecordOptions } from './videoRecorder.js'; import { safeHandle, getMainWindowId } from './ipcUtils.js'; @@ -55,7 +56,14 @@ export function registerMediaCaptureHandlers(): void { safeHandle( 'audio_capture_start', - async (event, options?: Partial & { sourceId?: string }) => { + async ( + event, + options?: Partial & { + sourceId?: string; + /** 用户在设置页选择的音频源偏好（主进程读不到 localStorage，由渲染进程传入） */ + preference?: AudioSourcePreference; + }, + ) => { if (activeAudioCapture) { activeAudioCapture.dispose(); activeAudioCapture = null; @@ -79,9 +87,17 @@ export function registerMediaCaptureHandlers(): void { }); try { - await activeAudioCapture.start(senderWin, options?.sourceId); + await activeAudioCapture.start(senderWin, options?.sourceId, { + preference: options?.preference, + }); + const decision = activeAudioCapture.sourceDecision; logger.info('[IPC] audio_capture_start 已启动'); - return { success: true }; + // 回传生效源：渲染进程据此分支诊断文案并写入会话元数据（内测归因） + return { + success: true, + sourceKind: activeAudioCapture.activeSourceKind, + sourceReason: decision?.reason, + }; } catch (err) { const message = err instanceof Error ? err.message : String(err); logger.error('[IPC] audio_capture_start failed:', message); diff --git a/client/src/features/classroom/hooks/useAudioRecovery.ts b/client/src/features/classroom/hooks/useAudioRecovery.ts index 72cc3208..f3adc3b6 100644 --- a/client/src/features/classroom/hooks/useAudioRecovery.ts +++ b/client/src/features/classroom/hooks/useAudioRecovery.ts @@ -2,14 +2,18 @@ * 课堂音频自动恢复 hook（静音诊断 / 设备变更重启） * * @ai-context: 从 useClassroomAudio 拆出的独立恢复层。两条恢复路径： - * ①输出设备不匹配诊断——系统环回只录默认输出设备，视频声音若输出到其他 - * 设备（HDMI/蓝牙）会表现为"音频块正常但持续静音"，检出后提示用户核对； - * ②设备变更重启——默认输出设备切换后环回仍绑定旧设备，devicechange 时 - * 自动 stop→start 重新绑定。 + * ①静音诊断——音频块正常但持续静音，成因随音频源不同（见下）； + * ②设备变更重启——仅端点环回需要（它绑定系统默认输出设备），进程环回 + * 与输出设备无关，切设备时不必重启。 + * @ai-context: 诊断文案必须按 sourceKind 分支（ADR-001）——对进程环回说 + * "请检查默认输出设备"是误导，它根本不受输出设备与系统音量影响，此时 + * 真实成因是目标窗口没在发声或声音来自别的应用。 * @ai-context: 仅依赖 refs 的稳定回调，重启经 restartingRef 互斥防并发。 */ import { useEffect, useRef, useCallback } from 'react'; import type { CaptureMode, SessionStatus } from '@/lib/capture'; +import type { AudioSourceKind } from '@/lib/capture/audioSourceStrategy'; +import { getAudioSourcePreference } from '@/lib/capture/audioSourcePreference'; import { computeChunkRms, SilenceTracker, getDefaultOutputDeviceLabel, subscribeDeviceChange, @@ -25,17 +29,25 @@ interface UseAudioRecoveryOptions { mode: CaptureMode; /** 会话选中窗口的源 ID，重启时沿用同一源 */ audioSourceId?: string | null; + /** 本次会话实际生效的音频源（由 audio_capture_start 回传） */ + sourceKind?: AudioSourceKind | null; onNotify: (type: 'warning' | 'error', message: string) => void; } -export function useAudioRecovery({ status, mode, audioSourceId, onNotify }: UseAudioRecoveryOptions) { +export function useAudioRecovery({ + status, mode, audioSourceId, sourceKind, onNotify, +}: UseAudioRecoveryOptions) { const notifyRef = useRef(onNotify); notifyRef.current = onNotify; const effectiveSourceRef = useRef(audioSourceId ?? undefined); const silenceTrackerRef = useRef(new SilenceTracker()); const restartingRef = useRef(false); + // 生效源用 ref 桥接：静音诊断的监听器依赖数组不含它，避免重订阅丢失计数 + const sourceKindRef = useRef(sourceKind ?? null); + sourceKindRef.current = sourceKind ?? null; const audioEnabled = status === 'capturing' && (mode === 'audio' || mode === 'mixed'); + const isProcessSource = sourceKind === 'process_loopback'; // 会话开始时重置恢复状态（audioSourceId 取会话启动瞬间的快照） useEffect(() => { @@ -53,7 +65,7 @@ export function useAudioRecovery({ status, mode, audioSourceId, onNotify }: UseA await window.electronAPI.invoke('audio_capture_stop'); await new Promise((r) => setTimeout(r, RESTART_CLEANUP_DELAY_MS)); const result = await window.electronAPI.invoke('audio_capture_start', { - ...AUDIO_START_OPTIONS, sourceId, + ...AUDIO_START_OPTIONS, sourceId, preference: getAudioSourcePreference(), }) as { success: boolean; error?: string }; if (result.success) silenceTrackerRef.current.reset(); else console.warn('[useAudioRecovery] 音频捕获重启失败:', result.error); @@ -66,12 +78,20 @@ export function useAudioRecovery({ status, mode, audioSourceId, onNotify }: UseA } }, []); - // 静音诊断：音频块正常但持续无声 → 提示核对系统默认输出设备 + // 静音诊断：音频块正常但持续无声 → 按生效源给出对应成因 useEffect(() => { if (!audioEnabled || !window.electronAPI) return; const off = window.electronAPI.on('audio_capture_chunk', (...args: unknown[]) => { const chunk = args[0] as { audioBuffer: ArrayBuffer }; if (!silenceTrackerRef.current.push(computeChunkRms(chunk.audioBuffer))) return; + + if (sourceKindRef.current === 'process_loopback') { + // 进程环回不受系统音量/输出设备影响，成因只能是目标窗口没在发声 + notifyRef.current('warning', + '持续收到静音音频：当前只采集目标窗口的声音，请确认该窗口正在播放，' + + '或在设置中改为采集「系统全部声音」'); + return; + } void getDefaultOutputDeviceLabel().then((label) => { notifyRef.current('warning', `持续收到静音音频：请确认视频声音正在播放，且输出到系统默认设备${label ? `「${label}」` : ''}（系统音频捕获只能录到默认输出设备的声音）`); @@ -80,9 +100,9 @@ export function useAudioRecovery({ status, mode, audioSourceId, onNotify }: UseA return off; }, [audioEnabled]); - // 设备变更自动重启：重新绑定新的默认输出设备 + // 设备变更自动重启：仅端点环回需要（它绑定系统默认输出设备） useEffect(() => { - if (!audioEnabled || !window.electronAPI) return; + if (!audioEnabled || !window.electronAPI || isProcessSource) return; const unsubscribe = subscribeDeviceChange(() => { console.info('[useAudioRecovery] 检测到音频设备变更，自动重启音频捕获'); void restartCapture(effectiveSourceRef.current).then((ok) => { @@ -91,5 +111,5 @@ export function useAudioRecovery({ status, mode, audioSourceId, onNotify }: UseA }); }); return unsubscribe; - }, [audioEnabled, restartCapture]); + }, [audioEnabled, isProcessSource, restartCapture]); } diff --git a/client/src/features/classroom/hooks/useClassroomCapture.ts b/client/src/features/classroom/hooks/useClassroomCapture.ts index 525e4302..4eb6d839 100644 --- a/client/src/features/classroom/hooks/useClassroomCapture.ts +++ b/client/src/features/classroom/hooks/useClassroomCapture.ts @@ -12,6 +12,7 @@ import { useState, useCallback, useEffect, useRef, useMemo } from 'react'; import { useToast } from '@/components/ui/Toast'; import { CaptureManager } from '@/lib/capture'; +import type { AudioSourceKind } from '@/lib/capture/audioSourceStrategy'; import type { CaptureMode, CaptureSidebarConfig, @@ -50,6 +51,10 @@ export function useClassroomCapture() { // ── 课中重点标记 ── const [bookmarks, setBookmarks] = useState<{ timestamp: number; label?: string }[]>([]); + // 本次会话实际生效的音频源（ADR-001）：由主进程选源后回传， + // 用于诊断文案分支与 UI 展示，避免对进程环回给出“检查输出设备”类误导提示 + const [audioSourceKind, setAudioSourceKind] = useState(null); + const notify = useCallback((type: 'success' | 'warning' | 'error' | 'info', message: string) => { toast({ type, message }); }, [toast]); @@ -107,9 +112,9 @@ export function useClassroomCapture() { captureManager, status, mode, onNotify: notify, }); - // 音频自动恢复：静音诊断 / 窗口源回退环回 / 设备变更重启 + // 音频自动恢复：静音诊断（文案按生效源分支）/ 设备变更重启 useAudioRecovery({ - status, mode, audioSourceId: selectedWindow?.id, onNotify: notify, + status, mode, audioSourceId: selectedWindow?.id, sourceKind: audioSourceKind, onNotify: notify, }); const analysis = useClassroomAnalysis({ @@ -150,6 +155,7 @@ export function useClassroomCapture() { onAnalyzeFull: analysis.handleAnalyze, onMergePartials: analysis.mergePartialNotes, onNotify: (type, message) => notify(type, message), + onAudioSourceResolved: setAudioSourceKind, }); const handleModeChange = useCallback((newMode: CaptureMode) => { @@ -214,6 +220,8 @@ export function useClassroomCapture() { liveTranscripts: events.liveTranscripts, // 音频健康 + VAD audioHealth, vadStats: events.vadStats, + // 本次会话生效的音频源（UI 可见，供内测归因） + audioSourceKind, // 课程上下文 courseMeta, setCourseMeta, aiDetectEnabled, setAiDetectEnabled, // 录制 diff --git a/client/src/features/classroom/hooks/useSessionControl.ts b/client/src/features/classroom/hooks/useSessionControl.ts index 2203df59..33a35290 100644 --- a/client/src/features/classroom/hooks/useSessionControl.ts +++ b/client/src/features/classroom/hooks/useSessionControl.ts @@ -12,6 +12,8 @@ import { useCallback } from 'react'; import { requireGatewayUrl } from '@/lib/ai/config'; import { soundPlayer } from '@/lib/audio/SoundPlayer'; import { analyzePartial } from '@/lib/ai/sessionAnalyzer'; +import { getAudioSourcePreference } from '@/lib/capture/audioSourcePreference'; +import type { AudioSourceKind } from '@/lib/capture/audioSourceStrategy'; import type { CaptureManager, CaptureMode, @@ -28,6 +30,10 @@ import type { interface IPCAudioStartResult { success: boolean; error?: string; + /** 实际生效的音频源（ADR-001 双源选择结果） */ + sourceKind?: AudioSourceKind; + /** 选源理由（含降级说明），供内测问题归因 */ + sourceReason?: string; } interface UseSessionControlOptions { @@ -56,12 +62,14 @@ interface UseSessionControlOptions { onAnalyzeFull: () => void; onMergePartials: (partials: string[], durationMs: number, keyframeCount: number) => Promise; onNotify: (type: 'warning', message: string) => void; + /** 音频源定下后回报（供诊断文案分支与 UI 展示） */ + onAudioSourceResolved?: (kind: AudioSourceKind | null) => void; } export function useSessionControl({ captureManager, selectedWindow, status, setStatus, mode, capturePath, config, courseMeta, frameRestartRef, audioCleanupRef, session, - onAnalyzeVideo, onAnalyzeFull, onMergePartials, onNotify, + onAnalyzeVideo, onAnalyzeFull, onMergePartials, onNotify, onAudioSourceResolved, }: UseSessionControlOptions) { /** 预检 AI 网关连通性（不可用仅提示，不阻断采集） */ const probeGateway = useCallback(async () => { @@ -130,14 +138,22 @@ export function useSessionControl({ if (audioEnabled) { try { - // 方案A：优先以选中窗口为音频源（直采 B站客户端/浏览器等目标应用声音）， - // 窗口级捕获不受支持时主进程会下发环回降级候选，由渲染端自动回退 + // 选源由主进程的 selectAudioSource 决定（ADR-001）：锁定具体窗口时 + // 优先进程环回（隔离其他应用杂音），否则用端点环回（不漏采）； + // 主进程读不到 localStorage，故偏好由渲染进程传入 const audioResult = await window.electronAPI.invoke('audio_capture_start', { chunkDurationMs: 5000, sampleRate: 16000, channels: 1, sourceId: selectedWindow.id, + preference: getAudioSourcePreference(), }) as IPCAudioStartResult; if (!audioResult.success) { console.warn('[useClassroomCapture] Audio start failed:', audioResult.error); + } else { + console.info( + `[useClassroomCapture] 音频源=${audioResult.sourceKind ?? 'unknown'}` + + `（${audioResult.sourceReason ?? '-'}）`, + ); + onAudioSourceResolved?.(audioResult.sourceKind ?? null); } } catch (audioErr) { console.warn('[useClassroomCapture] Audio unavailable:', audioErr); @@ -147,7 +163,7 @@ export function useSessionControl({ setStatus('error'); console.error('[useClassroomCapture] Start failed:', err); } - }, [selectedWindow, setStatus, session, probeGateway, capturePath, captureManager, config, mode, courseMeta]); + }, [selectedWindow, setStatus, session, probeGateway, capturePath, captureManager, config, mode, courseMeta, onAudioSourceResolved]); const handlePause = useCallback(() => { if (status === 'capturing') { diff --git a/client/src/lib/capture/audioSourcePreference.test.ts b/client/src/lib/capture/audioSourcePreference.test.ts new file mode 100644 index 00000000..5d094890 --- /dev/null +++ b/client/src/lib/capture/audioSourcePreference.test.ts @@ -0,0 +1,60 @@ +/** + * @ai-context: 音频源偏好持久化单测。重点覆盖"读取失败/非法值必须回落 + * 到 auto"——偏好读取绝不能阻断采集启动（见 audioSourcePreference 头注）。 + */ +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; +import { + getAudioSourcePreference, + setAudioSourcePreference, + AUDIO_SOURCE_PREFERENCE_KEY, + AUDIO_SOURCE_PREFERENCE_LABELS, +} from './audioSourcePreference'; + +describe('audioSourcePreference', () => { + beforeEach(() => { + localStorage.clear(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('未设置时默认 auto', () => { + expect(getAudioSourcePreference()).toBe('auto'); + }); + + it('可写入并读回三种合法值', () => { + for (const value of ['auto', 'force_process', 'force_endpoint'] as const) { + setAudioSourcePreference(value); + expect(getAudioSourcePreference()).toBe(value); + } + }); + + it('存储中的非法值回落到 auto', () => { + localStorage.setItem(AUDIO_SOURCE_PREFERENCE_KEY, 'force_microphone'); + expect(getAudioSourcePreference()).toBe('auto'); + }); + + it('localStorage 读取抛错时回落到 auto 而非抛出', () => { + vi.spyOn(Storage.prototype, 'getItem').mockImplementation(() => { + throw new Error('storage unavailable'); + }); + expect(() => getAudioSourcePreference()).not.toThrow(); + expect(getAudioSourcePreference()).toBe('auto'); + }); + + it('localStorage 写入抛错时静默降级而非抛出', () => { + vi.spyOn(Storage.prototype, 'setItem').mockImplementation(() => { + throw new Error('quota exceeded'); + }); + expect(() => setAudioSourcePreference('force_process')).not.toThrow(); + }); + + it('每种偏好都有非空的标签与说明（设置页依赖）', () => { + for (const value of ['auto', 'force_process', 'force_endpoint'] as const) { + const { label, hint } = AUDIO_SOURCE_PREFERENCE_LABELS[value]; + expect(label.length).toBeGreaterThan(0); + expect(hint.length).toBeGreaterThan(0); + } + }); +}); diff --git a/client/src/lib/capture/audioSourcePreference.ts b/client/src/lib/capture/audioSourcePreference.ts new file mode 100644 index 00000000..95ccb971 --- /dev/null +++ b/client/src/lib/capture/audioSourcePreference.ts @@ -0,0 +1,53 @@ +/** + * 音频源偏好持久化 + * + * @ai-context: 见 ADR-001。偏好由用户在设置页选择，采集启动时读取并经 + * audio_capture_start 传给主进程参与选源；主进程无法访问 localStorage， + * 故必须由渲染进程读取后传递（与 aiConfig.gatewayUrl 同一模式）。 + * @ai-context: 读写全程静默降级——localStorage 不可用（隐私模式/配额满） + * 时回落到 'auto'，绝不因偏好读取失败阻断采集启动。 + */ + +import type { AudioSourcePreference } from './audioSourceStrategy'; + +/** localStorage key（不可改，否则用户设置丢失） */ +export const AUDIO_SOURCE_PREFERENCE_KEY = 'keban_audio_source_preference'; + +const VALID: readonly AudioSourcePreference[] = ['auto', 'force_process', 'force_endpoint']; + +/** 读取音频源偏好；无效或读取失败均返回 'auto' */ +export function getAudioSourcePreference(): AudioSourcePreference { + try { + const raw = localStorage.getItem(AUDIO_SOURCE_PREFERENCE_KEY); + if (raw && (VALID as readonly string[]).includes(raw)) { + return raw as AudioSourcePreference; + } + } catch { /* 静默降级 */ } + return 'auto'; +} + +/** 保存音频源偏好 */ +export function setAudioSourcePreference(preference: AudioSourcePreference): void { + try { + localStorage.setItem(AUDIO_SOURCE_PREFERENCE_KEY, preference); + } catch { /* 静默降级 */ } +} + +/** 偏好项的用户可读说明（设置页展示用） */ +export const AUDIO_SOURCE_PREFERENCE_LABELS: Record< + AudioSourcePreference, + { label: string; hint: string } +> = { + auto: { + label: '自动（推荐）', + hint: '锁定具体窗口时只采该窗口的声音（隔离其他应用杂音）；采集整屏时采系统全部声音', + }, + force_process: { + label: '仅目标窗口声音', + hint: '始终只采集目标窗口所在应用的声音，不受系统音量影响；换用其他播放器时可能采不到', + }, + force_endpoint: { + label: '系统全部声音', + hint: '采集电脑正在播放的所有声音，不会漏采；其他应用的提示音也会被一并录入', + }, +}; diff --git a/client/src/pages/SettingsPage.tsx b/client/src/pages/SettingsPage.tsx index b99b4664..e0452198 100644 --- a/client/src/pages/SettingsPage.tsx +++ b/client/src/pages/SettingsPage.tsx @@ -10,6 +10,7 @@ import SoundSettings from './settings/SoundSettings'; import ModeSettings from './settings/ModeSettings'; import ShortcutSettings from './settings/ShortcutSettings'; import FlashcardSettings from './settings/FlashcardSettings'; +import AudioCaptureSettings from './settings/AudioCaptureSettings'; // 延迟组：有网络/IPC/DB 操作的重组件 const AIProviderSettings = lazy(() => import('./settings/AIProviderSettings')); @@ -126,14 +127,17 @@ export default function SettingsPage() { + + + }> - + - + - + diff --git a/client/src/pages/settings/AudioCaptureSettings.tsx b/client/src/pages/settings/AudioCaptureSettings.tsx new file mode 100644 index 00000000..7ca10692 --- /dev/null +++ b/client/src/pages/settings/AudioCaptureSettings.tsx @@ -0,0 +1,86 @@ +/** + * @ai-context: 设置页组件：AudioCaptureSettings。课堂助手音频源偏好选择。 + * 选项语义见 ADR-001 的双源互补设计：进程环回"干净但可能漏采"、 + * 端点环回"不漏采但含全部系统声音"，故不设默认优劣，交由用户按场景选。 + * @ai-context: 偏好写 localStorage，采集启动时由 useSessionControl 读取并 + * 经 IPC 传给主进程（主进程无法访问 localStorage）。 + */ +import { useState, useCallback } from 'react'; +import { Card } from '@/components/ui'; +import { useToast } from '@/components/ui/Toast'; +import { cn } from '@/lib/utils'; +import { soundPlayer } from '@/lib/audio/SoundPlayer'; +import { Volume2, Check } from 'lucide-react'; +import type { AudioSourcePreference } from '@/lib/capture/audioSourceStrategy'; +import { + getAudioSourcePreference, + setAudioSourcePreference, + AUDIO_SOURCE_PREFERENCE_LABELS, +} from '@/lib/capture/audioSourcePreference'; + +const OPTIONS: AudioSourcePreference[] = ['auto', 'force_process', 'force_endpoint']; + +/** + * 课堂助手音频采集设置 + * + * 决定采集"仅目标窗口的声音"还是"系统全部声音"： + * 前者可隔离 QQ/微信提示音等杂音且不受系统音量影响， + * 后者不会漏采跨应用的声音。 + */ +export default function AudioCaptureSettings() { + const { toast } = useToast(); + const [preference, setPreference] = useState(getAudioSourcePreference); + + const handleSelect = useCallback((next: AudioSourcePreference) => { + if (next === preference) return; + soundPlayer.play('ui_toggle_on'); + setPreference(next); + setAudioSourcePreference(next); + toast({ type: 'success', message: '音频采集方式已更新，下次开始采集时生效' }); + }, [preference, toast]); + + return ( + +
+
+ +
+
+

课堂音频采集

+

+ 决定课堂助手采集哪些声音。仅 Windows 10 2004 及以上支持按窗口采集， + 不支持时会自动使用系统声音。 +

+
+
+ +
+ {OPTIONS.map((option) => { + const { label, hint } = AUDIO_SOURCE_PREFERENCE_LABELS[option]; + const active = preference === option; + return ( + + ); + })} +
+ + ); +} diff --git a/docs/adr/ADR-001-audio-capture-process-loopback.md b/docs/adr/ADR-001-audio-capture-process-loopback.md index c8a850b3..9b5e8044 100644 --- a/docs/adr/ADR-001-audio-capture-process-loopback.md +++ b/docs/adr/ADR-001-audio-capture-process-loopback.md @@ -2,7 +2,7 @@ ## 状态 -已接受（Phase 1 可行性验证已通过，Phase 0/2 待实施） +已接受（Phase 0/1/2/3 已实施；线下课堂麦克风源 Phase 4 待做） ## 日期 @@ -125,6 +125,26 @@ Phase 1 spike 实测结果（`client/native/process-audio/`，全部通过）： 目标窗口关闭、采集中降级触发、采集线程 30 分钟长跑 - 不回归项：client 与 ai-gateway 现有测试全绿 +## 实施记录 + +| 阶段 | 内容 | 状态 | +|---|---|---| +| Phase 1 | 原生模块可行性验证（`client/native/process-audio/`） | ✅ 全部验收通过 | +| Phase 0 | `AudioSourceProvider` 抽象层 + 选源策略纯函数（12 例单测） | ✅ 零行为变更 | +| Phase 2 | 流式采集（采集线程 + ThreadSafeFunction）、Provider 接入、构建链路（`native:build` / electron-builder / CI 编译） | ✅ flag 曾默认关 | +| Phase 3 | 偏好持久化 + 设置页「课堂音频采集」、诊断文案按源分支、生效源经 IPC 回传并在 UI 可见、flag 默认开启 | ✅ 本阶段 | +| Phase 4 | 线下课堂 `MicrophoneProvider`（视觉轨方案另案） | ⏳ 待做 | + +Phase 3 的关键取舍： + +- **flag 反转为默认开启**，`ENTROPY_PROCESS_LOOPBACK=0` 保留为应急总闸—— + 现场排障可不重新发版即关闭该能力 +- **诊断文案必须按源分支**：对进程环回提示"请检查系统默认输出设备"是误导 + （它不受输出设备与系统音量影响），此时真实成因是目标窗口未发声 +- **devicechange 自动重启仅端点环回保留**：进程环回与输出设备解耦，无需重绑 +- 生效源与选源理由随 `audio_capture_start` 返回值回传渲染进程，写入日志与 + UI，供内测问题归因 + ## 相关决策 - 暂无前置 ADR（本项目首个 ADR）