diff --git a/.changeset/add-new-calls.md b/.changeset/add-new-calls.md new file mode 100644 index 000000000..05125226e --- /dev/null +++ b/.changeset/add-new-calls.md @@ -0,0 +1,7 @@ +--- +default: minor +--- + +# Add experimental new calls + +Introduce an opt-in new call system behind a single experimental setting. It uses LiveKit JS on web and desktop and native LiveKit on supported mobile devices, with Element Call remaining the fallback when the setting is off. diff --git a/.github/workflows/tauri-build.yml b/.github/workflows/tauri-build.yml index fac99cb0f..0b2d29fad 100644 --- a/.github/workflows/tauri-build.yml +++ b/.github/workflows/tauri-build.yml @@ -461,6 +461,10 @@ jobs: shell: bash run: mise -y run tauri:setup:ios + - name: Download iOS frameworks + shell: bash + run: pnpm ios:frameworks + - name: Symlink icons to the ios folder shell: bash run: mise run icons:symlink --write --force diff --git a/.ignore b/.ignore new file mode 100644 index 000000000..b2f5e7e8a --- /dev/null +++ b/.ignore @@ -0,0 +1,9 @@ +!.slim/deepwork/ +!.slim/deepwork/** + +# BEGIN oh-my-opencode-slim worktrees +!.slim/ +!.slim/worktrees.json +!.slim/worktrees/ +!.slim/worktrees/** +# END oh-my-opencode-slim worktrees diff --git a/package.json b/package.json index e40a6aefe..864342332 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "typecheck": "tsc", "tauri": "tauri", "tauri:cef": "node scripts/tauri.js cef", + "ios:frameworks": "node scripts/download-ios-frameworks.js", "tauri:wry": "node scripts/tauri.js wry", "test": "vitest", "test:ui": "vitest --ui", @@ -46,6 +47,7 @@ "@choochmeque/tauri-plugin-sharekit-api": "0.4.0-rc.5", "@fontsource-variable/nunito": "5.2.7", "@fontsource/space-mono": "5.2.9", + "@livekit/components-react": "2.9.23", "@lottiefiles/dotlottie-react": "^0.12.0", "@noble/hashes": "^2.2.0", "@phosphor-icons/react": "^2.1.10", @@ -97,6 +99,7 @@ "leaflet": "1.9.4", "linkify-react": "^4.3.3", "linkifyjs": "^4.3.3", + "livekit-client": "2.21.0", "marked": "^18.0.5", "matrix-js-sdk": "42.0.0", "matrix-widget-api": "^1.17.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 73d8ac5e0..3deb3da51 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,6 +43,9 @@ importers: '@fontsource/space-mono': specifier: 5.2.9 version: 5.2.9 + '@livekit/components-react': + specifier: 2.9.23 + version: 2.9.23(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tslib@2.8.1) '@lottiefiles/dotlottie-react': specifier: ^0.12.0 version: 0.12.3(react@18.3.1) @@ -196,6 +199,9 @@ importers: linkifyjs: specifier: ^4.3.3 version: 4.3.3 + livekit-client: + specifier: 2.21.0 + version: 2.21.0(@types/dom-mediacapture-record@1.0.22) marked: specifier: ^18.0.5 version: 18.0.5 @@ -952,6 +958,9 @@ packages: resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==} hasBin: true + '@bufbuild/protobuf@1.10.1': + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==} + '@choochmeque/tauri-plugin-notifications-api@0.5.0-rc.11': resolution: {integrity: sha512-TSuf8HQ4Ayp9IKKB5v2yTdgKMa/I0bqbcACj0IZ46CSpzn/HKner/BKG7rM8tGOarEcyBd/VXnDhK5cJA/YMtw==} @@ -1401,6 +1410,15 @@ packages: '@noble/hashes': optional: true + '@floating-ui/core@1.8.0': + resolution: {integrity: sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==} + + '@floating-ui/dom@1.7.6': + resolution: {integrity: sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==} + + '@floating-ui/utils@0.2.12': + resolution: {integrity: sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==} + '@fontsource-variable/nunito@5.2.7': resolution: {integrity: sha512-2N8QhatkyKgSUbAGZO2FYLioxA32+RyI1EplVLawbpkGjUeui9Qg9VMrpkCaik1ydjFjfLV+kzQ0cGEsMrMenQ==} @@ -1622,6 +1640,32 @@ packages: '@kwsites/file-exists@1.1.1': resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} + '@livekit/components-core@0.12.14': + resolution: {integrity: sha512-6OKP/1Ok2fCZewDLKd3SzaTb7KvfZl/6hjggI+TgUdhp0Y7HBg3+tHA7YmhZRc0BO8wIyVy4VgTPn7qkLHt2nQ==} + engines: {node: '>=18'} + peerDependencies: + livekit-client: ^2.20.1 + tslib: ^2.6.2 + + '@livekit/components-react@2.9.23': + resolution: {integrity: sha512-clO+0g/u3YBpuOvnAjUSAJBH/o7w+RpUAseswjiSML9rHrXlTUhwBNm8LPk+qN5GOU3A6lzsNnFYVpUHUBVOkg==} + engines: {node: '>=18'} + peerDependencies: + '@livekit/krisp-noise-filter': ^0.2.12 || ^0.3.0 || ^0.4.0 + livekit-client: ^2.20.1 + react: '>=18' + react-dom: '>=18' + tslib: ^2.6.2 + peerDependenciesMeta: + '@livekit/krisp-noise-filter': + optional: true + + '@livekit/mutex@1.1.1': + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} + + '@livekit/protocol@1.50.4': + resolution: {integrity: sha512-L1uggNQAqyY21smQY8AllyOYbcv9Me9TaxwuLytL1R8ck9nbYPmQLNwEDi3pOFGAMa5F8I2nUi2Jc59W5awxlA==} + '@lottiefiles/dotlottie-react@0.12.3': resolution: {integrity: sha512-b0k0Lakj2hmhyIkwJSpKySh6xoelpgWRaijyrCb6fraOCnzuitKglVTtYs0VWf72rk+2aSmC/0IK8j8wruVfOw==} peerDependencies: @@ -3101,6 +3145,9 @@ packages: '@types/dockerode@4.0.1': resolution: {integrity: sha512-cmUpB+dPN955PxBEuXE3f6lKO1hHiIGYJA46IVF3BJpNsZGvtBDcRnlrHYHtOH/B6vtDOyl2kZ2ShAu3mgc27Q==} + '@types/dom-mediacapture-record@1.0.22': + resolution: {integrity: sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==} + '@types/estree@0.0.39': resolution: {integrity: sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==} @@ -4346,6 +4393,9 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + jose@6.2.4: + resolution: {integrity: sha512-N8acGzVsQy6M/fjFcxtysNc4Q379TcM5dM/qKkNtsHFji88yANnXTr7BLeP75iPnFwBfQzM/jg2BZ9+HZrHCZA==} + jotai@2.20.1: resolution: {integrity: sha512-dnuKfU/GLi8B28RRMjQ3AfoN7kfzP8o41+AX2FmITZqEMY8PHnjABq+VkEooomLwYaGjda+pgy0yFSjaHX/ZPg==} engines: {node: '>=12.20.0'} @@ -4516,6 +4566,11 @@ packages: linkifyjs@4.3.3: resolution: {integrity: sha512-P8aEP5U/D1/IlTY2OeYsErdwh9bGuLE30NcXtKEjgdHcahveQoQwM2yZNsioQHsWFz0P7KKudisbrzCgR0sDHg==} + livekit-client@2.21.0: + resolution: {integrity: sha512-RBUhPkV/sl1nzl8lokVlK5uATPwn0AlsudCBZXissw/kDl9yz8ac4pNJ43iPpMVoHOeBYH/BZ3vUC1adqa/zFQ==} + peerDependencies: + '@types/dom-mediacapture-record': ^1 + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} @@ -4532,6 +4587,10 @@ packages: lodash@4.18.1: resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} + loglevel@1.9.1: + resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==} + engines: {node: '>= 0.6.0'} + loglevel@1.9.2: resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==} engines: {node: '>= 0.6.0'} @@ -5061,6 +5120,9 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + safe-array-concat@1.1.4: resolution: {integrity: sha512-wtZlHyOje6OZTGqAoaDKxFkgRtkF9CnHAVnCHKfuj200wAgL+bSJhdsCD2l0Qx/2ekEXjPWcyKkfGb5CPboslg==} engines: {node: '>=0.4'} @@ -5092,10 +5154,17 @@ packages: scroll-into-view-if-needed@3.1.0: resolution: {integrity: sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==} + sdp-transform@2.15.0: + resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==} + hasBin: true + sdp-transform@3.0.0: resolution: {integrity: sha512-gfYVRGxjHkGF2NPeUWHw5u6T/KGFtS5/drPms73gaSuMaVHKCY3lpLnGDfswVQO0kddeePoti09AwhYP4zA8dQ==} hasBin: true + sdp@3.2.2: + resolution: {integrity: sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -5441,6 +5510,9 @@ packages: resolution: {integrity: sha512-phPGCwqr2+Qo0fwniCE8e4pKnGu/yFb5nD5Y8bf0EEeiI5GklnACYA9GFy/DrAeRrKHXvHn+1SUsOWgJp6RO+g==} engines: {node: '>= 0.4'} + typed-emitter@2.1.0: + resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -5519,6 +5591,12 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + usehooks-ts@3.1.1: + resolution: {integrity: sha512-I4diPp9Cq6ieSUH2wu+fDAVQO43xwtulo+fKEidHUwZPnYImbtkTjzIJYcDcJqxgmX31GVqNFURodvcgHcW0pA==} + engines: {node: '>=16.15.0'} + peerDependencies: + react: ^16.8.0 || ^17 || ^18 || ^19 || ^19.0.0-rc + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -5729,6 +5807,10 @@ packages: resolution: {integrity: sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==} engines: {node: '>=20'} + webrtc-adapter@9.0.6: + resolution: {integrity: sha512-CHbl2ZQbxx164IgWRgzJno4hWtM4tFbRam1QfI3Yxhs3w/DvqluVxVWeXs3oL5/fbGkSNLKo0Ty5MgUWceNhog==} + engines: {node: '>=6.0.0', npm: '>=3.10.0'} + whatwg-mimetype@5.0.0: resolution: {integrity: sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==} engines: {node: '>=20'} @@ -6648,6 +6730,8 @@ snapshots: dependencies: css-tree: 3.2.1 + '@bufbuild/protobuf@1.10.1': {} + '@choochmeque/tauri-plugin-notifications-api@0.5.0-rc.11': dependencies: '@tauri-apps/api': 2.11.1 @@ -6929,6 +7013,17 @@ snapshots: optionalDependencies: '@noble/hashes': 2.2.0 + '@floating-ui/core@1.8.0': + dependencies: + '@floating-ui/utils': 0.2.12 + + '@floating-ui/dom@1.7.6': + dependencies: + '@floating-ui/core': 1.8.0 + '@floating-ui/utils': 0.2.12 + + '@floating-ui/utils@0.2.12': {} + '@fontsource-variable/nunito@5.2.7': {} '@fontsource/space-mono@5.2.9': {} @@ -7110,6 +7205,32 @@ snapshots: transitivePeerDependencies: - supports-color + '@livekit/components-core@0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)': + dependencies: + '@floating-ui/dom': 1.7.6 + livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + loglevel: 1.9.1 + rxjs: 7.8.2 + tslib: 2.8.1 + + '@livekit/components-react@2.9.23(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tslib@2.8.1)': + dependencies: + '@livekit/components-core': 0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1) + clsx: 2.1.1 + events: 3.3.0 + jose: 6.2.4 + livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + tslib: 2.8.1 + usehooks-ts: 3.1.1(react@18.3.1) + + '@livekit/mutex@1.1.1': {} + + '@livekit/protocol@1.50.4': + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@lottiefiles/dotlottie-react@0.12.3(react@18.3.1)': dependencies: '@lottiefiles/dotlottie-web': 0.40.1 @@ -8165,6 +8286,8 @@ snapshots: '@types/node': 24.13.2 '@types/ssh2': 1.15.5 + '@types/dom-mediacapture-record@1.0.22': {} + '@types/estree@0.0.39': {} '@types/estree@1.0.9': {} @@ -9577,6 +9700,8 @@ snapshots: jiti@2.7.0: {} + jose@6.2.4: {} + jotai@2.20.1(@babel/core@7.29.7)(@babel/template@7.29.7)(@types/react@18.3.31)(react@18.3.1): optionalDependencies: '@babel/core': 7.29.7 @@ -9722,6 +9847,19 @@ snapshots: linkifyjs@4.3.3: {} + livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22): + dependencies: + '@livekit/mutex': 1.1.1 + '@livekit/protocol': 1.50.4 + '@types/dom-mediacapture-record': 1.0.22 + events: 3.3.0 + jose: 6.2.4 + loglevel: 1.9.2 + sdp-transform: 2.15.0 + tslib: 2.8.1 + typed-emitter: 2.1.0 + webrtc-adapter: 9.0.6 + locate-path@6.0.0: dependencies: p-locate: 5.0.0 @@ -9734,6 +9872,8 @@ snapshots: lodash@4.18.1: {} + loglevel@1.9.1: {} + loglevel@1.9.2: {} long@5.3.2: {} @@ -10389,6 +10529,10 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.62.0 fsevents: 2.3.3 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + safe-array-concat@1.1.4: dependencies: call-bind: 1.0.9 @@ -10426,8 +10570,12 @@ snapshots: dependencies: compute-scroll-into-view: 3.1.1 + sdp-transform@2.15.0: {} + sdp-transform@3.0.0: {} + sdp@3.2.2: {} + semver@6.3.1: {} semver@7.8.4: {} @@ -10904,6 +11052,10 @@ snapshots: possible-typed-array-names: 1.1.0 reflect.getprototypeof: 1.0.10 + typed-emitter@2.1.0: + optionalDependencies: + rxjs: 7.8.2 + typescript@5.9.3: {} ua-is-frozen@0.1.2: {} @@ -10966,6 +11118,11 @@ snapshots: dependencies: react: 18.3.1 + usehooks-ts@3.1.1(react@18.3.1): + dependencies: + lodash.debounce: 4.0.8 + react: 18.3.1 + util-deprecate@1.0.2: {} uuid@10.0.0: {} @@ -11119,6 +11276,10 @@ snapshots: webidl-conversions@8.0.1: {} + webrtc-adapter@9.0.6: + dependencies: + sdp: 3.2.2 + whatwg-mimetype@5.0.0: {} whatwg-url@16.0.1(@noble/hashes@2.2.0): diff --git a/scripts/download-ios-frameworks.js b/scripts/download-ios-frameworks.js new file mode 100644 index 000000000..143d4d624 --- /dev/null +++ b/scripts/download-ios-frameworks.js @@ -0,0 +1,150 @@ +#!/usr/bin/env node +//MISE description="Download and verify the iOS XCFrameworks used by tauri-plugin-livekit-mobile" +// +// iOS-only: LiveKit ships the WebRTC engine and the Rust UniFFI bindings as +// binary XCFrameworks. The app links them via `bundle.iOS.frameworks` in +// src-tauri/tauri.conf.json, so they must exist before `tauri ios build`. +// Binaries are not committed; this script provisions them reproducibly into +// the gitignored src-tauri/Frameworks/ directory. +// +// Safety properties: +// - every download is SHA-256 verified against a pinned digest before use +// (fail closed: a checksum mismatch changes nothing on disk); +// - extraction happens in a temp directory and is moved into place with a +// single rename, so a failure never leaves a partial install; +// - re-running is a no-op when the installed frameworks already match the +// pinned digests (see Frameworks/.manifest.json). +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import process from 'node:process'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { PrefixedLogger } from './utils/console-style.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const log = new PrefixedLogger('[ios-frameworks]'); + +const FRAMEWORKS_DIR = resolve(__dirname, '../src-tauri/Frameworks'); +const MANIFEST_PATH = join(FRAMEWORKS_DIR, '.manifest.json'); + +const FRAMEWORKS = [ + { + name: 'LiveKitWebRTC', + url: 'https://github.com/livekit/webrtc-xcframework/releases/download/144.7559.11/LiveKitWebRTC.xcframework.zip', + sha256: '07c5caf718058af3c528dcabd257298c40e5a8527e4fb9f47c48336ba5899853', + }, + { + name: 'RustLiveKitUniFFI', + url: 'https://github.com/livekit/livekit-uniffi-xcframework/releases/download/0.0.6/RustLiveKitUniFFI.xcframework.zip', + sha256: '0d3f2ce159a224c728f8b131068d53bbf9b13d968cda0edc68a6a2290f2651ed', + }, +]; + +function sha256Hex(buffer) { + return createHash('sha256').update(buffer).digest('hex'); +} + +async function readManifest() { + try { + return JSON.parse(await readFile(MANIFEST_PATH, 'utf8')); + } catch { + return {}; + } +} + +async function writeManifest(manifest) { + await writeFile(MANIFEST_PATH, `${JSON.stringify(manifest, null, 2)}\n`); +} + +// XCFramework zips contain symlinks; use the platform unzip tool rather than +// adding a JS dependency. iOS builds are macOS-only, so ditto is always +// available in practice; unzip is the fallback. +function findExtractor() { + for (const tool of ['ditto', 'unzip']) { + if (!spawnSync(tool, ['--help'], { stdio: 'ignore' }).error) return tool; + } + throw new Error('Neither ditto nor unzip is available to extract XCFramework archives'); +} + +function extractZip(zipPath, destination) { + const tool = findExtractor(); + const args = + tool === 'ditto' ? ['-x', '-k', zipPath, destination] : ['-q', zipPath, '-d', destination]; + const result = spawnSync(tool, args, { stdio: 'inherit' }); + if (result.status !== 0) { + throw new Error(`${tool} failed to extract ${zipPath} (exit ${result.status})`); + } +} + +async function provision(framework, manifest) { + const { name, url, sha256 } = framework; + const target = join(FRAMEWORKS_DIR, `${name}.xcframework`); + + if (manifest[name] === sha256 && existsSync(join(target, 'Info.plist'))) { + log.info(`${name}.xcframework already installed (verified), skipping`); + return; + } + + log.info(`Downloading ${name} from ${url}`); + const response = await fetch(url, { redirect: 'follow' }); + if (!response.ok) { + throw new Error(`Failed to download ${name}: HTTP ${response.status} ${response.statusText}`); + } + const zipBuffer = Buffer.from(await response.arrayBuffer()); + + const actual = sha256Hex(zipBuffer); + if (actual !== sha256) { + throw new Error( + `Checksum mismatch for ${name}: expected ${sha256}, got ${actual}. Refusing to install.` + ); + } + log.info(`Verified SHA-256 for ${name}`); + + // Extract to a temp directory on the same filesystem so the final rename + // into Frameworks/ is atomic and a failure touches nothing installed. + const tempDir = await mkdtemp(join(FRAMEWORKS_DIR, '.tmp-')); + try { + const zipPath = join(tempDir, `${name}.zip`); + await writeFile(zipPath, zipBuffer); + extractZip(zipPath, tempDir); + + const extracted = join(tempDir, `${name}.xcframework`); + if (!existsSync(join(extracted, 'Info.plist'))) { + throw new Error( + `Downloaded archive for ${name} did not contain ${name}.xcframework; refusing to install.` + ); + } + + // Swap the verified framework into place. The previous install, if any, + // is moved aside (not deleted) until the new one is in position. + if (existsSync(target)) { + await rename(target, join(tempDir, '__replaced__')); + } + await rename(extracted, target); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + + manifest[name] = sha256; + await writeManifest(manifest); + log.info(`Installed ${name}.xcframework`); +} + +async function main() { + await mkdir(FRAMEWORKS_DIR, { recursive: true }); + const manifest = await readManifest(); + // Deliberately sequential: each framework is verified, installed, and + // recorded in the manifest as its own atomic step, so a failure on a later + // framework never disturbs an earlier verified install. + for (const framework of FRAMEWORKS) { + // oxlint-disable-next-line no-await-in-loop + await provision(framework, manifest); + } +} + +main().catch((error) => { + log.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/src-tauri/.gitignore b/src-tauri/.gitignore index 489be3277..dd5680a87 100644 --- a/src-tauri/.gitignore +++ b/src-tauri/.gitignore @@ -2,3 +2,6 @@ # will have compiled files and executables /target/ /gen/schemas + +# Downloaded by scripts/download-ios-frameworks.js (pinned, checksummed binaries) +/Frameworks/ diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index c33e2e7ff..fc6944632 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -5411,6 +5411,7 @@ dependencies = [ "tauri-plugin-fs", "tauri-plugin-global-shortcut", "tauri-plugin-http", + "tauri-plugin-livekit-mobile", "tauri-plugin-log", "tauri-plugin-notifications", "tauri-plugin-opener", @@ -6717,6 +6718,20 @@ dependencies = [ "urlpattern", ] +[[package]] +name = "tauri-plugin-livekit-mobile" +version = "0.2.0" +source = "git+https://github.com/SableClient/tauri-plugin-livekit-mobile.git?rev=b936c9ca87b2b845e4cfff8cd846028e45005d95#b936c9ca87b2b845e4cfff8cd846028e45005d95" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "thiserror 2.0.19", + "tokio", +] + [[package]] name = "tauri-plugin-log" version = "2.9.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index ee6cfdbb3..8cdc48092 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -124,6 +124,7 @@ tauri-plugin-notifications = { git = "https://github.com/SableClient/tauri-plugi ] } tauri-plugin-edge-to-edge = { git = "https://github.com/SableClient/tauri-plugin-edge-to-edge.git", rev = "33c6116c27be28c06df5a9d02231ecc5fdeb93c5" } tauri-plugin-sharekit = { git = "https://github.com/Choochmeque/tauri-plugin-sharekit", rev = "9f2b4c5d8a4f0ab910d900ba234ed8bae0ab854b" } +tauri-plugin-livekit-mobile = { git = "https://github.com/SableClient/tauri-plugin-livekit-mobile.git", rev = "b936c9ca87b2b845e4cfff8cd846028e45005d95" } [target.'cfg(target_os = "ios")'.dependencies] objc2 = "0.6" diff --git a/src-tauri/Info.plist b/src-tauri/Info.plist index 99b9ecb70..e1425ba12 100644 --- a/src-tauri/Info.plist +++ b/src-tauri/Info.plist @@ -10,5 +10,10 @@ Sable needs camera access for video calls. NSMicrophoneUsageDescription Sable needs microphone access for voice and video calls. + UIBackgroundModes + + audio + voip + diff --git a/src-tauri/capabilities/mobile.json b/src-tauri/capabilities/mobile.json index e5500a57d..1cb676dbd 100644 --- a/src-tauri/capabilities/mobile.json +++ b/src-tauri/capabilities/mobile.json @@ -7,6 +7,7 @@ "core:window:allow-set-badge-count", "deep-link:default", "edge-to-edge:default", + "livekit-mobile:default", "notifications:default", "sharekit:default" ] diff --git a/src-tauri/gen/android/app/build.gradle.kts b/src-tauri/gen/android/app/build.gradle.kts index 10397a780..5d277fb4f 100644 --- a/src-tauri/gen/android/app/build.gradle.kts +++ b/src-tauri/gen/android/app/build.gradle.kts @@ -61,6 +61,11 @@ android { ) } } + compileOptions { + // tauri-plugin-livekit-mobile requires it: LiveKit/WebRTC call java.time + // APIs that only exist from API 26, and minSdk here is 24. + isCoreLibraryDesugaringEnabled = true + } kotlinOptions { jvmTarget = "1.8" } @@ -74,10 +79,21 @@ rust { } dependencies { + // livekit-android pulls protobuf-javalite while tink (via unifiedpush connector) + // pulls protobuf-java, and the two ship the same com.google.protobuf classes. + // Only the full runtime has the RuntimeVersion class tink's gencode needs, so it + // wins; it is a superset of the lite classes livekit's gencode references. + modules { + module("com.google.protobuf:protobuf-javalite") { + replacedBy("com.google.protobuf:protobuf-java") + } + } + implementation("androidx.webkit:webkit:1.14.0") implementation("androidx.appcompat:appcompat:1.7.1") implementation("androidx.activity:activity-ktx:1.10.1") implementation("com.google.android.material:material:1.12.0") + coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.1.5") testImplementation("junit:junit:4.13.2") androidTestImplementation("androidx.test.ext:junit:1.1.4") androidTestImplementation("androidx.test.espresso:espresso-core:3.5.0") diff --git a/src-tauri/gen/android/build.gradle.kts b/src-tauri/gen/android/build.gradle.kts index a91af1aef..d5b85e453 100644 --- a/src-tauri/gen/android/build.gradle.kts +++ b/src-tauri/gen/android/build.gradle.kts @@ -14,6 +14,9 @@ allprojects { repositories { google() mavenCentral() + // livekit-android pulls com.github.davidliu:audioswitch, published only + // on JitPack. + maven { setUrl("https://jitpack.io") } } } diff --git a/src-tauri/ios-entitlements/sable.debug.entitlements b/src-tauri/ios-entitlements/sable.debug.entitlements new file mode 100644 index 000000000..6631ffa6f --- /dev/null +++ b/src-tauri/ios-entitlements/sable.debug.entitlements @@ -0,0 +1,6 @@ + + + + + + diff --git a/src-tauri/ios-entitlements/sable.release.entitlements b/src-tauri/ios-entitlements/sable.release.entitlements new file mode 100644 index 000000000..723ff878d --- /dev/null +++ b/src-tauri/ios-entitlements/sable.release.entitlements @@ -0,0 +1,12 @@ + + + + + aps-environment + development + com.apple.security.application-groups + + group.moe.sable.client + + + diff --git a/src-tauri/ios-entitlements/sable.share-extension.debug.entitlements b/src-tauri/ios-entitlements/sable.share-extension.debug.entitlements new file mode 100644 index 000000000..6631ffa6f --- /dev/null +++ b/src-tauri/ios-entitlements/sable.share-extension.debug.entitlements @@ -0,0 +1,6 @@ + + + + + + diff --git a/src-tauri/ios-entitlements/sable.share-extension.release.entitlements b/src-tauri/ios-entitlements/sable.share-extension.release.entitlements new file mode 100644 index 000000000..6707d84a1 --- /dev/null +++ b/src-tauri/ios-entitlements/sable.share-extension.release.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.application-groups + + group.moe.sable.client + + + diff --git a/src-tauri/ios-project.yml b/src-tauri/ios-project.yml index 07080a98b..71f67f721 100644 --- a/src-tauri/ios-project.yml +++ b/src-tauri/ios-project.yml @@ -60,9 +60,7 @@ targets: # APPLICATION_EXTENSION_API_ONLY must stay unset: NO fails Xcode's # validation, YES breaks UIApplication.open in ShareViewController. SKIP_INSTALL: YES - {{#if apple.development-team}} - DEVELOPMENT_TEAM: {{apple.development-team}} - {{/if}} + DEVELOPMENT_TEAM: PK66NJM372 {{app.name}}_iOS: type: application platform: iOS @@ -92,7 +90,7 @@ targets: properties: LSRequiresIPhoneOS: true UILaunchStoryboardName: LaunchScreen - UIBackgroundModes: [audio] + UIBackgroundModes: [audio, voip] NSMicrophoneUsageDescription: Sable uses your microphone for calls. NSCameraUsageDescription: Sable uses your camera for video calls. UIRequiredDeviceCapabilities: [arm64, metal] @@ -107,7 +105,6 @@ targets: - UIInterfaceOrientationLandscapeRight CFBundleShortVersionString: {{apple.bundle-version-short}} CFBundleVersion: "{{apple.bundle-version}}" - UIBackgroundModes: [audio] NSPhotoLibraryAddUsageDescription: Sable saves images you download to your photo library. # Mirrors plugins.deep-link in tauri.conf.json: the plugin only patches # these in when cargo reruns its build script, so a fresh `tauri ios @@ -121,10 +118,6 @@ targets: {{this.key}}: {{this.value}}{{/each}} entitlements: path: {{app.name}}_iOS/{{app.name}}_iOS.entitlements - properties: - aps-environment: development - com.apple.security.application-groups: - - group.{{app.identifier}} scheme: environmentVariables: RUST_BACKTRACE: full diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 20fa2fb6e..3fb0ad629 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -355,6 +355,7 @@ pub fn run() { #[cfg(mobile)] let builder = builder .plugin(tauri_plugin_edge_to_edge::init()) + .plugin(tauri_plugin_livekit_mobile::init()) .plugin(tauri_plugin_sharekit::init()); #[cfg(target_os = "android")] diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 7fb809737..bccd176b9 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -24,7 +24,13 @@ }, "iOS": { "template": "src-tauri/ios-project.yml", - "frameworks": ["AudioToolbox", "Photos"] + "minimumSystemVersion": "16.0", + "frameworks": [ + "AudioToolbox", + "Photos", + "Frameworks/LiveKitWebRTC.xcframework", + "Frameworks/RustLiveKitUniFFI.xcframework" + ] }, "linux": { "deb": { diff --git a/src-tauri/tauri.local.ios.conf.json b/src-tauri/tauri.local.ios.conf.json new file mode 100644 index 000000000..97948f54d --- /dev/null +++ b/src-tauri/tauri.local.ios.conf.json @@ -0,0 +1,3 @@ +{ + "identifier": "moe.sable.client.local" +} diff --git a/src/app/components/CallEmbedProvider.tsx b/src/app/components/CallEmbedProvider.tsx index 78b2b5ee0..b5bbc05ba 100644 --- a/src/app/components/CallEmbedProvider.tsx +++ b/src/app/components/CallEmbedProvider.tsx @@ -17,6 +17,7 @@ import { useSelectedRoom } from '$hooks/router/useSelectedRoom'; import { ScreenSize, useScreenSizeContext } from '$hooks/useScreenSize'; import { IncomingCallModal } from '$features/call/IncomingCallModal'; import { toCallEmbedStartError } from '$plugins/call/callEmbedError'; +import { LivekitJsCallManagerProvider } from '$features/call/livekitJsCallManager'; function CallUtils({ embed }: { embed: CallEmbed }) { const setCallEmbed = useSetAtom(callEmbedAtom); @@ -80,8 +81,10 @@ export function CallEmbedProvider({ children }: CallEmbedProviderProps) { {callEmbed && } - - {children} + + + {children} +
& { account?: string; issuer?: string; }; - 'org.matrix.msc4143.rtc_foci'?: [ - { - livekit_service_url: string; - type: 'livekit'; - }, - ]; + 'org.matrix.msc4143.rtc_foci'?: LivekitTransportConfig[]; +}; + +export const getLivekitTransports = ( + discovery: Pick | undefined +): LivekitTransportConfig[] => { + const foci = discovery?.['org.matrix.msc4143.rtc_foci']; + if (!Array.isArray(foci)) return []; + + return foci.filter( + (focus): focus is LivekitTransportConfig => + typeof focus === 'object' && + focus !== null && + focus.type === 'livekit' && + typeof focus.livekit_service_url === 'string' + ); }; export const autoDiscovery = async ( diff --git a/src/app/features/call-status/CallControl.tsx b/src/app/features/call-status/CallControl.tsx index 5162b8b08..c716c65bd 100644 --- a/src/app/features/call-status/CallControl.tsx +++ b/src/app/features/call-status/CallControl.tsx @@ -23,7 +23,7 @@ type MicrophoneButtonProps = { onToggle: () => Promise; disabled?: boolean; }; -function MicrophoneButton({ enabled, onToggle, disabled }: MicrophoneButtonProps) { +export function MicrophoneButton({ enabled, onToggle, disabled }: MicrophoneButtonProps) { return ( void; disabled?: boolean; }; -function SoundButton({ enabled, onToggle, disabled }: SoundButtonProps) { +export function SoundButton({ enabled, onToggle, disabled }: SoundButtonProps) { return ( Promise; disabled?: boolean; }; -function VideoButton({ enabled, onToggle, disabled }: VideoButtonProps) { +export function VideoButton({ enabled, onToggle, disabled }: VideoButtonProps) { return ( void; disabled?: boolean; }; -function ScreenShareButton({ enabled, onToggle, disabled }: ScreenShareButtonProps) { +export function ScreenShareButton({ enabled, onToggle, disabled }: ScreenShareButtonProps) { return ( (); + +export function HangupChip({ + compact, + onHangup, +}: { + compact: boolean; + onHangup: () => Promise; +}) { + const [hangupState, hangup] = useAsyncCallback(useCallback(() => onHangup(), [onHangup])); + const exiting = + hangupState.status === AsyncStatus.Loading || hangupState.status === AsyncStatus.Success; + + return ( + + ) : ( + sizedIcon(PhoneDisconnect, '50', { filled: true }) + ) + } + disabled={exiting} + outlined + onClick={() => hangup()} + > + {!compact && ( + + End + + )} + + ); +} + +/** + * The persistent call bar both engines render into. Only the control cluster + * differs, so it is passed in. + */ +export function CallStatusShell({ + room, + compact, + connected, + controls, +}: { + room: Room; + compact: boolean; + connected: boolean; + controls: ReactNode; +}) { + const callSession = useCallSession(room); + const callMembers = useCallMembers(room, callSession); + const memberVisible = connected && callMembers.length > 0; + + return ( + + + {memberVisible ? ( + + + + ) : ( + + )} + + {!compact && } + + {memberVisible && ( + + + + )} + + {memberVisible && !compact && } + + {compact && ( + + + + )} + {controls} + + + ); +} diff --git a/src/app/features/call-status/LivekitCallStatus.tsx b/src/app/features/call-status/LivekitCallStatus.tsx new file mode 100644 index 000000000..3afc8183e --- /dev/null +++ b/src/app/features/call-status/LivekitCallStatus.tsx @@ -0,0 +1,72 @@ +import { Box } from 'folds'; +import { useAtom } from 'jotai'; +import { RoomContext, useLocalParticipant } from '@livekit/components-react'; +import { useMatrixClient } from '$hooks/useMatrixClient'; +import { ScreenSize, useScreenSize } from '$hooks/useScreenSize'; +import { livekitJsCallSoundAtom, type LivekitJsCallSession } from '$state/livekitJsCall'; +import { MicrophoneButton, ScreenShareButton, SoundButton, VideoButton } from './CallControl'; +import { CallStatusShell, HangupChip } from './CallStatusShell'; +import { StatusDivider } from './components'; + +function LivekitCallControl({ + compact, + onHangup, +}: { + compact: boolean; + onHangup: () => Promise; +}) { + const { localParticipant, isMicrophoneEnabled, isCameraEnabled, isScreenShareEnabled } = + useLocalParticipant(); + const [sound, setSound] = useAtom(livekitJsCallSoundAtom); + + return ( + + + localParticipant.setMicrophoneEnabled(!isMicrophoneEnabled)} + /> + setSound(!sound)} /> + {!compact && } + localParticipant.setCameraEnabled(!isCameraEnabled)} + /> + {!compact && ( + void localParticipant.setScreenShareEnabled(!isScreenShareEnabled)} + /> + )} + + + + + ); +} + +export function LivekitCallStatus({ session }: { session: LivekitJsCallSession }) { + const mx = useMatrixClient(); + const screenSize = useScreenSize(); + const room = mx.getRoom(session.roomId); + const compact = screenSize === ScreenSize.Mobile; + + if (!room) return null; + + return ( + + + + ) : ( + + ) + } + /> + ); +} diff --git a/src/app/features/call-status/NativeCallStatus.tsx b/src/app/features/call-status/NativeCallStatus.tsx new file mode 100644 index 000000000..2a8f44df6 --- /dev/null +++ b/src/app/features/call-status/NativeCallStatus.tsx @@ -0,0 +1,50 @@ +import { Box } from 'folds'; +import { useMatrixClient } from '$hooks/useMatrixClient'; +import { ScreenSize, useScreenSize } from '$hooks/useScreenSize'; +import type { NativeCallSession } from '$state/nativeCall'; +import { MicrophoneButton, VideoButton } from './CallControl'; +import { CallStatusShell, HangupChip } from './CallStatusShell'; +import { StatusDivider } from './components'; + +function NativeCallControl({ session, compact }: { session: NativeCallSession; compact: boolean }) { + // Media commands are rejected until the native room has connected. + const disabled = session.lifecycle !== 'connected'; + + return ( + + + session.setMicrophoneEnabled(!session.microphoneEnabled)} + disabled={disabled} + /> + {!compact && } + session.setCameraEnabled(!session.cameraEnabled)} + disabled={disabled} + /> + + + + + ); +} + +export function NativeCallStatus({ session }: { session: NativeCallSession }) { + const mx = useMatrixClient(); + const screenSize = useScreenSize(); + const room = mx.getRoom(session.roomId); + const compact = screenSize === ScreenSize.Mobile; + + if (!room) return null; + + return ( + } + /> + ); +} diff --git a/src/app/features/call/CallDevicePreview.css.ts b/src/app/features/call/CallDevicePreview.css.ts new file mode 100644 index 000000000..90252a911 --- /dev/null +++ b/src/app/features/call/CallDevicePreview.css.ts @@ -0,0 +1,51 @@ +import { style } from '@vanilla-extract/css'; +import { color, config, toRem } from 'folds'; + +export const PreviewSurface = style({ + position: 'relative', + width: '100%', + aspectRatio: '16 / 9', + borderRadius: config.radii.R400, + background: '#14171f', + color: color.Surface.OnContainer, + overflow: 'hidden', +}); + +export const PreviewVideo = style({ + position: 'absolute', + inset: 0, + width: '100%', + height: '100%', + objectFit: 'cover', + // Front cameras read as a mirror to the person looking at them. + transform: 'scaleX(-1)', +}); + +export const DeviceSelect = style({ + width: '100%', + minWidth: 0, + minHeight: toRem(36), + padding: `0 ${config.space.S200}`, + borderRadius: config.radii.R400, + border: `1px solid ${color.Surface.ContainerLine}`, + background: color.Surface.Container, + color: color.Surface.OnContainer, + font: 'inherit', + fontSize: toRem(14), +}); + +export const LevelTrack = style({ + width: '100%', + height: toRem(6), + borderRadius: config.radii.R400, + background: color.Surface.ContainerLine, + overflow: 'hidden', +}); + +export const LevelFill = style({ + width: '100%', + height: '100%', + transformOrigin: 'left center', + background: color.Success.Main, + transition: 'transform 80ms linear', +}); diff --git a/src/app/features/call/CallDevicePreview.test.tsx b/src/app/features/call/CallDevicePreview.test.tsx new file mode 100644 index 000000000..24bcd4f1e --- /dev/null +++ b/src/app/features/call/CallDevicePreview.test.tsx @@ -0,0 +1,96 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CallDevicePreview } from './CallDevicePreview'; + +const mocks = vi.hoisted(() => ({ + usePreviewTracks: vi.fn<() => unknown[] | undefined>(), + useMediaDeviceSelect: vi.fn<(o: { kind: string }) => { devices: MediaDeviceInfo[] }>(), + previewOptions: undefined as unknown, + attach: vi.fn<(el: HTMLElement) => void>(), + detach: vi.fn<(el: HTMLElement) => void>(), +})); + +vi.mock('@livekit/components-react', () => ({ + // Mirror the real hook: it only returns tracks for the sources requested. + usePreviewTracks: (options: { audio: unknown; video: unknown }) => { + mocks.previewOptions = options; + const tracks = mocks.usePreviewTracks() ?? []; + return tracks.filter((track) => { + const { kind } = track as { kind: string }; + return kind === 'video' ? options.video !== false : options.audio !== false; + }); + }, + useMediaDeviceSelect: (options: { kind: string }) => mocks.useMediaDeviceSelect(options), + useTrackVolume: () => 0.25, +})); + +vi.mock('livekit-client', () => ({ + Track: { Kind: { Video: 'video', Audio: 'audio' } }, +})); + +const device = (deviceId: string, label: string, kind: string) => + ({ deviceId, label, kind }) as MediaDeviceInfo; + +const videoTrack = { kind: 'video', attach: mocks.attach, detach: mocks.detach }; +const audioTrack = { kind: 'audio' }; + +const props = { + microphone: true, + video: true, + onAudioDeviceChange: vi.fn<(id: string) => void>(), + onVideoDeviceChange: vi.fn<(id: string) => void>(), +}; + +beforeEach(() => { + mocks.usePreviewTracks.mockReset().mockReturnValue([videoTrack, audioTrack]); + mocks.useMediaDeviceSelect.mockReset().mockImplementation(({ kind }) => ({ + devices: + kind === 'audioinput' + ? [device('mic-1', 'Built-in Mic', kind), device('mic-2', 'USB Mic', kind)] + : [device('cam-1', 'FaceTime HD', kind)], + })); + mocks.attach.mockReset(); + mocks.detach.mockReset(); + props.onAudioDeviceChange.mockReset(); + props.onVideoDeviceChange.mockReset(); +}); + +describe('CallDevicePreview', () => { + it('attaches the preview camera track to a video element', () => { + render(); + + expect(mocks.attach).toHaveBeenCalledOnce(); + expect(screen.queryByText('Camera is off')).not.toBeInTheDocument(); + }); + + it('detaches the camera track on unmount so the call can claim it', () => { + const { unmount } = render(); + unmount(); + + expect(mocks.detach).toHaveBeenCalledOnce(); + }); + + it('requests only the devices the user actually enabled', () => { + render(); + + expect(mocks.previewOptions).toEqual({ audio: { deviceId: 'mic-2' }, video: false }); + expect(screen.getByText('Camera is off')).toBeInTheDocument(); + }); + + it('reports a chosen microphone so the call can honour it', async () => { + render(); + + await userEvent.selectOptions(screen.getByLabelText('Microphone'), 'mic-2'); + + expect(props.onAudioDeviceChange).toHaveBeenCalledWith('mic-2'); + }); + + it('shows a microphone level only while the microphone is on', () => { + const { rerender } = render(); + expect(screen.getByRole('meter', { name: 'Microphone level' })).toBeInTheDocument(); + + rerender(); + expect(screen.queryByRole('meter', { name: 'Microphone level' })).not.toBeInTheDocument(); + }); +}); diff --git a/src/app/features/call/CallDevicePreview.tsx b/src/app/features/call/CallDevicePreview.tsx new file mode 100644 index 000000000..c5573c0a1 --- /dev/null +++ b/src/app/features/call/CallDevicePreview.tsx @@ -0,0 +1,141 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { Box, Text, config, toRem } from 'folds'; +import { useMediaDeviceSelect, usePreviewTracks, useTrackVolume } from '@livekit/components-react'; +import { Track, type LocalAudioTrack, type LocalVideoTrack } from 'livekit-client'; +import { VideoCameraSlash, sizedIcon } from '$components/icons/phosphor'; +import * as css from './CallDevicePreview.css'; + +type DeviceSelectProps = { + label: string; + kind: MediaDeviceKind; + deviceId?: string; + onChange: (deviceId: string) => void; + permissionsGranted: boolean; +}; + +function DeviceSelect({ label, kind, deviceId, onChange, permissionsGranted }: DeviceSelectProps) { + const { devices } = useMediaDeviceSelect({ kind, requestPermissions: permissionsGranted }); + + return ( + + {label} + + + ); +} + +function MicrophoneLevel({ track }: { track?: LocalAudioTrack }) { + const volume = useTrackVolume(track); + const level = Math.min(1, volume * 3); + + return ( + + Microphone level +
+
+
+ + ); +} + +function VideoPreview({ track }: { track?: LocalVideoTrack }) { + const videoRef = useRef(null); + + useEffect(() => { + const element = videoRef.current; + if (!track || !element) return undefined; + track.attach(element); + return () => { + track.detach(element); + }; + }, [track]); + + if (!track) { + return ( + + {sizedIcon(VideoCameraSlash, '400')} + Camera is off + + ); + } + + return ( + + + ); +} + +export type CallDevicePreviewProps = { + microphone: boolean; + video: boolean; + audioDeviceId?: string; + videoDeviceId?: string; + onAudioDeviceChange: (deviceId: string) => void; + onVideoDeviceChange: (deviceId: string) => void; +}; + +export function CallDevicePreview({ + microphone, + video, + audioDeviceId, + videoDeviceId, + onAudioDeviceChange, + onVideoDeviceChange, +}: CallDevicePreviewProps) { + const tracks = usePreviewTracks({ + audio: microphone ? { deviceId: audioDeviceId } : false, + video: video ? { deviceId: videoDeviceId } : false, + }); + + const videoTrack = useMemo( + () => tracks?.find((track) => track.kind === Track.Kind.Video) as LocalVideoTrack | undefined, + [tracks] + ); + const audioTrack = useMemo( + () => tracks?.find((track) => track.kind === Track.Kind.Audio) as LocalAudioTrack | undefined, + [tracks] + ); + + return ( + + + {microphone && } + + + + + + ); +} diff --git a/src/app/features/call/CallView.test.tsx b/src/app/features/call/CallView.test.tsx new file mode 100644 index 000000000..4b4afd1c5 --- /dev/null +++ b/src/app/features/call/CallView.test.tsx @@ -0,0 +1,123 @@ +import { render, screen } from '@testing-library/react'; +import { describe, expect, it, vi } from 'vitest'; +import { LivekitJsCallStatus } from './CallView'; +import { NativeCallSurface } from './NativeCallSurface'; +import type { NativeCallSession } from '$state/nativeCall'; + +vi.mock('./livekitMobileBridge', () => ({ + getNativeCallState: vi.fn<() => Promise>(() => Promise.reject(new Error('not on device'))), + listenNativeCallSnapshot: vi.fn<() => Promise<() => void>>(() => Promise.resolve(() => {})), + setNativeCallRemoteVideoOverlay: vi.fn<() => Promise>(() => Promise.resolve({})), + clearNativeCallRemoteVideoOverlay: vi.fn<() => Promise>(() => Promise.resolve({})), + setNativeCallLocalVideoOverlay: vi.fn<() => Promise>(() => Promise.resolve({})), + clearNativeCallLocalVideoOverlay: vi.fn<() => Promise>(() => Promise.resolve({})), +})); + +vi.mock('$hooks/useRoom', () => ({ useRoom: () => ({ roomId: '!room:example.org' }) })); +vi.mock('$hooks/router/useSelectedRoom', () => ({ + useSelectedRoom: () => '!room:example.org', +})); +vi.mock('$hooks/useCall', () => ({ useCallSession: () => ({}), useCallMembers: () => [] })); +vi.mock('./LivekitCallParticipant', () => ({ + useCallParticipantProfile: () => ({ name: 'Bob' }), + CallParticipantAvatar: () =>
, +})); + +const nativeSession = (lifecycle: NativeCallSession['lifecycle']): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId: '!room:example.org', + callId: 'call-id', + lifecycle, + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: vi.fn<() => Promise>().mockResolvedValue(undefined), +}); + +describe('LiveKit JS call status', () => { + it('reports progress without exposing backend or transport details', () => { + render( + {}} + /> + ); + + expect(screen.getByText('Preparing call')).toBeInTheDocument(); + expect(screen.queryByText(/livekit|token|url|secret|e2ee/i)).not.toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'End' })).toBeInTheDocument(); + }); + + it('explains a setup failure in plain language', () => { + render( + {}} + /> + ); + + expect(screen.getByText('Call failed')).toBeInTheDocument(); + expect(screen.getByText('Could not connect to the call.')).toBeInTheDocument(); + expect(screen.queryByText(/token|url|secret|error:/i)).not.toBeInTheDocument(); + }); + + it('gives an unsupported-encryption failure a dismiss route', () => { + const onHangup = vi.fn<() => void>(); + render( + + ); + + expect( + screen.getByText('Encrypted calls are not supported on this device.') + ).toBeInTheDocument(); + screen.getByRole('button', { name: 'Dismiss' }).click(); + expect(onHangup).toHaveBeenCalledOnce(); + }); +}); + +describe('native call surface', () => { + it('shows the local tile and call controls when connected', () => { + render( {}} />); + + expect(screen.getByText('You')).toBeInTheDocument(); + expect(screen.getAllByRole('button')).toHaveLength(3); + expect(screen.getByRole('button', { name: 'Mute microphone' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'Start camera' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'End call' })).toBeInTheDocument(); + }); + + it('keeps media toggles disabled while connecting', () => { + render( {}} />); + + expect(screen.getByText('Connecting')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Mute microphone' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'Start camera' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'End call' })).toBeEnabled(); + }); + + it('gives failed calls an explicit dismiss route', () => { + const onHangup = vi.fn<() => void>(); + render( + + ); + + expect(screen.getByText('Call failed')).toBeInTheDocument(); + expect(screen.getByText('Native call connection failed.')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Dismiss' })).toBeInTheDocument(); + screen.getByRole('button', { name: 'Dismiss' }).click(); + expect(onHangup).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/app/features/call/CallView.tsx b/src/app/features/call/CallView.tsx index 62d79cd6d..ae324a009 100644 --- a/src/app/features/call/CallView.tsx +++ b/src/app/features/call/CallView.tsx @@ -13,6 +13,13 @@ import { CallMemberRenderer } from './CallMemberCard'; import { PrescreenControls } from './PrescreenControls'; import { callEmbedAtom, callEmbedStartErrorAtom } from '$state/callEmbed'; import { canJoinCall } from './callStartCapabilities'; +import type { LivekitJsCallSession } from '$state/livekitJsCall'; +import { livekitJsCallAtom } from '$state/livekitJsCall'; +import { nativeCallAtom } from '$state/nativeCall'; +import { LivekitJsCallSurface } from './LivekitJsCallSurface'; +import { NativeCallSurface } from './NativeCallSurface'; +import { CallStatusBar } from './callChrome'; +import { livekitJsCallStatus } from './callClient'; function LivekitServerMissingMessage() { return ( @@ -80,6 +87,16 @@ function WidgetPreparationErrorMessage({ message }: { message: string }) { ); } +export function LivekitJsCallStatus({ + session, + onHangup, +}: { + session: Pick; + onHangup: () => void; +}) { + return ; +} + function CallPrescreen() { const room = useRoom(); const callEmbed = useAtomValue(callEmbedAtom); @@ -168,11 +185,22 @@ export function CallView({ resizable }: CallViewProps) { const callEmbed = useCallEmbed(); const callJoined = useCallJoined(callEmbed); - - const currentJoined = callEmbed?.roomId === room.roomId && callJoined; - + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const nativeCall = useAtomValue(nativeCallAtom); + + const livekitJsCallForRoom = livekitJsCall?.roomId === room.roomId ? livekitJsCall : undefined; + const nativeCallForRoom = nativeCall?.roomId === room.roomId ? nativeCall : undefined; + const livekitJsRoom = + livekitJsCallForRoom?.lifecycle === 'active' ? livekitJsCallForRoom.room : undefined; + const currentJoined = + !livekitJsCallForRoom && !nativeCallForRoom && callEmbed?.roomId === room.roomId && callJoined; + + // A native call renders video tiles and a control bar, which need most of the + // viewport; the 0.3 default is sized for the Element Call participant list. const [heightRatio, setHeightRatio] = useState(isMobile ? 0.3 : 0.72); const [availableHeight, setAvailableHeight] = useState(0); + const effectiveHeightRatio = + isMobile && nativeCallForRoom ? Math.max(heightRatio, 0.75) : heightRatio; useEffect(() => { if (!resizable || !callViewRef.current) return undefined; @@ -263,12 +291,18 @@ export function CallView({ resizable }: CallViewProps) { minWidth: toRem(280), height: resizable ? availableHeight > 0 - ? `${availableHeight * heightRatio}px` - : `${heightRatio * 100}dvh` + ? `${availableHeight * effectiveHeightRatio}px` + : `${effectiveHeightRatio * 100}dvh` : undefined, borderBottom: `1px solid var(--sable-surface-container-line)`, zIndex: 20, - backgroundColor: currentJoined ? 'transparent' : undefined, + backgroundColor: + livekitJsRoom || nativeCallForRoom + ? color.Background.Container + : currentJoined + ? 'transparent' + : undefined, + overflow: livekitJsRoom || nativeCallForRoom ? 'hidden' : undefined, pointerEvents: currentJoined ? 'none' : 'all', }} > @@ -284,8 +318,27 @@ export function CallView({ resizable }: CallViewProps) { /> )} - {!currentJoined && } - + {!currentJoined && !livekitJsCallForRoom && !nativeCallForRoom && } + {livekitJsCallForRoom && livekitJsRoom ? ( + void livekitJsCallForRoom.hangup()} + /> + ) : livekitJsCallForRoom ? ( + void livekitJsCallForRoom.hangup()} + /> + ) : nativeCallForRoom ? ( + void nativeCallForRoom.hangup()} + /> + ) : ( + + )} {resizable && ( + ), + color: { + Critical: { Container: 'red', OnContainer: 'white' }, + Surface: { OnContainer: 'white' }, + Warning: { Container: 'yellow', OnContainer: 'black' }, + }, + config: { + radii: { R500: '5px' }, + space: { S100: '4px', S200: '8px', S300: '12px' }, + }, + Text: ({ children }: { children: ReactNode }) => {children}, + toRem: (value: number) => `${value}px`, +})); + +const room = {} as Room; +const initialMedia = { microphone: true, camera: false, sound: true }; + +const videoTrack = (source: string, isLocal: boolean, identity = 'hash-alice') => ({ + source, + participant: { isLocal, identity }, + publication: {}, +}); + +const placeholderTrack = (source: string, isLocal: boolean, identity = 'hash-alice') => ({ + source, + participant: { isLocal, identity }, + publication: undefined, +}); + +const surfaceElement = () => document.body.querySelector('[data-livekit-call-surface]'); +const controlsElement = () => document.body.querySelector('[data-livekit-controls]'); + +beforeEach(() => { + getDefaultStore().set(livekitJsCallInitialMediaAppliedAtom, false); + mocks.useConnectionState.mockReset().mockReturnValue('connected'); + mocks.useTracks.mockReset().mockReturnValue([]); + mocks.useParticipants.mockReset().mockReturnValue([]); + mocks.setMicrophoneEnabled.mockReset().mockResolvedValue(undefined); + mocks.setCameraEnabled.mockReset().mockResolvedValue(undefined); + mocks.onDeviceError = undefined; +}); + +describe('LiveKit JS call surface', () => { + it('publishes with the devices chosen in the prescreen preview', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(true, { deviceId: 'mic-2' }); + expect(mocks.setCameraEnabled).toHaveBeenCalledWith(true, { deviceId: 'cam-1' }); + }); + + it('leaves device selection to the browser when the user never picked one', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(true, undefined); + expect(mocks.setCameraEnabled).toHaveBeenCalledWith(true, undefined); + }); + + it('applies the prescreen media choice once encryption is ready', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(true, undefined); + expect(mocks.setCameraEnabled).toHaveBeenCalledWith(true, undefined); + }); + + it('joins muted when the prescreen microphone was off', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(false, undefined); + expect(mocks.setCameraEnabled).not.toHaveBeenCalled(); + }); + + it('keeps the call bar state when the surface remounts, instead of re-muting', () => { + const media = { microphone: false, camera: false, sound: true }; + const view = render( + {}} /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledTimes(1); + view.unmount(); + + render( {}} />); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledTimes(1); + }); + + it('publishes nothing until encryption is ready', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(mocks.setCameraEnabled).not.toHaveBeenCalled(); + }); + + it('mutes incoming audio while the shared sound toggle is off', () => { + const store = createStore(); + store.set(livekitJsCallSoundAtom, false); + + render( + + {}} + /> + + ); + + expect(screen.getByTestId('room-audio')).toHaveAttribute('data-muted', 'true'); + }); + + it('lets LiveKit derive publish controls from the token grants', () => { + render( + {}} /> + ); + + expect(screen.getByTestId('control-bar')).toHaveAttribute('data-controls', '{"leave":false}'); + expect(screen.getByTestId('device-menu')).toHaveAttribute('data-kind', 'audiooutput'); + }); + + it('surfaces a denied microphone instead of failing silently', async () => { + render( + {}} /> + ); + + act(() => mocks.onDeviceError?.({ source: 'microphone' })); + + expect( + await screen.findByText('Microphone unavailable. Check your browser permissions.') + ).toBeInTheDocument(); + }); + + it('withholds media controls until call encryption is ready', () => { + render( + {}} + /> + ); + + expect(screen.queryByTestId('control-bar')).not.toBeInTheDocument(); + expect(screen.getByText('Securing call…')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'End call' })).toBeInTheDocument(); + }); + + it('keeps audio-only calls understandable with persistent controls', () => { + render( + {}} /> + ); + + expect(screen.getByTestId('room-audio')).toBeInTheDocument(); + expect(screen.getByText('Audio call')).toBeInTheDocument(); + expect(screen.getByTestId('control-bar')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'End call' })).toBeInTheDocument(); + }); + + it('shows every participant when no camera is published, not an empty grid', () => { + mocks.useTracks.mockReturnValue([ + placeholderTrack('camera', true), + placeholderTrack('camera', false), + ]); + mocks.useParticipants.mockReturnValue([{ identity: 'hash-alice' }, { identity: 'hash-bob' }]); + + render( + {}} /> + ); + + expect(screen.queryByTestId('grid-layout')).not.toBeInTheDocument(); + expect(screen.queryByTestId('focus-layout')).not.toBeInTheDocument(); + expect(screen.getAllByTestId('participant-avatar')).toHaveLength(2); + expect(screen.getByText('@alice:example.org')).toBeInTheDocument(); + expect(screen.getByText('@bob:example.org')).toBeInTheDocument(); + }); + + it('labels video tiles with the Matrix display name, never the LiveKit identity hash', () => { + mocks.useTracks.mockReturnValue([videoTrack('camera', false)]); + + render( + {}} /> + ); + + expect(screen.getByTestId('grid-layout')).toBeInTheDocument(); + expect(screen.getByText('@alice:example.org')).toBeInTheDocument(); + expect(screen.queryByText(/hash-/)).not.toBeInTheDocument(); + }); + + it('fills the call view container rather than the viewport', () => { + const { container } = render( + {}} /> + ); + + const surface = container.querySelector('[data-livekit-call-surface]'); + expect(surface).not.toBeNull(); + expect(surface).toHaveStyle({ position: 'relative', width: '100%', height: '100%' }); + expect(screen.getByRole('region', { name: 'Call' })).toBe(surface); + }); + + it('puts the screen share in focus and keeps other tracks in a carousel', () => { + mocks.useTracks.mockReturnValue([ + videoTrack('screen_share', false), + videoTrack('camera', false), + ]); + + render( + {}} /> + ); + + expect(screen.getByTestId('focus-layout-container')).toBeInTheDocument(); + expect(screen.getByTestId('focus-layout')).toHaveAttribute('data-focus-source', 'screen_share'); + expect(screen.getByTestId('carousel-layout')).toHaveAttribute('data-track-count', '1'); + expect(screen.queryByTestId('grid-layout')).not.toBeInTheDocument(); + expect(screen.queryByText('Audio call')).not.toBeInTheDocument(); + }); + + it('labels the focused screen share instead of leaving the default tile on top', () => { + mocks.useTracks.mockReturnValue([ + videoTrack('screen_share', false), + videoTrack('camera', false), + ]); + + render( + {}} /> + ); + + const focus = within(screen.getByTestId('focus-layout')); + expect(focus.getByText("@alice:example.org's screen")).toBeInTheDocument(); + expect(focus.queryByText(/hash-/)).not.toBeInTheDocument(); + }); + + it('keeps an active local screen share on stage instead of suppressing it', () => { + mocks.useTracks.mockReturnValue([ + videoTrack('screen_share', true), + videoTrack('camera', false), + placeholderTrack('camera', true), + ]); + + render( + {}} /> + ); + + expect(screen.getByTestId('focus-layout')).toHaveAttribute('data-focus-source', 'screen_share'); + expect(screen.getByRole('status')).toHaveTextContent('Sharing your screen'); + expect(screen.getByTestId('carousel-layout')).toHaveAttribute('data-track-count', '2'); + }); + + it('shows connection loss feedback without replacing the media canvas', () => { + mocks.useConnectionState.mockReturnValue('reconnecting'); + + render( + {}} /> + ); + + expect(screen.getByRole('status')).toHaveTextContent('Reconnecting…'); + expect(screen.getByTestId('room-audio')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'End call' })).toBeInTheDocument(); + }); + + it('hides idle controls from view and hit testing, then reveals them on a canvas tap', () => { + vi.useFakeTimers(); + try { + render( + {}} + /> + ); + + const surface = surfaceElement()!; + const controls = controlsElement()!; + const pill = screen.getByTestId('control-bar').parentElement!; + + act(() => vi.advanceTimersByTime(3500)); + expect(controls).toHaveStyle({ opacity: '0', visibility: 'hidden' }); + expect(pill).toHaveStyle({ pointerEvents: 'none' }); + + act(() => fireEvent.pointerDown(surface)); + expect(controls).toHaveStyle({ opacity: '1', visibility: 'visible' }); + expect(pill).toHaveStyle({ pointerEvents: 'auto' }); + } finally { + vi.useRealTimers(); + } + }); + + it('reveals hidden controls on a key press, the only route a keyboard user has', () => { + vi.useFakeTimers(); + try { + render( + {}} + /> + ); + + const surface = surfaceElement()!; + const controls = controlsElement()!; + + act(() => vi.advanceTimersByTime(3500)); + expect(controls).toHaveStyle({ visibility: 'hidden' }); + + // focusIn cannot fire from a Tab press: visibility:hidden removes the + // controls from the tab order and nothing else inside the surface is + // focusable, so keydown is the reveal a keyboard user can actually reach. + act(() => fireEvent.keyDown(surface, { key: 'Tab' })); + expect(controls).toHaveStyle({ visibility: 'visible' }); + } finally { + vi.useRealTimers(); + } + }); + + it('labels the shared control surface for assistive technology', () => { + render( + {}} /> + ); + + expect(screen.getByRole('group', { name: 'Call controls' })).toBe(controlsElement()); + }); + + it('ends the call from the control bar', () => { + const onHangup = vi.fn<() => void>(); + render( + + ); + + screen.getByRole('button', { name: 'End call' }).click(); + expect(onHangup).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/app/features/call/LivekitJsCallSurface.tsx b/src/app/features/call/LivekitJsCallSurface.tsx new file mode 100644 index 000000000..cbd076b6e --- /dev/null +++ b/src/app/features/call/LivekitJsCallSurface.tsx @@ -0,0 +1,491 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { useAtom, useAtomValue } from 'jotai'; +import { Box, Button, color, config, Text, toRem } from 'folds'; +import { + CarouselLayout, + ConnectionQualityIndicator, + ControlBar, + MediaDeviceMenu, + FocusLayout, + FocusLayoutContainer, + GridLayout, + ParticipantTile, + RoomAudioRenderer, + RoomContext, + TrackMutedIndicator, + useConnectionState, + useEnsureTrackRef, + useIsSpeaking, + useLocalParticipant, + useParticipants, + useTracks, + VideoTrack, +} from '@livekit/components-react'; +import { ConnectionState, Track, type Participant, type Room } from 'livekit-client'; +import { SpeakerHigh, sizedIcon } from '$components/icons/phosphor'; +import { useCallMembers, useCallSession } from '$hooks/useCall'; +import { useRoom } from '$hooks/useRoom'; +import { + livekitJsCallInitialMediaAppliedAtom, + livekitJsCallSoundAtom, + type LivekitJsCallMedia, +} from '$state/livekitJsCall'; +import { buildRtcIdentityMap, type UserIdByRtcIdentity } from './livekitCallIdentity'; +import { + CallParticipantAvatar, + CallParticipantName, + useCallParticipantProfile, +} from './LivekitCallParticipant'; +import { CallControlBar, CallLayout } from './callChrome'; +import * as css from './LivekitJsCallSurface.css'; + +const controlIdleDelay = 3500; + +// Camera carries a placeholder so participants without video still get a tile; +// screen share must not, or every participant would fake a shared screen. +const trackSources = [ + { source: Track.Source.Camera, withPlaceholder: true }, + { source: Track.Source.ScreenShare, withPlaceholder: false }, +]; + +const trackOptions = { onlySubscribed: false }; + +function CallTileContent({ userIdByIdentity }: { userIdByIdentity: UserIdByRtcIdentity }) { + const trackRef = useEnsureTrackRef(); + const { participant, publication, source } = trackRef; + const profile = useCallParticipantProfile( + participant.identity, + participant.isLocal, + userIdByIdentity + ); + const isScreenShare = source === Track.Source.ScreenShare; + + return ( + <> + {publication ? ( + + ) : ( +
+ +
+ )} +
+
+ {!isScreenShare && ( + + )} + + {isScreenShare ? `${profile.name}'s screen` : profile.name} + +
+ +
+ + ); +} + +function GridTile({ userIdByIdentity }: { userIdByIdentity: UserIdByRtcIdentity }) { + return ( + + + + ); +} + +function CarouselTile({ userIdByIdentity }: { userIdByIdentity: UserIdByRtcIdentity }) { + return ( + + + + ); +} + +function AudioCallParticipant({ + participant, + userIdByIdentity, +}: { + participant: Participant; + userIdByIdentity: UserIdByRtcIdentity; +}) { + const profile = useCallParticipantProfile( + participant.identity, + participant.isLocal, + userIdByIdentity, + 192 + ); + const speaking = useIsSpeaking(participant); + + return ( + + + + + ); +} + +function AudioCallLayout({ userIdByIdentity }: { userIdByIdentity: UserIdByRtcIdentity }) { + const participants = useParticipants(); + + return ( + + + Audio call + + + {participants.map((participant) => ( + + ))} + + + ); +} + +function MediaLayout({ + tracks, + userIdByIdentity, +}: { + tracks: ReturnType; + userIdByIdentity: UserIdByRtcIdentity; +}) { + const screenShare = tracks.find((track) => track.source === Track.Source.ScreenShare); + + if (screenShare) { + const remainingTracks = tracks.filter((track) => track !== screenShare); + return ( + + + + + {remainingTracks.length > 0 && ( + + + + )} + + ); + } + + return ( + + + + ); +} + +function ConnectionFeedback() { + const connectionState = useConnectionState(); + if (connectionState === ConnectionState.Connected) return null; + + // Only Disconnected is an actual loss. SignalReconnecting keeps media flowing + // while the signal link re-establishes, and Connecting is the initial + // handshake, so neither deserves the critical treatment. + const lost = connectionState === ConnectionState.Disconnected; + const label = + connectionState === ConnectionState.Connecting + ? 'Connecting…' + : lost + ? 'Connection lost' + : 'Reconnecting…'; + return ( + + {label} + + ); +} + +const deviceErrorMessages: Partial> = { + [Track.Source.Microphone]: 'Microphone unavailable. Check your browser permissions.', + [Track.Source.Camera]: 'Camera unavailable. Check your browser permissions.', + [Track.Source.ScreenShare]: 'Screen sharing was not started.', +}; + +function LivekitJsCallContent({ + e2eeReady, + initialMedia, + onHangup, +}: { + e2eeReady: boolean; + initialMedia: LivekitJsCallMedia; + onHangup: () => void; +}) { + const tracks = useTracks(trackSources, trackOptions); + const localScreenShare = tracks.some( + (track) => track.source === Track.Source.ScreenShare && track.participant.isLocal + ); + const hasVideo = tracks.some((track) => track.publication !== undefined); + const matrixRoom = useRoom(); + const callSession = useCallSession(matrixRoom); + const callMembers = useCallMembers(matrixRoom, callSession); + const userIdByIdentity = useMemo(() => buildRtcIdentityMap(callMembers), [callMembers]); + const [controlsVisible, setControlsVisible] = useState(true); + const [deviceError, setDeviceError] = useState(undefined); + const hideTimer = useRef | undefined>(undefined); + const { localParticipant } = useLocalParticipant(); + const soundEnabled = useAtomValue(livekitJsCallSoundAtom); + const [appliedInitialMedia, setAppliedInitialMedia] = useAtom( + livekitJsCallInitialMediaAppliedAtom + ); + + const handleDeviceError = useCallback(({ source }: { source: Track.Source }) => { + setDeviceError(deviceErrorMessages[source] ?? 'A media device is unavailable.'); + }, []); + + // Publishing before the local key is imported would send unencrypted frames, + // so the prescreen choice is applied on the first render where E2EE is ready. + useEffect(() => { + if (!e2eeReady || appliedInitialMedia) return; + setAppliedInitialMedia(true); + localParticipant + .setMicrophoneEnabled( + initialMedia.microphone, + initialMedia.audioDeviceId ? { deviceId: initialMedia.audioDeviceId } : undefined + ) + .catch(() => handleDeviceError({ source: Track.Source.Microphone })); + if (initialMedia.camera) { + localParticipant + .setCameraEnabled( + true, + initialMedia.videoDeviceId ? { deviceId: initialMedia.videoDeviceId } : undefined + ) + .catch(() => handleDeviceError({ source: Track.Source.Camera })); + } + }, [ + e2eeReady, + initialMedia, + localParticipant, + handleDeviceError, + appliedInitialMedia, + setAppliedInitialMedia, + ]); + + // Disable HTML5 PiP on all video elements in the LiveKit call surface. + // iOS WKWebView enables automatic PiP for video elements, which causes a + // "weird in-app PiP" overlay; disablePictureInPicture prevents it. + useEffect(() => { + const container = document.querySelector('[data-livekit-call-surface]'); + if (!container) return undefined; + const videos = container.querySelectorAll('video'); + videos.forEach((video) => { + (video as HTMLVideoElement).disablePictureInPicture = true; + }); + const observer = new MutationObserver(() => { + const newVideos = container.querySelectorAll('video'); + newVideos.forEach((video) => { + (video as HTMLVideoElement).disablePictureInPicture = true; + }); + }); + observer.observe(container, { childList: true, subtree: true }); + return () => observer.disconnect(); + }, []); + + const revealControls = useCallback(() => { + setControlsVisible(true); + if (hideTimer.current) clearTimeout(hideTimer.current); + hideTimer.current = setTimeout(() => { + if (!document.activeElement?.closest('[data-livekit-controls]')) { + setControlsVisible(false); + } + }, controlIdleDelay); + }, []); + + useEffect(() => { + revealControls(); + return () => { + if (hideTimer.current) clearTimeout(hideTimer.current); + }; + }, [revealControls]); + + return ( + setControlsVisible(true)} + > + + + {hasVideo ? ( + + ) : ( + + )} + + + {deviceError && ( + + {deviceError} + + + )} + {localScreenShare && ( + + Sharing your screen + + )} + setControlsVisible(true)} + > + {e2eeReady ? ( + <> + + + {sizedIcon(SpeakerHigh, '300')} + + + ) : ( + + Securing call… + + )} + + + + ); +} + +export function LivekitJsCallSurface({ + room, + e2eeReady, + initialMedia, + onHangup, +}: { + room: Room; + e2eeReady: boolean; + initialMedia: LivekitJsCallMedia; + onHangup: () => void; +}) { + return ( + + + + ); +} diff --git a/src/app/features/call/NativeCallSurface.css.ts b/src/app/features/call/NativeCallSurface.css.ts new file mode 100644 index 000000000..3047a7776 --- /dev/null +++ b/src/app/features/call/NativeCallSurface.css.ts @@ -0,0 +1,182 @@ +import { style } from '@vanilla-extract/css'; +import { color, config, toRem } from 'folds'; + +/** + * Native-call-specific styles. The dark canvas shell, the control bar pill, and + * the on/off media toggle are shared via `callChrome.css.ts`; what stays here is + * the in-flow layout the native path needs because its video overlay fully + * occludes anything beneath it: the transient status row, the tile grid, the + * tile slots whose rects JS reports to the native side, and the end-call button. + */ + +export const StatusRow = style({ + flexShrink: 0, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + padding: `calc(${config.space.S200} + env(safe-area-inset-top, 0px)) ${config.space.S300} ${config.space.S100}`, + textAlign: 'center', +}); + +/** Full-bleed stage for the dominant participant (FaceTime-style). */ +export const DominantStage = style({ + flex: 1, + minHeight: 0, + minWidth: 0, + position: 'relative', + display: 'flex', + flexDirection: 'column', +}); + +/** Edge-to-edge dominant tile: no rounding, full-bleed. */ +export const DominantTile = style({ + flex: 1, + position: 'relative', + display: 'flex', + flexDirection: 'column', + minWidth: 0, + minHeight: 0, + overflow: 'hidden', + background: color.Background.Container, +}); + +/** Small floating local preview card (bottom-right, above controls). */ +export const FloatingLocal = style({ + position: 'absolute', + bottom: `calc(${config.space.S200} + env(safe-area-inset-bottom, 0px))`, + right: `calc(${config.space.S200} + env(safe-area-inset-right, 0px))`, + width: '120px', + aspectRatio: '3 / 4', + zIndex: 3, + borderRadius: config.radii.R400, + overflow: 'hidden', + border: `1px solid ${color.Surface.ContainerLine}`, + background: color.Surface.Container, +}); + +/** Adaptive grid: 2 columns by default, 3 columns (compact, scrollable) at 7+ tiles. */ +export const TileGrid = style({ + flex: 1, + minHeight: 0, + minWidth: 0, + display: 'grid', + gap: config.space.S200, + padding: config.space.S200, + overflowY: 'auto', + selectors: { + '&[data-cols="2"]': { + gridTemplateColumns: 'repeat(2, 1fr)', + gridAutoRows: '1fr', + }, + '&[data-cols="3"]': { + gridTemplateColumns: 'repeat(3, 1fr)', + gridAutoRows: 'min-content', + }, + }, +}); + +export const Tile = style({ + position: 'relative', + display: 'flex', + flexDirection: 'column', + minWidth: 0, + minHeight: 0, + overflow: 'hidden', + borderRadius: config.radii.R400, + background: color.Surface.Container, + outline: `1px solid ${color.Surface.ContainerLine}`, + outlineOffset: '-1px', +}); + +/** Fixed-aspect tile for the compact 3-column grid (7+ participants). */ +export const TileFixed = style({ + aspectRatio: '3 / 4', + flexShrink: 0, +}); + +// The rect this element occupies is what JS reports to the native side; keep +// it free of labels so the native video never covers them. +export const TileSlot = style({ + position: 'relative', + flex: 1, + minHeight: 0, + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + gap: config.space.S100, +}); + +export const InitialsBadge = style({ + display: 'grid', + placeItems: 'center', + width: 'min(72px, 45%)', + aspectRatio: '1 / 1', + borderRadius: '50%', + background: color.SurfaceVariant.Container, + color: color.SurfaceVariant.OnContainer, + fontSize: toRem(22), + fontWeight: 600, + lineHeight: 1, + userSelect: 'none', +}); + +export const TileLabel = style({ + flexShrink: 0, + display: 'flex', + alignItems: 'center', + gap: config.space.S100, + padding: `${config.space.S100} ${config.space.S200}`, + color: color.Surface.OnContainer, + fontSize: toRem(13), + lineHeight: toRem(18), + borderTop: `1px solid ${color.Surface.ContainerLine}`, +}); + +export const TileLabelName = style({ + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', +}); + +/** Connection quality indicator dot. */ +export const QualityDot = style({ + width: '8px', + height: '8px', + borderRadius: '50%', + flexShrink: 0, + selectors: { + '&[data-quality="good"]': { background: color.Success.Main }, + '&[data-quality="poor"]': { background: color.Warning.Main }, + '&[data-quality="lost"]': { background: color.Critical.Main }, + '&[data-quality="excellent"]': { background: color.Success.Main }, + // Neutral rather than a token: it reads as "no signal yet" against the dark + // video stage, where a themed surface colour would look like a real state. + '&[data-quality="unknown"]': { background: color.Surface.ContainerLine }, + }, +}); + +/** Round red disconnect button (icon-only, LiveKit mobile idiom). */ +export const HangupButton = style({ + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + width: toRem(44), + height: toRem(44), + padding: 0, + border: 'none', + borderRadius: '50%', + background: '#f54336', + color: '#ffffff', + cursor: 'pointer', + transition: 'filter 120ms ease', + selectors: { + '&:hover': { + filter: 'brightness(1.12)', + }, + '&:focus-visible': { + outline: `2px solid ${color.Primary.Main}`, + outlineOffset: '2px', + }, + }, +}); diff --git a/src/app/features/call/NativeCallSurface.tsx b/src/app/features/call/NativeCallSurface.tsx new file mode 100644 index 000000000..ec0a49ed9 --- /dev/null +++ b/src/app/features/call/NativeCallSurface.tsx @@ -0,0 +1,721 @@ +import { type ReactNode, useEffect, useMemo, useRef, useState } from 'react'; +import { Box, config, Menu, MenuItem, Text, toRem } from 'folds'; +import type { NativeCallSession } from '$state/nativeCall'; +import { + ArrowsClockwise, + Check, + MicrophoneSlash, + PhoneDisconnect, + SpeakerHigh, + User, + VideoCameraSlash, + sizedIcon, +} from '$components/icons/phosphor'; +import { ResponsiveMenu } from '$components/ResponsiveMenu'; +import { useMenuAnchor } from '$hooks/useMenuAnchor'; +import { + clearNativeCallLocalVideoOverlay, + clearNativeCallRemoteVideoOverlay, + getNativeCallState, + listenNativeCallSnapshot, + setNativeCallLocalVideoOverlay, + setNativeCallRemoteVideoOverlay, + type NativeCallAudioRoute, + type NativeCallRemoteCamera, + type NativeCallRemoteParticipant, + type NativeCallSnapshot, +} from './livekitMobileBridge'; +import { useCallMembers, useCallSession } from '$hooks/useCall'; +import { useRoom } from '$hooks/useRoom'; +import { useSelectedRoom } from '$hooks/router/useSelectedRoom'; +import { buildRtcIdentityMap, type UserIdByRtcIdentity } from './livekitCallIdentity'; +import { CallParticipantAvatar, useCallParticipantProfile } from './LivekitCallParticipant'; +import { CallControlBar, CallLayout, CallMediaControls, CallStatusBar } from './callChrome'; +import { controlButton } from './callChrome.css'; +import { nativeCallLifecycleLabels, nativeCallStatus } from './callClient'; +import * as css from './NativeCallSurface.css'; + +export type NativeCallSurfaceProps = { + session: NativeCallSession; + onHangup: () => void; +}; + +const sameParticipants = ( + a: NativeCallRemoteParticipant[], + b: NativeCallRemoteParticipant[] +): boolean => + a.length === b.length && + a.every((participant, index) => { + const other = b[index]; + return ( + participant.identity === other?.identity && + participant.connectionQuality === other.connectionQuality && + participant.camera?.sid === other.camera?.sid && + participant.camera?.muted === other.camera?.muted && + participant.camera?.subscribed === other.camera?.subscribed && + participant.screenShare?.sid === other.screenShare?.sid && + participant.screenShare?.muted === other.screenShare?.muted && + participant.screenShare?.subscribed === other.screenShare?.subscribed + ); + }); + +/** + * Tracks the remote participant projection for this call. The controller owns + * the call lifecycle; this listener is a second, read-only consumer of the + * same snapshot stream (snapshots reach every listener in the connecting + * webview), so control-path behavior is untouched. + */ +function useNativeRemoteParticipants( + callId: string, + active: boolean +): NativeCallRemoteParticipant[] { + const [participants, setParticipants] = useState([]); + const participantsRef = useRef([]); + + useEffect(() => { + if (!active || !callId) { + participantsRef.current = []; + setParticipants([]); + return undefined; + } + + let disposed = false; + const apply = (snapshot: NativeCallSnapshot) => { + if (disposed || snapshot.callId !== callId) return; + const next = snapshot.remoteParticipants ?? []; + if (sameParticipants(participantsRef.current, next)) return; + participantsRef.current = next; + setParticipants(next); + }; + + // Seed from the current state so remounting mid-call shows participants + // before the next snapshot event arrives. + getNativeCallState() + .then(apply) + .catch(() => undefined); + const unlistenPromise = listenNativeCallSnapshot(apply); + // Mark the listen promise as handled immediately; cleanup re-attaches. + unlistenPromise.catch(() => undefined); + + return () => { + disposed = true; + unlistenPromise.then((unlisten) => unlisten()).catch(() => undefined); + }; + }, [callId, active]); + + return participants; +} + +type OverlayTarget = { + participantIdentity: string; + trackId: string; +}; + +/** + * True when the slot's center point is covered by another element (drawers, + * modal sheets, page transitions). Rect geometry alone cannot detect this: + * a drawer sliding over the call view moves nothing. Probing the topmost + * element at the slot's center is the reliable occlusion signal. + */ +function nativeSlotOccluded(slotNode: HTMLDivElement, rect: DOMRect): boolean { + const cx = rect.x + rect.width / 2; + const cy = rect.y + rect.height / 2; + if (cx < 0 || cy < 0 || cx > window.innerWidth || cy > window.innerHeight) return true; + const topElement = document.elementFromPoint(cx, cy); + return topElement === null || (topElement !== slotNode && !slotNode.contains(topElement)); +} + +/** + * Reserves the featured tile's DOM slot and reports its viewport-relative + * rect to the native video overlay. The native side renders a single remote + * video view exactly over the reported rect, so only one slot is reported at + * a time; a new target rebinds the view, a repeated rect repositions it. + */ +function useNativeVideoOverlay( + callId: string, + active: boolean, + target: OverlayTarget | undefined, + slotNode: HTMLDivElement | null +): void { + const targetKey = active && target ? `${target.participantIdentity}:${target.trackId}` : null; + + useEffect(() => { + if (!targetKey || !target || !slotNode) return undefined; + + let lastGeometryKey = ''; + const report = () => { + const rect = slotNode.getBoundingClientRect(); + // Slot hidden, outside the viewport (display:none page, scrolled away), or + // occluded by an overlaying page/drawer: hide the native overlay too, + // otherwise it stays painted at a stale position over unrelated content. + const offscreen = + rect.width <= 0 || + rect.height <= 0 || + rect.right < 0 || + rect.bottom < 0 || + rect.left > window.innerWidth || + rect.top > window.innerHeight || + nativeSlotOccluded(slotNode, rect); + if (offscreen) { + if (lastGeometryKey !== '') { + lastGeometryKey = ''; + void clearNativeCallRemoteVideoOverlay({ callId }).catch(() => undefined); + } + return; + } + const geometryKey = `${Math.round(rect.x)},${Math.round(rect.y)},${Math.round(rect.width)},${Math.round(rect.height)}`; + if (geometryKey === lastGeometryKey) return; + lastGeometryKey = geometryKey; + void setNativeCallRemoteVideoOverlay({ + callId, + participantIdentity: target.participantIdentity, + trackId: target.trackId, + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + devicePixelRatio: window.devicePixelRatio || 1, + }).catch(() => undefined); + }; + + report(); + const observer = new ResizeObserver(report); + observer.observe(slotNode); + window.addEventListener('resize', report); + // Scroll is the dominant rect-changing event: the message list (or any + // nested scroll container) moves the slot without resizing it. Capture + // phase reaches scrolls from nested containers, which don't bubble. + document.addEventListener('scroll', report, { capture: true, passive: true }); + return () => { + observer.disconnect(); + window.removeEventListener('resize', report); + document.removeEventListener('scroll', report, { capture: true }); + }; + }, [targetKey, target, slotNode, callId]); + + // No eligible target (camera muted/unpublished, reconnecting, error): the + // overlay must not linger over the tile grid. + useEffect(() => { + if (targetKey) return; + void clearNativeCallRemoteVideoOverlay({ callId }).catch(() => undefined); + }, [targetKey, callId]); + + useEffect( + () => () => { + // Unmount safety: a stale callId is a native no-op. + void clearNativeCallRemoteVideoOverlay({ callId }).catch(() => undefined); + }, + [callId] + ); +} + +/** + * Mirrors `useNativeVideoOverlay` for the local camera preview. Active only + * when the session's camera is on and the call is connected; reports the + * local tile's slot rect to the native side. + */ +function useNativeLocalVideoOverlay( + callId: string, + enabled: boolean, + cameraEnabled: boolean, + slotNode: HTMLDivElement | null +): void { + const active = enabled && cameraEnabled && slotNode; + + useEffect(() => { + if (!active || !slotNode) return undefined; + + let lastGeometryKey = ''; + const report = () => { + // Never tear down the preview while the page is hidden (e.g. PiP owns + // the layer during backgrounding). + if (document.visibilityState !== 'visible') return; + const rect = slotNode.getBoundingClientRect(); + // Slot hidden, outside the viewport (scrolled away), or occluded by an + // overlaying page/drawer: hide the native overlay too, otherwise it + // stays painted at a stale position over unrelated content. + const hidden = + rect.width <= 0 || + rect.height <= 0 || + rect.right < 0 || + rect.bottom < 0 || + rect.left > window.innerWidth || + rect.top > window.innerHeight || + nativeSlotOccluded(slotNode, rect); + if (hidden) { + if (lastGeometryKey !== '') { + lastGeometryKey = ''; + void clearNativeCallLocalVideoOverlay({ callId }).catch(() => undefined); + } + return; + } + const geometryKey = `${Math.round(rect.x)},${Math.round(rect.y)},${Math.round(rect.width)},${Math.round(rect.height)}`; + if (geometryKey === lastGeometryKey) return; + lastGeometryKey = geometryKey; + void setNativeCallLocalVideoOverlay({ + callId, + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + devicePixelRatio: window.devicePixelRatio || 1, + }).catch(() => undefined); + }; + + report(); + const observer = new ResizeObserver(report); + observer.observe(slotNode); + // Fires during slide transitions (transforms move the slot without any + // scroll/resize event), which is how recovery after a room change works. + const intersectionObserver = new IntersectionObserver(report, { + threshold: [0, 0.25, 0.5, 0.75, 1], + }); + intersectionObserver.observe(slotNode); + window.addEventListener('resize', report); + // Scroll is the dominant rect-changing event: any nested scroll container + // moves the slot without resizing it. Capture phase reaches scrolls from + // nested containers, which don't bubble. + document.addEventListener('scroll', report, { capture: true, passive: true }); + return () => { + observer.disconnect(); + intersectionObserver.disconnect(); + window.removeEventListener('resize', report); + document.removeEventListener('scroll', report, { capture: true }); + }; + }, [callId, slotNode, active]); + + useEffect(() => { + if (active) return; + void clearNativeCallLocalVideoOverlay({ callId }).catch(() => undefined); + }, [callId, active]); + + useEffect( + () => () => { + void clearNativeCallLocalVideoOverlay({ callId }).catch(() => undefined); + }, + [callId] + ); +} + +function LocalTile({ + session, + userIdByIdentity, + slotRef, + fixed, +}: { + session: NativeCallSession; + userIdByIdentity: UserIdByRtcIdentity; + slotRef?: (node: HTMLDivElement | null) => void; + fixed?: boolean; +}) { + const profile = useCallParticipantProfile('', true, userIdByIdentity); + + return ( +
+ {/* When the camera is on, the native local preview renders over this + slot; the avatar stays mounted underneath as the placeholder. */} +
+
+ +
+
+
+ {!session.microphoneEnabled && ( + + {sizedIcon(MicrophoneSlash, '200')} + + )} + You +
+
+ ); +} + +type RemoteTileProps = { + participant: NativeCallRemoteParticipant; + userIdByIdentity: UserIdByRtcIdentity; + videoBound: boolean; + slotRef?: (node: HTMLDivElement | null) => void; + fixed?: boolean; +}; + +function RemoteTile({ + participant, + userIdByIdentity, + videoBound, + slotRef, + fixed, +}: RemoteTileProps) { + const profile = useCallParticipantProfile(participant.identity, false, userIdByIdentity); + return ( +
+ {/* When video is bound, the native view renders exactly over this slot; + the avatar stays mounted underneath as the pre-video placeholder. */} +
+
+ +
+
+
+ + {participant.camera?.muted && ( + + {sizedIcon(VideoCameraSlash, '200')} + + )} + {profile.name} +
+
+ ); +} + +/** + * Output picker (speaker, receiver, Bluetooth). The button only appears once the + * platform actually reports routes, which is how it stays hidden on Android, + * where the command does not exist. + */ +function AudioRouteControl({ + session, + onMenuOpenChange, +}: { + session: NativeCallSession; + onMenuOpenChange: (open: boolean) => void; +}) { + const menu = useMenuAnchor(); + const [routes, setRoutes] = useState([]); + const open = menu.anchor !== undefined; + + useEffect(() => { + let disposed = false; + const load = async () => { + const next = await session.listAudioRoutes(); + if (!disposed) setRoutes(next); + }; + void load(); + return () => { + disposed = true; + }; + }, [session]); + + // The native video view sits above the webview, so it would paint over this + // menu. Report the open state so the surface can drop the overlay while it is + // up, the same way an occluding drawer does. + useEffect(() => { + onMenuOpenChange(open); + return () => onMenuOpenChange(false); + }, [open, onMenuOpenChange]); + + if (routes.length === 0) return null; + + return ( + + + {routes.map((route) => ( + + } + onClick={() => { + void session.selectAudioRoute(route.id); + menu.close(); + }} + > + {route.name} + + ))} + + + } + > + + + ); +} + +const qualityLabels: Record = { + excellent: 'Excellent connection', + good: 'Good connection', + poor: 'Poor connection', + lost: 'Connection lost', + unknown: 'Connection quality unknown', +}; + +/** The dot carries state through colour alone, so it needs its own label. */ +function QualityDot({ quality }: { quality: string | undefined }) { + const value = quality ?? 'unknown'; + return ( + + ); +} + +function RemoteDominantLabel({ + participant, + userIdByIdentity, +}: { + participant: NativeCallRemoteParticipant; + userIdByIdentity: UserIdByRtcIdentity; +}) { + const profile = useCallParticipantProfile(participant.identity, false, userIdByIdentity); + return ( + <> + + {participant.camera?.muted && ( + + {sizedIcon(VideoCameraSlash, '200')} + + )} + {profile.name} + + ); +} + +function RemoteDominantPlaceholder({ + participant, + userIdByIdentity, +}: { + participant: NativeCallRemoteParticipant; + userIdByIdentity: UserIdByRtcIdentity; +}) { + const profile = useCallParticipantProfile(participant.identity, false, userIdByIdentity, 192); + return ; +} + +/** + * Edge-to-edge tile for the layouts where one participant owns the whole + * stage: the local preview when the call is still empty, or the single remote + * in a two-person call (FaceTime-style). + */ +function DominantTile({ + slotRef, + videoBound, + placeholder, + label, +}: { + slotRef?: (node: HTMLDivElement | null) => void; + videoBound?: boolean; + placeholder: ReactNode; + label: ReactNode; +}) { + return ( +
+ {/* The slot's rect is what JS reports to the native video overlay; the + placeholder stays mounted underneath as the pre-video fallback. */} +
+
+ {placeholder} +
+
+
{label}
+
+ ); +} + +export function NativeCallSurface({ session, onHangup }: NativeCallSurfaceProps) { + const isError = session.lifecycle === 'error'; + const connected = session.lifecycle === 'connected'; + const remoteParticipants = useNativeRemoteParticipants(session.callId, !isError); + const matrixRoom = useRoom(); + const callSession = useCallSession(matrixRoom); + const callMembers = useCallMembers(matrixRoom, callSession); + const userIdByIdentity = useMemo(() => buildRtcIdentityMap(callMembers), [callMembers]); + + const featured = useMemo(() => { + const live = (track: NativeCallRemoteCamera | undefined): boolean => + track !== undefined && track.subscribed && !track.muted; + // A shared screen is the thing people are actually looking at, so it + // outranks any camera. + const sharing = remoteParticipants.find((p) => live(p.screenShare)); + if (sharing?.screenShare) { + return { participantIdentity: sharing.identity, trackId: sharing.screenShare.sid }; + } + const onCamera = remoteParticipants.find((p) => live(p.camera)); + if (!onCamera?.camera) return undefined; + return { participantIdentity: onCamera.identity, trackId: onCamera.camera.sid }; + }, [remoteParticipants]); + + // The room page stays mounted for the whole mobile slide-out transition, so + // geometry alone leaves the native video painted over the outgoing page until + // the slot finally clears the viewport. Route selection flips when the + // transition starts, which is the earliest honest "no longer on screen". + const selectedRoom = useSelectedRoom(); + const [routeMenuOpen, setRouteMenuOpen] = useState(false); + const overlayActive = connected && selectedRoom === session.roomId && !routeMenuOpen; + + const [slotNode, setSlotNode] = useState(null); + useNativeVideoOverlay(session.callId, overlayActive, featured, slotNode); + + const [localSlotNode, setLocalSlotNode] = useState(null); + useNativeLocalVideoOverlay(session.callId, overlayActive, session.cameraEnabled, localSlotNode); + + const remoteCount = remoteParticipants.length; + // Total tiles once the local self-tile joins the grid. 7+ switches the grid + // to its compact, scrollable three-column variant. + const tileCount = remoteCount + 1; + const compactGrid = tileCount > 6; + + // Two-person layout: the remote owns the stage and is video-bound here + // whenever its camera is the featured track. + const duoRemote = remoteCount === 1 ? remoteParticipants[0] : undefined; + const duoLive = + duoRemote !== undefined && connected && featured?.participantIdentity === duoRemote.identity; + + if (isError) { + return ( + + + + ); + } + + return ( + + {!connected && ( +
+ + {nativeCallLifecycleLabels[session.lifecycle]} + +
+ )} + {duoRemote ? ( +
+ + } + label={ + + } + /> + {session.cameraEnabled && ( +
+
+
+ {sizedIcon(User, '200')} +
+
+
+ )} +
+ ) : remoteCount === 0 ? ( +
+ + {!session.microphoneEnabled && ( + + {sizedIcon(MicrophoneSlash, '200')} + + )} + You + + } + /> +
+ ) : ( +
+ {remoteParticipants.map((participant) => { + const live = connected && featured?.participantIdentity === participant.identity; + return ( + + ); + })} + +
+ )} + + + {connected && } + {session.cameraEnabled && connected && ( + + )} + + + + ); +} diff --git a/src/app/features/call/PrescreenControls.tsx b/src/app/features/call/PrescreenControls.tsx index dd721c3cd..464fdf89c 100644 --- a/src/app/features/call/PrescreenControls.tsx +++ b/src/app/features/call/PrescreenControls.tsx @@ -8,6 +8,9 @@ import { useIsDirectRoom, useRoom } from '../../hooks/useRoom'; import { useCallEmbed, useCallJoined, useCallStart } from '../../hooks/useCallEmbed'; import { useCallPreferences } from '../../state/hooks/callPreferences'; import { ScreenSize, useScreenSizeContext } from '../../hooks/useScreenSize'; +import { settingsAtom } from '$state/settings'; +import { useSetting } from '$state/hooks/settings'; +import { CallDevicePreview } from './CallDevicePreview'; type PrescreenControlsProps = { canJoin?: boolean; @@ -28,70 +31,98 @@ export function PrescreenControls({ canJoin }: PrescreenControlsProps) { const disabled = inOtherCall || !canJoin; - const { microphone, video, sound, toggleMicrophone, toggleVideo, toggleSound } = - useCallPreferences(); + const { + microphone, + video, + sound, + audioDeviceId, + videoDeviceId, + toggleMicrophone, + toggleVideo, + toggleSound, + setAudioDeviceId, + setVideoDeviceId, + } = useCallPreferences(); + // Only the new-call path applies the chosen devices; Element Call picks its + // own, so previewing there would promise something we cannot honour. + const [newCallsEnabled] = useSetting(settingsAtom, 'newCallsEnabled'); + const showPreview = newCallsEnabled && !disabled && !joining && !compact; return ( - - + {showPreview && ( + + )} + - - - - + + + {!compact && } + - {!compact && } + {!compact && } - - - {room?.isCallRoom() && } - - - - - - + + {room?.isCallRoom() && } + + + + + + + ); } diff --git a/src/app/features/call/callChrome.css.ts b/src/app/features/call/callChrome.css.ts new file mode 100644 index 000000000..c5951ccd9 --- /dev/null +++ b/src/app/features/call/callChrome.css.ts @@ -0,0 +1,115 @@ +import { style } from '@vanilla-extract/css'; +import { color, config, toRem } from 'folds'; + +/** + * Shared call chrome styles: the canvas shell, the glassmorphic control + * pill, and the on/off media toggle button. Both call surfaces (livekit-js and + * native) render into these so the chrome reads consistently across engines + * while each engine keeps its own video substrate and connection logic. + */ + +/** + * The full-height dark canvas. `position / width / height` stay inline on the + * `CallLayout` component (the livekit-js surface asserts them via `toHaveStyle` + * and the canvas already used inline styles), so this class carries only the + * rest of the shell. + */ +export const callLayout = style({ + minHeight: 0, + overflow: 'hidden', + background: color.Background.Container, +}); + +/** + * Floating control bar that overlays the video canvas (livekit-js). Drives the + * auto-hide opacity/visibility transition; `opacity` / `visibility` are applied + * inline by `CallControlBar` from its `visible` prop, and the pill re-enables + * hit-testing, so the wrapper stays transparent to pointer events. + */ +export const controlBarOverlay = style({ + position: 'absolute', + left: 0, + right: 0, + bottom: 0, + zIndex: 4, + display: 'flex', + justifyContent: 'center', + padding: `${config.space.S200} ${config.space.S300} calc(${config.space.S300} + env(safe-area-inset-bottom, 0px))`, + transition: 'opacity 160ms ease, visibility 160ms ease', + pointerEvents: 'none', +}); + +/** + * In-flow control bar that sits below the tile grid (native). The native video + * overlay fully occludes anything beneath it, so controls live in normal flow + * next to (never on top of) a video slot, and never auto-hide. + */ +export const controlBarFlow = style({ + flexShrink: 0, + display: 'flex', + justifyContent: 'center', + padding: `${config.space.S100} ${config.space.S300} calc(${config.space.S300} + env(safe-area-inset-bottom, 0px))`, +}); + +/** + * The glassmorphic pill that holds the control buttons. Shared by both control + * bar layouts; only the wrapper (overlay vs flow) and hit-testing differ. + */ +export const controlPill = style({ + display: 'flex', + alignItems: 'center', + // Wraps rather than overflowing: the parent clips, so an unwrapped row loses + // the End call button off the edge on a narrow viewport. + flexWrap: 'wrap', + justifyContent: 'center', + gap: config.space.S200, + maxWidth: '100%', + padding: config.space.S100, + border: `1px solid ${color.Surface.ContainerLine}`, + borderRadius: config.radii.R500, + background: color.Surface.Container, + backdropFilter: 'blur(12px)', +}); + +/** + * Round on/off toggle for mic and camera. `data-on` drives the on (neutral) vs + * off (muted-red) affordance; identical to the native surface's previous button + * so its behavior and look are unchanged. + */ +export const controlButton = style({ + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + minWidth: toRem(44), + minHeight: toRem(44), + padding: 0, + border: 'none', + borderRadius: '50%', + font: 'inherit', + cursor: 'pointer', + transition: 'background-color 120ms ease', + selectors: { + '&[data-on="true"]': { + background: color.SurfaceVariant.Container, + color: color.SurfaceVariant.OnContainer, + }, + '&[data-on="true"]:hover:not(:disabled)': { + background: color.SurfaceVariant.ContainerHover, + }, + '&[data-on="false"]': { + background: color.Critical.Container, + color: color.Critical.OnContainer, + }, + '&[data-on="false"]:hover:not(:disabled)': { + background: color.Critical.ContainerHover, + }, + '&:focus-visible': { + outline: `2px solid var(--sable-primary-main, #7aa2ff)`, + outlineOffset: '2px', + }, + '&:disabled': { + opacity: 0.45, + cursor: 'default', + }, + }, +}); diff --git a/src/app/features/call/callChrome.tsx b/src/app/features/call/callChrome.tsx new file mode 100644 index 000000000..e1f0b4ec4 --- /dev/null +++ b/src/app/features/call/callChrome.tsx @@ -0,0 +1,241 @@ +import type { + CSSProperties, + FocusEventHandler, + KeyboardEventHandler, + PointerEventHandler, + ReactNode, +} from 'react'; +import { Box, Button, color, Text } from 'folds'; +import { + Microphone, + MicrophoneSlash, + VideoCamera, + VideoCameraSlash, + sizedIcon, +} from '$components/icons/phosphor'; +import type { CallClient, CallStatusView } from './callClient'; +import * as css from './callChrome.css'; + +/** + * The full-height dark canvas both call surfaces render into. The livekit-js + * surface layers absolutely-positioned tiles over it and marks it for + * automation (`callSurfaceMarker`) with pointer handlers that drive control + * auto-hide; the native surface stacks its status row, tile grid, and control + * bar in normal flow (`stack`), because native video overlays occlude anything + * beneath them. + */ +export type CallLayoutProps = { + children: ReactNode; + stack?: boolean; + callSurfaceMarker?: boolean; + className?: string; + style?: CSSProperties; + onPointerMove?: PointerEventHandler; + onPointerDown?: PointerEventHandler; + onFocusCapture?: FocusEventHandler; + onKeyDown?: KeyboardEventHandler; +}; + +export function CallLayout({ + children, + stack, + callSurfaceMarker, + className, + style, + onPointerMove, + onPointerDown, + onFocusCapture, + onKeyDown, +}: CallLayoutProps) { + const mergedClassName = [css.callLayout, className].filter(Boolean).join(' ') || undefined; + return ( + + {children} + + ); +} + +/** + * The control bar shell. `overlay` floats over the canvas and auto-hides + * (livekit-js: `visible` drives opacity/visibility, the pill re-enables + * hit-testing); `flow` sits in normal layout below the tiles and is always + * visible (native). The actual controls are passed as children: the two + * engines render different buttons (livekit-js uses its component-library + * `ControlBar` with device menus; native uses the shared `CallMediaControls` + * toggles), so only the shell is shared. + */ +export type CallControlBarProps = { + children: ReactNode; + layout?: 'overlay' | 'flow'; + visible?: boolean; + onFocusCapture?: FocusEventHandler; +}; + +export function CallControlBar({ + children, + layout = 'flow', + visible, + onFocusCapture, +}: CallControlBarProps) { + if (layout === 'overlay') { + return ( + +
+ {children} +
+ + ); + } + + return ( + +
{children}
+ + ); +} + +/** + * Full-screen status display: lifecycle label, optional error detail, and an + * End / Dismiss button. Used by the livekit-js surface for every non-active + * lifecycle (it has no room to render until `active`) and by the native surface + * for its terminal error state. Both feed a `CallStatusView` produced by their + * engine's status adapter, so the chrome and label maps stay shared. + */ +export function CallStatusBar({ + status, + onHangup, +}: { + status: CallStatusView; + onHangup: () => void; +}) { + const failed = status.phase === 'failed'; + return ( + + + {status.statusLabel} + {status.error && ( + + {status.error} + + )} + + + + ); +} + +/** + * Round on/off toggle button for mic and camera. `data-on` / `aria-pressed` + * carry the on (neutral) vs off (muted-red) state. Extracted from the native + * surface so the affordance is owned by the shared chrome. + */ +export type CallMediaToggleButtonProps = { + on: boolean; + disabled?: boolean; + label: string; + children: ReactNode; + onToggle: () => void; +}; + +export function CallMediaToggleButton({ + on, + disabled, + label, + children, + onToggle, +}: CallMediaToggleButtonProps) { + return ( + + ); +} + +/** + * Mic + camera toggles built from `CallMediaToggleButton`. Engines that expose + * direct media toggles (the native engine) render this inside their control + * bar; the livekit-js engine delegates to its component-library `ControlBar` + * instead, so it does not use this. + * + * The props are the media slice of `CallClient`, required: the native session + * always provides them. + */ +export type CallMediaControlsProps = Required< + Pick< + CallClient, + 'microphoneEnabled' | 'cameraEnabled' | 'setMicrophoneEnabled' | 'setCameraEnabled' + > +> & { + disabled?: boolean; +}; + +export function CallMediaControls({ + microphoneEnabled, + cameraEnabled, + setMicrophoneEnabled, + setCameraEnabled, + disabled, +}: CallMediaControlsProps) { + return ( + <> + void setMicrophoneEnabled(!microphoneEnabled)} + > + {sizedIcon(microphoneEnabled ? Microphone : MicrophoneSlash, '300', { + filled: !microphoneEnabled, + })} + + void setCameraEnabled(!cameraEnabled)} + > + {sizedIcon(cameraEnabled ? VideoCamera : VideoCameraSlash, '300', { + filled: cameraEnabled, + })} + + + ); +} diff --git a/src/app/features/call/callClient.ts b/src/app/features/call/callClient.ts new file mode 100644 index 000000000..b6fae3a07 --- /dev/null +++ b/src/app/features/call/callClient.ts @@ -0,0 +1,139 @@ +import type { + LivekitJsControllerFailure, + LivekitJsControllerLifecycle, +} from './livekitJsController'; +import type { NativeCallLifecycle } from '$state/nativeCall'; + +/** + * Coarse lifecycle phase both call engines map to. The engine-specific + * lifecycle unions genuinely differ: LiveKit JS splits "connecting" into + * joining-matrix / provisioning / connecting-livekit, and Native adds + * "reconnecting", so the shared chrome reasons about a coarse phase while each + * engine keeps its own union and its own user-facing label map. + */ +type CallPhase = 'idle' | 'connecting' | 'connected' | 'reconnecting' | 'stopping' | 'failed'; + +/** + * The contract every call engine satisfies. Both `NativeCallSession` (iOS, + * LiveKit Swift SDK) and `LivekitJsCallSession` (Web / Desktop Tauri, livekit-js) + * expose a room id, a lifecycle, and a hangup. + * + * Direct media toggles are optional: the native engine exposes them so its + * surface can render its own mic/camera buttons; the livekit-js engine delegates + * media controls to `@livekit/components-react`'s `ControlBar`, so it does not. + * Engine-specific fields (LiveKit's `room` / `e2eeReady` / `initialMedia`, + * Native's `callId` / `backend` / `error`) stay on the concrete session and are + * read directly by each surface. + */ +export interface CallClient { + readonly roomId: string; + readonly lifecycle: TLifecycle; + readonly hangup: () => Promise; + readonly microphoneEnabled?: boolean; + readonly cameraEnabled?: boolean; + readonly setMicrophoneEnabled?: (enabled: boolean) => Promise; + readonly setCameraEnabled?: (enabled: boolean) => Promise; +} + +/** + * The status slice the shared `CallStatusBar` renders: a coarse phase (drives + * the End vs Dismiss button), the resolved label line, and optional error + * detail. Each engine produces one from its own lifecycle + error shape. + */ +export type CallStatusView = { + phase: CallPhase; + statusLabel: string; + error?: string; +}; + +/** + * Lifecycle → user-facing label maps. Colocated so the two surfaces share one + * source of truth for status copy; they remain two maps because the lifecycle + * unions (and therefore the distinct labels "Preparing call", "Reconnecting", + * …) genuinely differ. + */ +const livekitJsLifecycleLabels: Record = { + idle: 'Idle', + 'joining-matrix': 'Joining call', + provisioning: 'Preparing call', + 'connecting-livekit': 'Connecting', + active: 'Connected', + stopping: 'Ending call', + failed: 'Call failed', +}; + +const livekitJsFailureMessages: Record = { + 'e2ee-unsupported': 'Encrypted calls are not supported on this device.', + 'e2ee-import-failed': 'Could not set up call encryption.', + 'setup-failed': 'Could not connect to the call.', +}; + +export const nativeCallLifecycleLabels: Record = { + starting: 'Starting call', + connecting: 'Connecting', + connected: 'Connected', + reconnecting: 'Reconnecting', + error: 'Call failed', +}; + +function livekitJsPhase(lifecycle: LivekitJsControllerLifecycle): CallPhase { + switch (lifecycle) { + case 'idle': + return 'idle'; + case 'active': + return 'connected'; + case 'stopping': + return 'stopping'; + case 'failed': + return 'failed'; + default: + // joining-matrix, provisioning, connecting-livekit are all "connecting". + return 'connecting'; + } +} + +function nativePhase(lifecycle: NativeCallLifecycle): CallPhase { + switch (lifecycle) { + case 'connected': + return 'connected'; + case 'reconnecting': + return 'reconnecting'; + case 'error': + return 'failed'; + default: + // starting, connecting are both "connecting". + return 'connecting'; + } +} + +/** + * Project a LiveKit JS session into the shared status view. Accepts just the + * lifecycle + failure slice so the CallView status wrapper (and tests) can feed + * a partial session. + */ +export function livekitJsCallStatus(session: { + lifecycle: LivekitJsControllerLifecycle; + failure: LivekitJsControllerFailure | null; +}): CallStatusView { + return { + phase: livekitJsPhase(session.lifecycle), + statusLabel: livekitJsLifecycleLabels[session.lifecycle], + error: session.failure ? livekitJsFailureMessages[session.failure] : undefined, + }; +} + +/** + * Project a native call session into the shared status view. Native carries its + * error as a plain string (the native SDK's failure message), so it is passed + * through verbatim. + */ +export function nativeCallStatus(session: { + lifecycle: NativeCallLifecycle; + error?: string; +}): CallStatusView { + return { + phase: nativePhase(session.lifecycle), + statusLabel: nativeCallLifecycleLabels[session.lifecycle], + error: session.error, + }; +} diff --git a/src/app/features/call/callStartCapabilities.ts b/src/app/features/call/callStartCapabilities.ts index 9de777230..b3d30dce9 100644 --- a/src/app/features/call/callStartCapabilities.ts +++ b/src/app/features/call/callStartCapabilities.ts @@ -1,6 +1,7 @@ -import type { Room } from '$types/matrix-sdk'; +import { EventType, type Room } from '$types/matrix-sdk'; -const CALL_MEMBER_EVENT_TYPE = 'org.matrix.msc3401.call.member'; +const CALL_MEMBER_EVENT_TYPE = EventType.RTCMembership; +const LEGACY_CALL_MEMBER_EVENT_TYPE = EventType.GroupCallMemberPrefix; export type CallStartBlocker = | 'missing_webrtc' @@ -45,7 +46,9 @@ export const evaluateCallStartCapabilities = ({ }: EvaluateCallStartCapabilitiesInput): CallStartCapabilities => { const blockers: CallStartBlocker[] = []; const hasCallMemberPermission = - room.currentState?.maySendStateEvent(CALL_MEMBER_EVENT_TYPE, myUserId) ?? false; + room.currentState?.maySendStateEvent(CALL_MEMBER_EVENT_TYPE, myUserId) || + room.currentState?.maySendStateEvent(LEGACY_CALL_MEMBER_EVENT_TYPE, myUserId) || + false; const inAnotherCall = !!activeCallRoomId && activeCallRoomId !== room.roomId; if (!rtcSupported) blockers.push('missing_webrtc'); diff --git a/src/app/features/call/callStartSelection.test.ts b/src/app/features/call/callStartSelection.test.ts new file mode 100644 index 000000000..ec6122205 --- /dev/null +++ b/src/app/features/call/callStartSelection.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import { selectCallStartOwner } from './callStartSelection'; + +describe('selectCallStartOwner', () => { + it('selects the native transport when available, even over LiveKit JS', () => { + expect(selectCallStartOwner({ newCallsEnabled: true, nativeCallAvailable: true })).toBe( + 'livekit-mobile' + ); + }); + + it('retains LiveKit JS when the native transport is unavailable', () => { + expect(selectCallStartOwner({ newCallsEnabled: true, nativeCallAvailable: false })).toBe( + 'livekit-js' + ); + }); + + it('selects LiveKit JS when new calls are enabled', () => { + expect(selectCallStartOwner({ newCallsEnabled: true })).toBe('livekit-js'); + }); + + it('falls back to Element Call when new calls are disabled', () => { + expect(selectCallStartOwner({ newCallsEnabled: false })).toBe('element'); + }); + + it('keeps the Element fallback when new calls are disabled, even if native is available', () => { + expect(selectCallStartOwner({ newCallsEnabled: false, nativeCallAvailable: true })).toBe( + 'element' + ); + }); +}); diff --git a/src/app/features/call/callStartSelection.ts b/src/app/features/call/callStartSelection.ts new file mode 100644 index 000000000..0f6951347 --- /dev/null +++ b/src/app/features/call/callStartSelection.ts @@ -0,0 +1,13 @@ +export type CallStartOwner = 'livekit-mobile' | 'livekit-js' | 'element'; + +export const selectCallStartOwner = ({ + newCallsEnabled, + nativeCallAvailable = false, +}: { + newCallsEnabled: boolean; + nativeCallAvailable?: boolean; +}): CallStartOwner => { + if (!newCallsEnabled) return 'element'; + if (nativeCallAvailable) return 'livekit-mobile'; + return 'livekit-js'; +}; diff --git a/src/app/features/call/livekitCallIdentity.test.ts b/src/app/features/call/livekitCallIdentity.test.ts new file mode 100644 index 000000000..b5f66b00b --- /dev/null +++ b/src/app/features/call/livekitCallIdentity.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import type { CallMembership } from '$types/matrix-sdk'; +import { buildRtcIdentityMap } from './livekitCallIdentity'; + +const membership = (rtcBackendIdentity: string, userId?: string, deviceId?: string) => + ({ + rtcBackendIdentity, + userId, + deviceId, + memberId: userId && deviceId ? `${userId}:${deviceId}` : rtcBackendIdentity, + }) as CallMembership; + +describe('buildRtcIdentityMap', () => { + it('maps each anonymised LiveKit identity back to its Matrix user', () => { + const identities = buildRtcIdentityMap([ + membership('sha256-alice', '@alice:example.org'), + membership('sha256-bob', '@bob:example.org'), + ]); + + expect(identities.get('sha256-alice')).toBe('@alice:example.org'); + expect(identities.get('sha256-bob')).toBe('@bob:example.org'); + }); + + it('skips memberships with no sender rather than mapping them to undefined', () => { + const identities = buildRtcIdentityMap([ + membership('sha256-alice', '@alice:example.org'), + membership('sha256-orphan'), + ]); + + expect(identities.has('sha256-orphan')).toBe(false); + expect(identities.size).toBe(1); + }); + + it('returns an empty map before any membership is known', () => { + expect(buildRtcIdentityMap([]).size).toBe(0); + }); + + it('also indexes the legacy user:device identity the SFU may report instead', () => { + const identities = buildRtcIdentityMap([ + membership('sha256-alice', '@alice:example.org', 'DEVICE'), + ]); + + expect(identities.get('sha256-alice')).toBe('@alice:example.org'); + expect(identities.get('@alice:example.org:DEVICE')).toBe('@alice:example.org'); + }); +}); diff --git a/src/app/features/call/livekitCallIdentity.ts b/src/app/features/call/livekitCallIdentity.ts new file mode 100644 index 000000000..d9a5f37a8 --- /dev/null +++ b/src/app/features/call/livekitCallIdentity.ts @@ -0,0 +1,29 @@ +import type { CallMembership } from '$types/matrix-sdk'; + +/** Maps a LiveKit participant identity to the Matrix user behind it. */ +export type UserIdByRtcIdentity = ReadonlyMap; + +/** + * Our own user and device, used to recognise our key among the ones + * `EncryptionKeyChanged` reports. The LiveKit identity itself cannot be derived + * locally: it depends on the membership format the SFU was told to expect, so + * it is only ever read off the event. + */ +export type LocalCallIdentity = { userId: string; deviceId: string | null }; + +// The SFU decides what a LiveKit participant identity looks like: the +// anonymised SHA-256 for sticky-event RTC memberships, or `user:device` on the +// legacy path. Index every candidate so either shape resolves. +export const buildRtcIdentityMap = (members: CallMembership[]): UserIdByRtcIdentity => { + const identities = new Map(); + members.forEach((member) => { + const { userId, deviceId } = member; + if (!userId) return; + [member.rtcBackendIdentity, member.memberId, deviceId && `${userId}:${deviceId}`].forEach( + (candidate) => { + if (candidate) identities.set(candidate, userId); + } + ); + }); + return identities; +}; diff --git a/src/app/features/call/livekitJsCallManager.test.tsx b/src/app/features/call/livekitJsCallManager.test.tsx new file mode 100644 index 000000000..e4ec50463 --- /dev/null +++ b/src/app/features/call/livekitJsCallManager.test.tsx @@ -0,0 +1,311 @@ +import type { ReactNode } from 'react'; +import { useEffect } from 'react'; +import { act, render, waitFor } from '@testing-library/react'; +import { createStore, Provider as JotaiProvider } from 'jotai'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import { MatrixClientProvider } from '$hooks/useMatrixClient'; +import { AutoDiscoveryInfoProvider } from '$hooks/useAutoDiscoveryInfo'; +import { CallEmbedRefContextProvider } from '$hooks/useCallEmbed'; +import { getSettings, settingsAtom, type Settings } from '$state/settings'; +import { livekitJsCallAtom } from '$state/livekitJsCall'; +import { nativeCallAtom, type NativeCallSession } from '$state/nativeCall'; +import { + type LivekitJsCallManager, + LivekitJsCallManagerProvider, + useLivekitJsCallManager, +} from './livekitJsCallManager'; +import type { LivekitJsControllerState } from './livekitJsController'; + +vi.mock('$hooks/useTheme', () => ({ + ThemeKind: { Light: 'light', Dark: 'dark' }, + useTheme: () => ({ kind: 'light' }), +})); + +vi.mock('$hooks/useClientConfig', () => ({ + useClientConfig: () => ({ elementCallUrl: undefined }), +})); + +const { createControllerMock } = vi.hoisted(() => ({ + createControllerMock: vi.fn<(...args: unknown[]) => unknown>(), +})); + +vi.mock('./livekitJsController', () => ({ + createLivekitJsController: createControllerMock, +})); + +type ControllerListener = (state: Readonly) => void; + +type FakeController = { + connect: ReturnType; + disconnect: ReturnType; + subscribe: (listener: ControllerListener) => () => void; + getState: () => LivekitJsControllerState; + emit: (state: Partial) => void; + listenerCount: () => number; +}; + +const makeFakeController = (): FakeController => { + const listeners = new Set(); + let state: LivekitJsControllerState = { + lifecycle: 'idle', + failure: null, + e2ee: { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + }, + }; + const snapshot = (): LivekitJsControllerState => ({ + ...state, + e2ee: { ...state.e2ee }, + }); + return { + connect: vi.fn<(...args: unknown[]) => Promise>(async () => undefined), + disconnect: vi.fn<() => Promise>(async () => undefined), + getState: () => snapshot(), + subscribe: (listener) => { + listeners.add(listener); + listener(snapshot()); + return () => { + listeners.delete(listener); + }; + }, + emit: (next) => { + state = { ...state, ...next }; + listeners.forEach((listener) => listener(snapshot())); + }, + listenerCount: () => listeners.size, + }; +}; + +const room = { roomId: '!room:example.org' } as Room; + +const makeNativeSession = (lifecycle: NativeCallSession['lifecycle']): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId: room.roomId, + callId: 'native-call-id', + lifecycle, + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, +}); + +const memberships: string[] = []; +const mx = { + matrixRTC: { + getRoomSession: vi.fn<() => unknown>(() => ({ memberships })), + }, +} as unknown as MatrixClient; + +type Harness = { + store: ReturnType; + controllers: FakeController[]; + wrapper: ({ children }: { children?: ReactNode }) => ReactNode; + managers: Array; + Consumer: () => null; +}; + +const createHarness = (settingsOverrides: Partial = {}): Harness => { + const store = createStore(); + store.set(settingsAtom, { ...getSettings(), ...settingsOverrides }); + const controllers: FakeController[] = []; + createControllerMock.mockImplementation(() => { + const controller = makeFakeController(); + controllers.push(controller); + return controller; + }); + const managers: Array = []; + const Consumer = (): null => { + managers.push(useLivekitJsCallManager()); + return null; + }; + const embedRef = { current: null }; + const wrapper = ({ children }: { children?: ReactNode }) => ( + + + + + {children} + + + + + ); + return { store, controllers, wrapper, managers, Consumer }; +}; + +const currentManager = (harness: Harness): LivekitJsCallManager => { + const manager = harness.managers[harness.managers.length - 1]; + expect(manager).toBeDefined(); + return manager!; +}; + +describe('LivekitJsCallManagerProvider', () => { + beforeEach(() => { + createControllerMock.mockReset(); + }); + + it('start then consumer unmount/replacement does not disconnect the controller', () => { + const harness = createHarness(); + const view = render( + + + + ); + const controller = harness.controllers[0]!; + act(() => { + currentManager(harness).start({ room, video: true }); + }); + expect(controller.connect).toHaveBeenCalledTimes(1); + expect(controller.connect.mock.calls[0]![0]).toMatchObject({ + room, + callIntent: 'video', + }); + + // Consumer unmounts (prescreen replaced by call view) while provider persists. + view.rerender({undefined}); + + expect(controller.disconnect).not.toHaveBeenCalled(); + expect(controller.listenerCount()).toBe(1); + + // A replacement consumer mounts and still sees the same controller/manager. + view.rerender( + + + + ); + expect(controller.disconnect).not.toHaveBeenCalled(); + }); + + it('explicit end via session hangup disconnects exactly once', async () => { + const harness = createHarness(); + render( + + + + ); + const controller = harness.controllers[0]!; + act(() => { + currentManager(harness).start({ room, video: false }); + }); + act(() => { + controller.emit({ lifecycle: 'active', room: {} as never }); + }); + const session = harness.store.get(livekitJsCallAtom); + expect(session?.roomId).toBe(room.roomId); + expect(session?.lifecycle).toBe('active'); + + await act(async () => { + await session!.hangup(); + }); + + expect(controller.disconnect).toHaveBeenCalledTimes(1); + }); + + it('child passive effect (auto-join) can start on initial mount', () => { + const harness = createHarness({ newCallsEnabled: true }); + const AutoJoinChild = (): null => { + const manager = useLivekitJsCallManager(); + useEffect(() => { + manager?.start({ room, video: false }); + }, [manager]); + return null; + }; + + render( + + + + ); + + const controller = harness.controllers[0]!; + expect(controller.connect).toHaveBeenCalledTimes(1); + expect(controller.connect.mock.calls[0]![0]).toMatchObject({ + room, + callIntent: 'audio', + }); + }); + + it('provider unmount disconnects exactly once', async () => { + const harness = createHarness(); + const view = render({undefined}); + const controller = harness.controllers[0]!; + + view.unmount(); + + await waitFor(() => { + expect(controller.disconnect).toHaveBeenCalledTimes(1); + }); + expect(controller.listenerCount()).toBe(0); + expect(harness.store.get(livekitJsCallAtom)).toBeUndefined(); + }); + + it('multiple consumers share one controller for one call', () => { + const harness = createHarness({ newCallsEnabled: true }); + render( + + + + + + ); + const controller = harness.controllers[0]!; + + // Prescreen starts the call. + act(() => { + harness.managers[0]!.start({ room, video: false }); + }); + act(() => { + controller.emit({ lifecycle: 'joining-matrix' }); + }); + // Room header and auto-join attempt the same call while one is active. + act(() => { + harness.managers[1]!.start({ room, video: false }); + harness.managers[2]!.start({ room, video: false }); + }); + + expect(controller.connect).toHaveBeenCalledTimes(1); + }); + + it('refuses to start while a native call is active and publishes no failed session', () => { + const harness = createHarness(); + harness.store.set(nativeCallAtom, makeNativeSession('connected')); + render( + + + + ); + const controller = harness.controllers[0]!; + + act(() => { + currentManager(harness).start({ room, video: false }); + }); + + expect(controller.connect).not.toHaveBeenCalled(); + expect(harness.store.get(livekitJsCallAtom)).toBeUndefined(); + }); + + it('starts when the native session is in a terminal error state', () => { + const harness = createHarness(); + harness.store.set(nativeCallAtom, makeNativeSession('error')); + render( + + + + ); + const controller = harness.controllers[0]!; + + act(() => { + currentManager(harness).start({ room, video: false }); + }); + + expect(controller.connect).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/app/features/call/livekitJsCallManager.tsx b/src/app/features/call/livekitJsCallManager.tsx new file mode 100644 index 000000000..4c7584003 --- /dev/null +++ b/src/app/features/call/livekitJsCallManager.tsx @@ -0,0 +1,158 @@ +import type { ReactNode } from 'react'; +import { createContext, useCallback, useContext, useEffect, useMemo, useRef } from 'react'; +import { useStore } from 'jotai'; +import type { Room } from '$types/matrix-sdk'; +import { createLivekitJsController } from './livekitJsController'; +import { + isLivekitJsCallActive, + livekitJsCallAtom, + livekitJsCallInitialMediaAppliedAtom, + livekitJsCallSoundAtom, + type LivekitJsCallMedia, +} from '$state/livekitJsCall'; +import { callEmbedAtom } from '$state/callEmbed'; +import { isNativeCallActive, nativeCallAtom } from '$state/nativeCall'; +import { useMatrixClient } from '$hooks/useMatrixClient'; +import { useAutoDiscoveryInfo } from '$hooks/useAutoDiscoveryInfo'; + +export type LivekitJsCallStartOptions = { + room: Room; + dm?: boolean; + video?: boolean; + microphone?: boolean; + sound?: boolean; + audioDeviceId?: string; + videoDeviceId?: string; +}; + +export type LivekitJsCallManager = { + start: (options: LivekitJsCallStartOptions) => void; +}; + +const LivekitJsCallManagerContext = createContext(undefined); + +export const useLivekitJsCallManager = (): LivekitJsCallManager | undefined => + useContext(LivekitJsCallManagerContext); + +type LivekitJsController = ReturnType; + +type LivekitJsCallManagerProviderProps = { + children?: ReactNode; +}; + +/** + * Provider-level owner of the single LiveKit JS call controller. Mounted with + * CallEmbedProvider so transient consumers (prescreen, room header, auto-join) + * never own or disconnect the controller themselves. Owns: creation, + * subscription -> livekitJsCallAtom, and provider-unmount disconnect. + */ +export function LivekitJsCallManagerProvider({ children }: LivekitJsCallManagerProviderProps) { + const mx = useMatrixClient(); + const discovery = useAutoDiscoveryInfo(); + const store = useStore(); + const roomIdRef = useRef(undefined); + const initialMediaRef = useRef({ + microphone: true, + camera: false, + sound: true, + }); + const controllerRef = useRef(undefined); + const generationRef = useRef(0); + + // Create and expose the controller synchronously during render so child + // passive effects (e.g. AutoJoinManager calling start()) always see the + // current controller before the provider's own subscription effect runs. + const controller = useMemo(() => { + generationRef.current += 1; + return createLivekitJsController(); + }, []); + const generation = generationRef.current; + if (controllerRef.current !== controller) controllerRef.current = controller; + + useEffect(() => { + controllerRef.current = controller; + const unsubscribe = controller.subscribe((controllerState) => { + const roomId = roomIdRef.current; + if (!roomId) return; + if (controllerState.lifecycle === 'idle') { + store.set(livekitJsCallAtom, undefined); + return; + } + store.set(livekitJsCallAtom, { + roomId, + initialMedia: initialMediaRef.current, + lifecycle: controllerState.lifecycle, + failure: controllerState.failure, + room: controllerState.lifecycle === 'active' ? controllerState.room : undefined, + e2eeReady: controllerState.e2ee.ready, + hangup: () => controller.disconnect(), + }); + }); + return () => { + unsubscribe(); + roomIdRef.current = undefined; + if (controllerRef.current === controller) controllerRef.current = undefined; + void controller.disconnect().finally(() => { + // A stale disconnect completing after a deliberate replacement must + // not clear the newer controller's session from the atom. + if (generationRef.current === generation) { + store.set(livekitJsCallAtom, undefined); + } + }); + }; + }, [controller, generation, store]); + + const start = useCallback( + ({ + room, + dm, + video, + microphone, + sound, + audioDeviceId, + videoDeviceId, + }: LivekitJsCallStartOptions) => { + // Guard mirrors nativeCallManager.start: a single call owner at a time. + // Without the native guard, connect would fail the call-owner lease and + // publish a spurious 'failed' session while the native call continues. + if ( + store.get(callEmbedAtom) || + isLivekitJsCallActive(store.get(livekitJsCallAtom)) || + isNativeCallActive(store.get(nativeCallAtom)) + ) { + return; + } + const activeController = controllerRef.current; + if (!activeController) return; + roomIdRef.current = room.roomId; + initialMediaRef.current = { + microphone: microphone ?? true, + camera: video ?? false, + sound: sound ?? true, + audioDeviceId, + videoDeviceId, + }; + store.set(livekitJsCallSoundAtom, sound ?? true); + store.set(livekitJsCallInitialMediaAppliedAtom, false); + void activeController + .connect({ + mx, + room, + discovery, + callIntent: video ? 'video' : 'audio', + dm, + ongoing: mx.matrixRTC.getRoomSession(room).memberships.length > 0, + }) + .catch(() => undefined); + }, + [mx, discovery, store] + ); + + const manager = useMemo(() => ({ start }), [start]); + + return ( + + {children} + + ); +} diff --git a/src/app/features/call/livekitJsController.test.ts b/src/app/features/call/livekitJsController.test.ts new file mode 100644 index 000000000..d14f4fb9a --- /dev/null +++ b/src/app/features/call/livekitJsController.test.ts @@ -0,0 +1,438 @@ +import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; + +import { + MatrixRTCSessionEvent, + type CallMembership, + type MatrixRTCSession, +} from '$types/matrix-sdk'; +import type { MatrixClient, Room as MatrixRoom } from '$types/matrix-sdk'; +import type { RoomOptions } from 'livekit-client'; +import { + createLivekitJsController, + type LivekitJsControllerDependencies, +} from './livekitJsController'; +import type { + LivekitMatrixKeyProvider, + LivekitMatrixKeyProviderState, +} from './livekitMatrixKeyProvider'; +import type { LocalCallIdentity } from './livekitCallIdentity'; +import { resetCallOwnerForTests } from '$state/callOwner'; + +const transport = { + type: 'livekit' as const, + livekit_service_url: 'https://sfu.example', +}; +const room = { + roomId: '!room:example.org', + loadMembersIfNeeded: () => Promise.resolve(), +} as unknown as MatrixRoom; +const membership = { + userId: '@alice:example.org', + deviceId: 'DEVICE', + rtcBackendIdentity: 'local-backend-identity', +} as CallMembership; + +const deferred = (): { + promise: Promise; + resolve: (value: T) => void; +} => { + let resolve!: (value: T) => void; + const promise = new Promise((promiseResolve) => { + resolve = promiseResolve; + }); + return { promise, resolve }; +}; + +type SessionHandler = (...args: unknown[]) => void; + +type TestSession = MatrixRTCSession & { + handlers: Map; +}; + +const makeSession = (order: string[] = []): TestSession => { + const handlers = new Map(); + const session = { + handlers, + memberships: [] as CallMembership[], + slotId: 'm.call#real-slot', + on: vi.fn<(...args: unknown[]) => void>().mockImplementation((event, handler) => { + handlers.set(event as MatrixRTCSessionEvent, handler as SessionHandler); + }), + removeListener: vi.fn<(...args: unknown[]) => void>().mockImplementation((event, handler) => { + if (handlers.get(event as MatrixRTCSessionEvent) === handler) { + handlers.delete(event as MatrixRTCSessionEvent); + } + }), + joinRTCSession: vi + .fn<(...args: unknown[]) => void>() + .mockImplementation(() => order.push('join')), + getOldestMembership: vi.fn<() => CallMembership | undefined>().mockReturnValue(undefined), + leaveRoomSession: vi.fn().mockImplementation(async () => { + order.push('leave'); + return true; + }), + } as unknown as TestSession; + return session; +}; + +const emitOwnMembership = (session: TestSession): void => { + session.memberships = [membership]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)?.([], [membership]); +}; + +type FakeProvider = { + state: LivekitMatrixKeyProviderState; + attach: Mock<(session: MatrixRTCSession, localIdentity: LocalCallIdentity) => void>; + detach: Mock<() => void>; + getKeyState: Mock<() => LivekitMatrixKeyProviderState>; + subscribe: Mock< + (listener: (state: Readonly) => void) => () => void + >; +}; + +const makeProvider = (state: Partial = {}): FakeProvider => { + const provider = { + state: { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + ...state, + }, + attach: vi.fn<(session: MatrixRTCSession, localIdentity: LocalCallIdentity) => void>(), + detach: vi.fn<() => void>(), + getKeyState: vi.fn<() => LivekitMatrixKeyProviderState>(), + subscribe: + vi.fn<(listener: (state: Readonly) => void) => () => void>(), + } as FakeProvider; + provider.getKeyState.mockImplementation(() => provider.state); + provider.subscribe.mockImplementation((listener) => { + listener(provider.state); + return () => undefined; + }); + return provider; +}; + +const makeClient = (session: MatrixRTCSession): MatrixClient => + ({ + getDeviceId: () => 'DEVICE', + getSafeUserId: () => '@alice:example.org', + matrixRTC: { getRoomSession: () => session }, + }) as unknown as MatrixClient; + +const makeDependencies = ( + session: TestSession, + order: string[], + provider = makeProvider() +): { + dependencies: LivekitJsControllerDependencies; + provider: FakeProvider; + roomOptions: { value?: RoomOptions }; + livekitRoom: { + connect: Mock<(url: string, token: string, options?: unknown) => Promise>; + disconnect: Mock<() => Promise>; + }; +} => { + const worker = { terminate: vi.fn<() => void>() } as unknown as Worker; + provider.attach.mockImplementation(() => order.push('attach')); + provider.detach.mockImplementation(() => order.push('detach')); + const livekitRoom = { + connect: vi + .fn<(url: string, token: string, options?: unknown) => Promise>() + .mockImplementation(async () => { + order.push('connect'); + }), + disconnect: vi.fn<() => Promise>().mockImplementation(async () => { + order.push('disconnect'); + }), + }; + const roomOptions: { value?: RoomOptions } = {}; + return { + provider, + roomOptions, + livekitRoom, + dependencies: { + createKeyProvider: () => provider as unknown as LivekitMatrixKeyProvider, + isE2EESupported: () => true, + createWorker: () => { + order.push('worker'); + return worker; + }, + createRoom: (options) => { + order.push('room'); + roomOptions.value = options; + return livekitRoom; + }, + getPreferredTransport: async () => { + order.push('transport'); + return transport; + }, + provisionToken: async () => { + order.push('provision'); + return { url: 'wss://livekit.example', jwt: 'jwt' }; + }, + }, + }; +}; + +const connectToActive = async ( + controller: ReturnType, + session: TestSession +): Promise => { + const connectPromise = controller.connect({ mx: makeClient(session), room }); + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await connectPromise; +}; + +describe('livekit JS controller', () => { + beforeEach(() => { + resetCallOwnerForTests(); + }); + + it('attaches E2EE before joining and provisions before connecting one Room', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, provider, roomOptions, livekitRoom } = makeDependencies(session, order); + const controller = createLivekitJsController(dependencies); + + await connectToActive(controller, session); + + expect(order).toEqual([ + 'attach', + 'transport', + 'join', + 'provision', + 'worker', + 'room', + 'connect', + ]); + expect(provider.attach).toHaveBeenCalledBefore(session.joinRTCSession as Mock); + expect(provider.attach).toHaveBeenCalledWith(session, { + userId: '@alice:example.org', + deviceId: 'DEVICE', + }); + expect(session.joinRTCSession).toHaveBeenCalledWith( + { + userId: '@alice:example.org', + deviceId: 'DEVICE', + memberId: '@alice:example.org:DEVICE', + }, + [{ ...transport, livekit_alias: room.roomId }], + undefined, + { + callIntent: 'audio', + membershipEventExpiryMs: 30 * 60 * 1000, + notificationType: 'notification', + manageMediaKeys: true, + } + ); + expect(roomOptions.value?.encryption).toEqual({ + keyProvider: provider, + worker: expect.anything(), + }); + expect(controller.getState().room).toBe(livekitRoom); + expect(controller.getState().lifecycle).toBe('active'); + }); + + it('refuses unsupported E2EE without joining or creating a Room', async () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + dependencies.isE2EESupported = () => false; + const controller = createLivekitJsController(dependencies); + + await controller.connect({ mx: makeClient(session), room }); + + expect(session.joinRTCSession).not.toHaveBeenCalled(); + expect(controller.getState()).toMatchObject({ + lifecycle: 'failed', + failure: 'e2ee-unsupported', + }); + }); + + it('refuses a provider import failure without connecting LiveKit', async () => { + const session = makeSession(); + const provider = makeProvider({ lastImportFailure: 'import-failed' }); + const { dependencies } = makeDependencies(session, [], provider); + const controller = createLivekitJsController(dependencies); + + await controller.connect({ mx: makeClient(session), room }); + + expect(session.joinRTCSession).not.toHaveBeenCalled(); + expect(controller.getState()).toMatchObject({ + lifecycle: 'failed', + failure: 'e2ee-import-failed', + }); + }); + + it('disconnects LiveKit before leaving MatrixRTC and cleans up idempotently', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, provider } = makeDependencies(session, order); + const controller = createLivekitJsController(dependencies); + await connectToActive(controller, session); + + await Promise.all([controller.disconnect(), controller.disconnect()]); + + expect(order.indexOf('disconnect')).toBeGreaterThan(-1); + expect(order.indexOf('detach')).toBeGreaterThan(order.indexOf('disconnect')); + expect(order.indexOf('disconnect')).toBeLessThan(order.indexOf('leave')); + expect(order.indexOf('detach')).toBeLessThan(order.indexOf('leave')); + expect(provider.detach).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('detaches once when setup fails after MatrixRTC join', async () => { + const session = makeSession(); + const { dependencies, provider } = makeDependencies(session, []); + dependencies.provisionToken = vi + .fn>() + .mockRejectedValue(new Error('provision failed')); + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ + mx: makeClient(session), + room, + }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await connectPromise; + + expect(provider.detach).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('failed'); + }); + + it('cancels during provisioning without creating a Room or becoming active', async () => { + const session = makeSession(); + const pendingProvision = deferred<{ url: string; jwt: string }>(); + const { dependencies, provider, roomOptions } = makeDependencies(session, []); + dependencies.provisionToken = vi + .fn>() + .mockImplementation(() => pendingProvision.promise); + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ + mx: makeClient(session), + room, + }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await vi.waitFor(() => expect(dependencies.provisionToken).toHaveBeenCalledOnce()); + const disconnectPromise = controller.disconnect(); + pendingProvision.resolve({ url: 'wss://livekit.example', jwt: 'jwt' }); + await Promise.all([connectPromise, disconnectPromise]); + + expect(roomOptions.value).toBeUndefined(); + expect(provider.detach).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('cancels during Room.connect and cleans up its stale room before leaving', async () => { + const session = makeSession(); + const pendingConnect = deferred(); + const { dependencies, provider } = makeDependencies(session, []); + const livekitRoom = { + connect: vi + .fn<(url: string, token: string, options?: unknown) => Promise>() + .mockImplementation(() => pendingConnect.promise), + disconnect: vi.fn<() => Promise>().mockResolvedValue(undefined), + }; + dependencies.createRoom = () => livekitRoom; + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ + mx: makeClient(session), + room, + }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await vi.waitFor(() => expect(livekitRoom.connect).toHaveBeenCalledOnce()); + const disconnectPromise = controller.disconnect(); + pendingConnect.resolve(); + await Promise.all([connectPromise, disconnectPromise]); + + expect(livekitRoom.disconnect).toHaveBeenCalledOnce(); + expect(provider.detach).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().lifecycle).toBe('idle'); + expect(controller.getState().lifecycle).not.toBe('active'); + }); + + it('rejects duplicate setup and cancellation leaves started MatrixRTC membership', async () => { + const session = makeSession(); + const { dependencies, provider } = makeDependencies(session, []); + const controller = createLivekitJsController(dependencies); + const firstConnect = controller.connect({ mx: makeClient(session), room }); + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + + await expect(controller.connect({ mx: makeClient(session), room })).rejects.toThrow( + 'already in use' + ); + await controller.disconnect(); + await firstConnect; + + expect(session.leaveRoomSession).toHaveBeenCalledWith(5000); + expect(provider.detach).toHaveBeenCalledOnce(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('exposes only lifecycle and state methods', () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + const controller = createLivekitJsController(dependencies); + + expect(Object.keys(controller).toSorted()).toEqual([ + 'connect', + 'disconnect', + 'getState', + 'subscribe', + ]); + }); + + it('disconnects the previous controller before a replacement connects', async () => { + const firstSession = makeSession(); + const first = makeDependencies(firstSession, []); + const firstController = createLivekitJsController(first.dependencies); + await connectToActive(firstController, firstSession); + + const replacementSession = makeSession(); + const replacement = makeDependencies(replacementSession, [], makeProvider({ ready: true })); + const replacementController = createLivekitJsController(replacement.dependencies); + + await firstController.disconnect(); + expect(firstController.getState().room).toBeUndefined(); + await connectToActive(replacementController, replacementSession); + + expect(replacementController.getState().lifecycle).toBe('active'); + }); +}); diff --git a/src/app/features/call/livekitJsController.ts b/src/app/features/call/livekitJsController.ts new file mode 100644 index 000000000..6e5bacf00 --- /dev/null +++ b/src/app/features/call/livekitJsController.ts @@ -0,0 +1,364 @@ +import { Room as LivekitRoom, type RoomOptions } from 'livekit-client'; +import type { AutoDiscoveryInfo } from '../../cs-api'; +import type { MatrixClient, MatrixRTCSession, Room as MatrixRoom } from '$types/matrix-sdk'; +import { + disconnectLivekitThenLeaveMatrixRTC, + joinAndProvisionMatrixRTC, + leaveMatrixRTCOnPageHide, +} from './matrixRtcCallLifecycle'; +import { + LivekitMatrixKeyProvider, + type LivekitMatrixKeyProviderState, + isLivekitE2EESupported, +} from './livekitMatrixKeyProvider'; +import { getPreferredLivekitTransport, provisionLivekitToken } from './livekitProvisioning'; +import { acquireCallOwner, type CallOwnerLease } from '$state/callOwner'; + +export type LivekitJsControllerLifecycle = + | 'idle' + | 'joining-matrix' + | 'provisioning' + | 'connecting-livekit' + | 'active' + | 'stopping' + | 'failed'; + +export type LivekitJsControllerFailure = 'e2ee-unsupported' | 'e2ee-import-failed' | 'setup-failed'; + +export type LivekitJsControllerState = { + lifecycle: LivekitJsControllerLifecycle; + failure: LivekitJsControllerFailure | null; + room?: LivekitRoom; + e2ee: Readonly; +}; + +type LivekitJsControllerStateListener = (state: Readonly) => void; + +type LivekitJsConnectOptions = { + mx: MatrixClient; + room: MatrixRoom; + discovery?: Pick; + callIntent?: 'audio' | 'video'; + dm?: boolean; + ongoing?: boolean; +}; + +type LivekitRoomLike = Pick; + +type LivekitJsControllerBase = { + connect: (options: LivekitJsConnectOptions) => Promise; + disconnect: () => Promise; + getState: () => Readonly; + subscribe: (listener: LivekitJsControllerStateListener) => () => void; +}; + +export type LivekitJsControllerDependencies = { + createRoom?: (options: RoomOptions) => LivekitRoomLike; + createWorker?: () => Worker; + createKeyProvider?: () => LivekitMatrixKeyProvider; + isE2EESupported?: () => boolean; + getPreferredTransport?: typeof getPreferredLivekitTransport; + provisionToken?: typeof provisionLivekitToken; +}; + +type ControllerRecord = { + session: MatrixRTCSession; + provider: LivekitMatrixKeyProvider; + worker?: Worker; + room?: LivekitRoomLike; + matrixJoinStarted: boolean; + providerDetached: boolean; + ownerLease: CallOwnerLease; + cancelled: boolean; + e2eeFailure: boolean; + cancelMembershipWait?: () => void; + removePageHideListener?: () => void; + removeKeyStateListener?: () => void; + cleanupPromise?: Promise; + resourcesReady: Promise; + resolveResources: () => void; +}; + +const initialE2EEState: LivekitMatrixKeyProviderState = { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, +}; + +const defaultCreateRoom = (options: RoomOptions): LivekitRoomLike => new LivekitRoom(options); + +const defaultCreateWorker = (): Worker => + new Worker(new URL('livekit-client/e2ee-worker', import.meta.url), { + type: 'module', + }); + +export function createLivekitJsController(dependencies: LivekitJsControllerDependencies = {}) { + const createRoom = dependencies.createRoom ?? defaultCreateRoom; + const createWorker = dependencies.createWorker ?? defaultCreateWorker; + const createKeyProvider = + dependencies.createKeyProvider ?? (() => new LivekitMatrixKeyProvider()); + const supportsE2EE = dependencies.isE2EESupported ?? isLivekitE2EESupported; + const getPreferredTransport = dependencies.getPreferredTransport ?? getPreferredLivekitTransport; + const provisionToken = dependencies.provisionToken ?? provisionLivekitToken; + + let state: LivekitJsControllerState = { + lifecycle: 'idle', + failure: null, + room: undefined, + e2ee: initialE2EEState, + }; + let record: ControllerRecord | undefined; + let operation: Promise | undefined; + const listeners = new Set(); + + const publish = (changes: Partial): void => { + state = { ...state, ...changes }; + const snapshot = { ...state, e2ee: { ...state.e2ee } }; + listeners.forEach((listener) => { + try { + listener(snapshot); + } catch { + // A state observer must not interrupt lifecycle cleanup. + } + }); + }; + + const cleanup = async ( + current: ControllerRecord, + result: 'idle' | 'failed', + failure: LivekitJsControllerFailure | null + ): Promise => { + if (current.cleanupPromise) { + await current.cleanupPromise; + return; + } + + current.cancelled = true; + current.cancelMembershipWait?.(); + current.cancelMembershipWait = undefined; + publish({ lifecycle: 'stopping' }); + const detachProvider = (): void => { + if (current.providerDetached) return; + current.providerDetached = true; + try { + current.provider.detach(); + } catch {} + }; + current.cleanupPromise = (async () => { + if (!current.room && !current.worker) await current.resourcesReady; + const stopRoom = async (): Promise => { + await current.room?.disconnect(); + }; + if (current.matrixJoinStarted) { + await disconnectLivekitThenLeaveMatrixRTC(async () => { + try { + await stopRoom(); + } finally { + detachProvider(); + } + }, current.session); + } else { + try { + await stopRoom(); + } catch { + // Cleanup continues even when a setup room rejects disconnect. + } finally { + detachProvider(); + } + } + current.worker?.terminate(); + current.removePageHideListener?.(); + current.removePageHideListener = undefined; + current.removeKeyStateListener?.(); + current.removeKeyStateListener = undefined; + current.ownerLease.release(); + if (record === current) record = undefined; + publish({ + lifecycle: result, + failure, + room: undefined, + ...(result === 'idle' ? { e2ee: initialE2EEState } : {}), + }); + })(); + await current.cleanupPromise; + }; + + const setup = async ( + current: ControllerRecord, + connectOptions: LivekitJsConnectOptions + ): Promise => { + let failure: LivekitJsControllerFailure | null = null; + try { + publish({ lifecycle: 'joining-matrix', failure: null }); + if (!supportsE2EE()) { + failure = 'e2ee-unsupported'; + } else { + current.provider.attach(current.session, { + userId: connectOptions.mx.getSafeUserId(), + deviceId: connectOptions.mx.getDeviceId(), + }); + if (current.e2eeFailure || current.provider.getKeyState().lastImportFailure) { + failure = 'e2ee-import-failed'; + } else { + const joined = await joinAndProvisionMatrixRTC({ + mx: connectOptions.mx, + room: connectOptions.room, + session: current.session, + discovery: connectOptions.discovery, + getPreferredTransport, + provisionToken, + callIntent: connectOptions.callIntent ?? 'audio', + ...(connectOptions.ongoing + ? {} + : { + notificationType: connectOptions.dm ? 'ring' : 'notification', + }), + manageMediaKeys: true, + isCancelled: () => current.cancelled, + onStage: (stage) => publish({ lifecycle: stage }), + onMembershipWait: (cancel) => { + current.cancelMembershipWait = cancel; + }, + onJoinStarted: () => { + current.matrixJoinStarted = true; + current.removePageHideListener = leaveMatrixRTCOnPageHide(current.session); + }, + }); + + if (current.e2eeFailure || current.provider.getKeyState().lastImportFailure) { + failure = 'e2ee-import-failed'; + } else if (!current.cancelled) { + publish({ lifecycle: 'connecting-livekit' }); + current.worker = createWorker(); + current.room = createRoom({ + // Both default to false in livekit-client. Without them a + // multi-party call receives every published layer at full quality + // and keeps sending video nobody is displaying. The native engine + // already enables both. + adaptiveStream: true, + dynacast: true, + encryption: { + keyProvider: current.provider, + worker: current.worker, + }, + }); + await current.room.connect(joined.provisioned.url, joined.provisioned.jwt); + if (current.e2eeFailure) failure = 'e2ee-import-failed'; + } + } + } + } catch { + failure = current.e2eeFailure ? 'e2ee-import-failed' : 'setup-failed'; + } finally { + current.resolveResources(); + } + + if (failure) { + await cleanup(current, 'failed', failure); + } else if (current.cancelled) { + await cleanup(current, 'idle', null); + } else { + publish({ + lifecycle: 'active', + failure: null, + room: current.room as LivekitRoom, + }); + } + }; + + const connect = (connectOptions: LivekitJsConnectOptions): Promise => { + if (state.lifecycle === 'failed' && !record && !operation) { + publish({ + lifecycle: 'idle', + failure: null, + room: undefined, + e2ee: initialE2EEState, + }); + } + if (record || operation || state.lifecycle !== 'idle') { + return Promise.reject(new Error('LiveKit JS call controller is already in use')); + } + + const ownerLease = acquireCallOwner('livekit-js', connectOptions.room.roomId); + if (!ownerLease) { + publish({ + lifecycle: 'failed', + failure: 'setup-failed', + room: undefined, + }); + return Promise.resolve(); + } + + let resolveResources!: () => void; + const resourcesReady = new Promise((resolve) => { + resolveResources = resolve; + }); + let session: MatrixRTCSession; + let provider: LivekitMatrixKeyProvider; + try { + session = connectOptions.mx.matrixRTC.getRoomSession(connectOptions.room); + provider = createKeyProvider(); + } catch { + ownerLease.release(); + publish({ + lifecycle: 'failed', + failure: 'setup-failed', + room: undefined, + }); + return Promise.resolve(); + } + const current: ControllerRecord = { + session, + provider, + matrixJoinStarted: false, + cancelled: false, + e2eeFailure: false, + providerDetached: false, + ownerLease, + resourcesReady, + resolveResources, + }; + record = current; + current.removeKeyStateListener = current.provider.subscribe((e2ee) => { + publish({ e2ee }); + if (e2ee.lastImportFailure) { + current.e2eeFailure = true; + current.cancelMembershipWait?.(); + if (current.room) void cleanup(current, 'failed', 'e2ee-import-failed'); + } + }); + + operation = setup(current, connectOptions).finally(() => { + operation = undefined; + }); + return operation; + }; + + const disconnect = async (): Promise => { + if (!record) { + if (state.lifecycle === 'failed') { + publish({ lifecycle: 'idle', failure: null }); + } + return; + } + await cleanup(record, 'idle', null); + await operation; + }; + + const controller: LivekitJsControllerBase = { + connect, + disconnect, + getState: (): Readonly => ({ + ...state, + e2ee: { ...state.e2ee }, + }), + subscribe: (listener: LivekitJsControllerStateListener): (() => void) => { + listeners.add(listener); + listener({ ...state, e2ee: { ...state.e2ee } }); + return () => listeners.delete(listener); + }, + }; + + return controller; +} diff --git a/src/app/features/call/livekitMatrixKeyProvider.test.ts b/src/app/features/call/livekitMatrixKeyProvider.test.ts new file mode 100644 index 000000000..084b7bf9f --- /dev/null +++ b/src/app/features/call/livekitMatrixKeyProvider.test.ts @@ -0,0 +1,298 @@ +import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; +import { MatrixRTCSessionEvent } from '$types/matrix-sdk'; +import { LivekitMatrixKeyProvider, isLivekitE2EESupported } from './livekitMatrixKeyProvider'; + +type EncryptionKeyChangedHandler = ( + key: Uint8Array, + keyIndex: number, + membershipParts: unknown, + rtcBackendIdentity: string +) => void; + +type SessionOn = (event: MatrixRTCSessionEvent, handler: EncryptionKeyChangedHandler) => void; + +type TestSession = { + on: Mock; + off: Mock; + reemitEncryptionKeys: Mock<() => void>; +}; + +const deferred = (): { + promise: Promise; + resolve: (value: T) => void; + reject: (reason?: unknown) => void; +} => { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; + }); + return { promise, resolve, reject }; +}; + +const session = (): TestSession => ({ + on: vi.fn(), + off: vi.fn(), + reemitEncryptionKeys: vi.fn<() => void>(), +}); + +const handlerFor = (testSession: TestSession): EncryptionKeyChangedHandler => + testSession.on.mock.calls.find( + ([event]) => event === MatrixRTCSessionEvent.EncryptionKeyChanged + )?.[1] as EncryptionKeyChangedHandler; + +describe('LivekitMatrixKeyProvider', () => { + const localIdentity = { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }; + const localParts = { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }; + const remoteParts = { userId: '@bob:example.org', deviceId: 'BOBDEVICE' }; + const importedKey = { imported: true } as unknown as CryptoKey; + const importKey = vi.fn().mockResolvedValue(importedKey); + + beforeEach(() => { + importKey.mockClear(); + vi.stubGlobal('crypto', { subtle: { importKey } }); + }); + + it('attaches, re-emits tracked keys, and forwards HKDF material with identity and index', async () => { + const testSession = session(); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + const matrixKey = new Uint8Array([1, 2, 3, 4]); + + expect(provider.getOptions()).toMatchObject({ + ratchetWindowSize: 10, + keyringSize: 256, + sharedKey: false, + }); + + provider.attach(testSession as never, localIdentity); + + expect(testSession.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function) + ); + expect(testSession.reemitEncryptionKeys).toHaveBeenCalledOnce(); + + handlerFor(testSession)(matrixKey, 7, localParts, 'hashed-member'); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledOnce()); + + expect(importKey).toHaveBeenCalledWith('raw', matrixKey, 'HKDF', false, [ + 'deriveBits', + 'deriveKey', + ]); + expect(onSetEncryptionKey).toHaveBeenCalledWith(importedKey, 'hashed-member', 7); + expect(provider.getKeyState()).toEqual({ + ready: true, + localOutboundIdentity: 'hashed-member', + keyIndex: 7, + lastImportFailure: null, + }); + }); + + it('removes the old session listener before attaching a new one', () => { + const oldSession = session(); + const newSession = session(); + const provider = new LivekitMatrixKeyProvider(); + + provider.attach(oldSession as never, localIdentity); + const oldHandler = handlerFor(oldSession); + provider.attach(newSession as never, localIdentity); + + expect(oldSession.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + oldHandler + ); + expect(newSession.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + oldHandler + ); + }); + + it('detaches the exact listener and does not expose raw key material', async () => { + const testSession = session(); + const provider = new LivekitMatrixKeyProvider(); + const rawKey = new Uint8Array([9, 8, 7]); + + provider.attach(testSession as never, localIdentity); + const handler = handlerFor(testSession); + handler(rawKey, 3, {}, 'member'); + await vi.waitFor(() => expect(provider.getKeys()).toHaveLength(1)); + provider.detach(); + + expect(testSession.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + handler + ); + expect(provider.getKeys()[0]?.key).toBe(importedKey); + expect(provider.getKeys()[0]?.key).not.toBe(rawKey); + }); + + it('does not forward an import that completes after detach', async () => { + const testSession = session(); + const pendingImport = deferred(); + importKey.mockImplementationOnce(() => pendingImport.promise); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + + provider.attach(testSession as never, localIdentity); + handlerFor(testSession)(new Uint8Array([1]), 1, {}, 'member'); + await vi.waitFor(() => expect(importKey).toHaveBeenCalledOnce()); + provider.detach(); + pendingImport.resolve(importedKey); + await Promise.resolve(); + await Promise.resolve(); + + expect(onSetEncryptionKey).not.toHaveBeenCalled(); + }); + + it('does not forward an import that completes after attaching a replacement session', async () => { + const oldSession = session(); + const newSession = session(); + const pendingImport = deferred(); + importKey.mockImplementationOnce(() => pendingImport.promise); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + + provider.attach(oldSession as never, localIdentity); + handlerFor(oldSession)(new Uint8Array([1]), 1, {}, 'old-member'); + await vi.waitFor(() => expect(importKey).toHaveBeenCalledOnce()); + provider.attach(newSession as never, localIdentity); + pendingImport.resolve(importedKey); + await Promise.resolve(); + await Promise.resolve(); + + expect(onSetEncryptionKey).not.toHaveBeenCalled(); + }); + + it('keeps the effective key index from regressing when imports complete out of order', async () => { + const testSession = session(); + const firstImport = deferred(); + const secondImport = deferred(); + importKey + .mockImplementationOnce(() => firstImport.promise) + .mockImplementationOnce(() => secondImport.promise); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + + provider.attach(testSession as never, localIdentity); + const handler = handlerFor(testSession); + handler(new Uint8Array([1]), 1, localParts, 'member'); + handler(new Uint8Array([2]), 2, localParts, 'member'); + await vi.waitFor(() => expect(importKey).toHaveBeenCalledTimes(2)); + + secondImport.resolve(importedKey); + await Promise.resolve(); + expect(onSetEncryptionKey).not.toHaveBeenCalled(); + firstImport.resolve(importedKey); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledTimes(2)); + + expect(onSetEncryptionKey.mock.calls[0]).toEqual([importedKey, 'member', 1]); + expect(onSetEncryptionKey.mock.calls[1]).toEqual([importedKey, 'member', 2]); + expect(provider.getKeyState().keyIndex).toBe(2); + }); + + it('tracks key-index regression independently for each participant', async () => { + const testSession = session(); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + + provider.attach(testSession as never, localIdentity); + const handler = handlerFor(testSession); + handler(new Uint8Array([1]), 10, remoteParts, 'remote-member'); + handler(new Uint8Array([2]), 1, localParts, 'local-member'); + + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledTimes(2)); + + expect(onSetEncryptionKey).toHaveBeenNthCalledWith(1, importedKey, 'remote-member', 10); + expect(onSetEncryptionKey).toHaveBeenNthCalledWith(2, importedKey, 'local-member', 1); + expect(provider.getKeyState()).toMatchObject({ + ready: true, + localOutboundIdentity: 'local-member', + keyIndex: 1, + lastImportFailure: null, + }); + }); + + it('records a safe failure when key import is rejected', async () => { + const testSession = session(); + importKey.mockRejectedValueOnce(new Error('raw key internals')); + const provider = new LivekitMatrixKeyProvider(); + + provider.attach(testSession as never, localIdentity); + handlerFor(testSession)(new Uint8Array([1]), 1, localParts, 'member'); + + await vi.waitFor(() => expect(provider.getKeyState().lastImportFailure).toBe('import-failed')); + expect(JSON.stringify(provider.getKeyState())).not.toContain('raw key internals'); + + handlerFor(testSession)(new Uint8Array([2]), 2, localParts, 'member'); + await vi.waitFor(() => expect(provider.getKeyState().ready).toBe(true)); + expect(provider.getKeyState().lastImportFailure).toBeNull(); + }); + + it('records a safe failure when WebCrypto import support is missing', async () => { + vi.stubGlobal('crypto', { subtle: {} }); + const testSession = session(); + const provider = new LivekitMatrixKeyProvider(); + + provider.attach(testSession as never, localIdentity); + handlerFor(testSession)(new Uint8Array([1]), 1, {}, 'member'); + + await vi.waitFor(() => + expect(provider.getKeyState().lastImportFailure).toBe('webcrypto-unavailable') + ); + }); +}); + +describe('isLivekitE2EESupported', () => { + beforeEach(() => { + vi.stubGlobal('crypto', { subtle: { importKey: vi.fn() } }); + }); + + it('fails closed when the current LiveKit API reports unsupported E2EE', () => { + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: undefined, + }); + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: undefined, + }); + + expect(isLivekitE2EESupported()).toBe(false); + }); + + it('reports support when LiveKit and WebCrypto are available', () => { + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: vi.fn<() => void>(), + }); + + expect(isLivekitE2EESupported()).toBe(true); + }); +}); diff --git a/src/app/features/call/livekitMatrixKeyProvider.ts b/src/app/features/call/livekitMatrixKeyProvider.ts new file mode 100644 index 000000000..8492c99d7 --- /dev/null +++ b/src/app/features/call/livekitMatrixKeyProvider.ts @@ -0,0 +1,209 @@ +import { BaseKeyProvider, isE2EESupported } from 'livekit-client'; +import { + MatrixRTCSessionEvent, + type CallMembershipIdentityParts, + type MatrixRTCSession, +} from '$types/matrix-sdk'; +import type { LocalCallIdentity } from './livekitCallIdentity'; + +export const isLivekitE2EESupported = (): boolean => { + const subtle = globalThis.crypto?.subtle; + return typeof subtle?.importKey === 'function' && isE2EESupported(); +}; + +export type LivekitMatrixKeyImportFailure = 'webcrypto-unavailable' | 'import-failed'; + +export type LivekitMatrixKeyProviderState = { + ready: boolean; + localOutboundIdentity: string | null; + keyIndex: number | null; + lastImportFailure: LivekitMatrixKeyImportFailure | null; +}; + +export type LivekitMatrixKeyProviderStateListener = ( + state: Readonly +) => void; + +type KeyImportResult = + | { + keyMaterial: CryptoKey; + rtcBackendIdentity: string; + encryptionKeyIndex: number; + isLocal: boolean; + } + | { failure: LivekitMatrixKeyImportFailure }; + +export class LivekitMatrixKeyProvider extends BaseKeyProvider { + private rtcSession?: MatrixRTCSession; + private localIdentity: LocalCallIdentity | null = null; + private attachmentGeneration = 0; + private nextImportSequence = 0; + private nextUpdateSequence = 0; + private readonly pendingUpdates = new Map(); + private readonly acceptedKeyIndices = new Map(); + private readonly stateListeners = new Set(); + private localOutboundIdentity: string | null = null; + private state: LivekitMatrixKeyProviderState = { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + }; + + public constructor() { + super({ + ratchetWindowSize: 10, + keyringSize: 256, + sharedKey: false, + }); + } + + public attach(session: MatrixRTCSession, localIdentity: LocalCallIdentity): void { + this.detach(); + this.rtcSession = session; + this.localIdentity = localIdentity; + session.on(MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged); + session.reemitEncryptionKeys(); + } + + public detach(): void { + this.attachmentGeneration += 1; + this.nextImportSequence = 0; + this.nextUpdateSequence = 0; + this.pendingUpdates.clear(); + this.acceptedKeyIndices.clear(); + this.localIdentity = null; + this.localOutboundIdentity = null; + this.updateState({ + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + }); + + if (!this.rtcSession) return; + + this.rtcSession.off(MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged); + this.rtcSession = undefined; + } + + public getKeyState(): Readonly { + return { ...this.state }; + } + + public subscribe(listener: LivekitMatrixKeyProviderStateListener): () => void { + this.stateListeners.add(listener); + listener(this.getKeyState()); + return () => { + this.stateListeners.delete(listener); + }; + } + + private readonly onEncryptionKeyChanged = ( + encryptionKey: Uint8Array, + encryptionKeyIndex: number, + membershipParts: CallMembershipIdentityParts, + rtcBackendIdentity: string + ): void => { + const generation = this.attachmentGeneration; + const sequence = this.nextImportSequence++; + const isLocal = + membershipParts.userId === this.localIdentity?.userId && + membershipParts.deviceId === this.localIdentity?.deviceId; + const subtle = globalThis.crypto?.subtle; + if (!subtle || typeof subtle.importKey !== 'function') { + this.enqueueUpdate(generation, sequence, { + failure: 'webcrypto-unavailable', + }); + return; + } + + let importPromise: Promise; + try { + importPromise = subtle.importKey('raw', encryptionKey, 'HKDF', false, [ + 'deriveBits', + 'deriveKey', + ]); + } catch { + this.enqueueUpdate(generation, sequence, { failure: 'import-failed' }); + return; + } + + void importPromise.then( + (keyMaterial) => { + this.enqueueUpdate(generation, sequence, { + keyMaterial, + rtcBackendIdentity, + encryptionKeyIndex, + isLocal, + }); + }, + () => { + this.enqueueUpdate(generation, sequence, { failure: 'import-failed' }); + } + ); + }; + + private enqueueUpdate(generation: number, sequence: number, result: KeyImportResult): void { + if (generation !== this.attachmentGeneration) return; + this.pendingUpdates.set(sequence, result); + + while (this.pendingUpdates.has(this.nextUpdateSequence)) { + const update = this.pendingUpdates.get(this.nextUpdateSequence); + this.pendingUpdates.delete(this.nextUpdateSequence); + this.nextUpdateSequence += 1; + if (!update) continue; + + if ('failure' in update) { + this.updateState({ lastImportFailure: update.failure }); + continue; + } + + const lastParticipantKeyIndex = this.acceptedKeyIndices.get(update.rtcBackendIdentity) ?? -1; + if (update.encryptionKeyIndex < lastParticipantKeyIndex) continue; + + try { + this.onSetEncryptionKey( + update.keyMaterial, + update.rtcBackendIdentity, + update.encryptionKeyIndex + ); + } catch { + this.updateState({ lastImportFailure: 'import-failed' }); + continue; + } + + this.acceptedKeyIndices.set( + update.rtcBackendIdentity, + Math.max( + this.acceptedKeyIndices.get(update.rtcBackendIdentity) ?? -1, + update.encryptionKeyIndex + ) + ); + + // Only the local key clears the failure flag: a remote participant's key + // succeeding says nothing about whether our own outbound key imported. + if (update.isLocal) { + this.localOutboundIdentity = update.rtcBackendIdentity; + this.updateState({ + ready: true, + localOutboundIdentity: update.rtcBackendIdentity, + keyIndex: update.encryptionKeyIndex, + lastImportFailure: null, + }); + } + } + } + + private updateState(changes: Partial): void { + this.state = { ...this.state, ...changes }; + const state = this.getKeyState(); + this.stateListeners.forEach((listener) => { + try { + listener(state); + } catch { + // A state observer must not interrupt key updates. + } + }); + } +} diff --git a/src/app/features/call/livekitMobileBridge.ts b/src/app/features/call/livekitMobileBridge.ts new file mode 100644 index 000000000..7deb0f510 --- /dev/null +++ b/src/app/features/call/livekitMobileBridge.ts @@ -0,0 +1,329 @@ +import { addPluginListener, invoke, type PluginListener } from '@tauri-apps/api/core'; +import { listen, type UnlistenFn } from '@tauri-apps/api/event'; + +export type NativeCallConnectionState = + | 'idle' + | 'connecting' + | 'connected' + | 'reconnecting' + | 'failed'; + +export type NativeCallFailureCode = + | 'invalid_request' + | 'busy' + | 'permission_denied' + | 'connect_failed' + | 'media_failed' + | 'disconnected' + | 'cancelled' + | 'unavailable' + | 'unexpected'; + +export type NativeCallCapabilities = { + supported: boolean; + microphone: boolean; + backgroundAudio: boolean; + nativeRoom: boolean; + camera: boolean; + nativeVideoOverlay: boolean; + callKit: boolean; +}; + +export type NativeCallEncryptionKeyPayload = { + identity: string; + keyIndex: number; + key: string; +}; + +export type NativeCallRemoteCamera = { + sid: string; + muted: boolean; + subscribed: boolean; +}; + +export type NativeCallRemoteParticipant = { + identity: string; + camera?: NativeCallRemoteCamera; + screenShare?: NativeCallRemoteCamera; + connectionQuality?: string; +}; + +export type NativeCallSnapshot = { + revision: number; + callId: string | null; + connectionState: NativeCallConnectionState; + microphoneEnabled: boolean; + cameraEnabled: boolean; + participantCount: number; + // Present on current native builds; optional so older payloads and test + // fixtures without the field remain valid. + remoteParticipants?: NativeCallRemoteParticipant[]; + lastError?: { code: NativeCallFailureCode; message: string }; +}; + +export type ConnectNativeCallRequest = { + callId: string; + url: string; + token: string; + microphoneEnabled: boolean; + encryptionKeys?: NativeCallEncryptionKeyPayload[]; +}; + +export type SetNativeCallEncryptionKeyRequest = { + callId: string; + identity: string; + keyIndex: number; + key: string; +}; + +/** + * Pins the single native-rendered remote camera view over a DOM tile. `x`, + * `y`, `width`, `height` are the tile's viewport-relative CSS rect; the + * native side maps them into view coordinates. A repeated call with the same + * track repositions the view; a new track rebinds it. + */ +export type SetNativeCallRemoteVideoOverlayRequest = { + callId: string; + participantIdentity: string; + trackId: string; + x: number; + y: number; + width: number; + height: number; + devicePixelRatio: number; +}; + +export type ClearNativeCallRemoteVideoOverlayRequest = { + callId: string; +}; + +export type SetNativeCallLocalVideoOverlayRequest = { + callId: string; + x: number; + y: number; + width: number; + height: number; + devicePixelRatio: number; +}; + +export type ClearNativeCallLocalVideoOverlayRequest = { + callId: string; +}; + +export type ReportSystemIncomingCallRequest = { + uuid: string; + callerName: string; +}; + +export type StartSystemCallRequest = { + callId: string; + uuid: string; + callerName: string; +}; + +export type AnswerSystemCallRequest = { + callId: string; + uuid: string; +}; + +export type EndSystemCallRequest = { + callId: string; + remoteEnded?: boolean; +}; + +export type SetSystemCallMutedRequest = { + callId: string; + muted: boolean; +}; + +export type GetAudioRoutesRequest = { + callId: string; +}; + +export type SetAudioRouteRequest = { + callId: string; + routeId: string; +}; + +export type UpdateCallDisplayRequest = { + callId: string; + callerName: string; + hasVideo?: boolean; +}; + +export type ReportAnsweredElsewhereRequest = { + callId: string; +}; + +export type ReportDeclinedElsewhereRequest = { + callId: string; +}; + +export type ReportUnansweredRequest = { + callId: string; +}; + +export type DeclineSystemCallRequest = { + callId: string; + reason: string; +}; + +/** `name` is display-ready; `type` is the bounded platform vocabulary. */ +export type NativeCallAudioRoute = { + id: string; + name: string; + type: string; + current: boolean; +}; + +export type GetAudioRoutesResponse = { + routes: NativeCallAudioRoute[]; + receiver: NativeCallSnapshot; +}; + +export type SystemCallActionKind = 'answer' | 'end' | 'mute'; + +export type SystemCallAction = { + action: SystemCallActionKind; + uuid: string; + muted?: boolean; +}; + +const NATIVE_CALL_EVENT = 'plugin:livekit-mobile://native-call-event'; + +export const getNativeCallCapabilities = (): Promise => + invoke('plugin:livekit-mobile|get_native_call_capabilities'); + +export const connectNativeCall = (request: ConnectNativeCallRequest): Promise => + invoke('plugin:livekit-mobile|connect_native_call', { payload: request }); + +export const disconnectNativeCall = (request: { callId: string }): Promise => + invoke('plugin:livekit-mobile|disconnect_native_call', { payload: request }); + +export const setNativeCallMicrophoneEnabled = (request: { + callId: string; + enabled: boolean; +}): Promise => + invoke('plugin:livekit-mobile|set_native_call_microphone_enabled', { + payload: request, + }); + +export const setNativeCallCameraEnabled = (request: { + callId: string; + enabled: boolean; +}): Promise => + invoke('plugin:livekit-mobile|set_native_call_camera_enabled', { + payload: request, + }); + +export const setNativeCallPiPEnabled = (request: { + callId: string; + enabled: boolean; +}): Promise => + invoke('plugin:livekit-mobile|set_native_call_pip_enabled', { + payload: request, + }); + +export const switchNativeCallCamera = (request: { callId: string }): Promise => + invoke('plugin:livekit-mobile|switch_native_call_camera', { + payload: request, + }); + +export const setNativeCallEncryptionKey = ( + request: SetNativeCallEncryptionKeyRequest +): Promise => + invoke('plugin:livekit-mobile|set_native_call_encryption_key', { + payload: request, + }); + +export const setNativeCallRemoteVideoOverlay = ( + request: SetNativeCallRemoteVideoOverlayRequest +): Promise => + invoke('plugin:livekit-mobile|set_native_call_remote_video_overlay', { + payload: request, + }); + +export const clearNativeCallRemoteVideoOverlay = ( + request: ClearNativeCallRemoteVideoOverlayRequest +): Promise => + invoke('plugin:livekit-mobile|clear_native_call_remote_video_overlay', { + payload: request, + }); + +export const setNativeCallLocalVideoOverlay = ( + request: SetNativeCallLocalVideoOverlayRequest +): Promise => + invoke('plugin:livekit-mobile|set_native_call_local_video_overlay', { + payload: request, + }); + +export const clearNativeCallLocalVideoOverlay = ( + request: ClearNativeCallLocalVideoOverlayRequest +): Promise => + invoke('plugin:livekit-mobile|clear_native_call_local_video_overlay', { + payload: request, + }); + +export const getNativeCallState = (): Promise => + invoke('plugin:livekit-mobile|get_native_call_state'); + +export const listenNativeCallSnapshot = ( + handler: (snapshot: NativeCallSnapshot) => void +): Promise => + listen(NATIVE_CALL_EVENT, ({ payload }) => handler(payload)); + +export const reportSystemIncomingCall = (request: ReportSystemIncomingCallRequest): Promise => + invoke('plugin:livekit-mobile|report_system_incoming_call', { payload: request }); + +export const startSystemCall = (request: StartSystemCallRequest): Promise => + invoke('plugin:livekit-mobile|start_system_call', { payload: request }); + +export const answerSystemCall = (request: AnswerSystemCallRequest): Promise => + invoke('plugin:livekit-mobile|answer_system_call', { payload: request }); + +export const endSystemCall = (request: EndSystemCallRequest): Promise => + invoke('plugin:livekit-mobile|end_system_call', { payload: request }); + +export const setSystemCallMuted = (request: SetSystemCallMutedRequest): Promise => + invoke('plugin:livekit-mobile|set_system_call_muted', { payload: request }); + +export const drainPendingSystemCallActions = (): Promise => + invoke('plugin:livekit-mobile|drain_pending_system_call_actions'); + +export const fulfillAnswerCall = (uuid: string): Promise => + invoke('plugin:livekit-mobile|fulfill_answer_call', { payload: { uuid } }); + +export const fulfillEndCall = (uuid: string): Promise => + invoke('plugin:livekit-mobile|fulfill_end_call', { payload: { uuid } }); + +export const reportSystemCallConnected = (uuid: string): Promise => + invoke('plugin:livekit-mobile|report_system_call_connected', { payload: { uuid } }); + +export const getAudioRoutes = (request: GetAudioRoutesRequest): Promise => + invoke('plugin:livekit-mobile|get_audio_routes', { payload: request }); + +export const setAudioRoute = (request: SetAudioRouteRequest): Promise => + invoke('plugin:livekit-mobile|set_audio_route', { payload: request }); + +export const updateCallDisplay = (request: UpdateCallDisplayRequest): Promise => + invoke('plugin:livekit-mobile|update_call_display', { payload: request }); + +export const reportSystemCallAnsweredElsewhere = ( + request: ReportAnsweredElsewhereRequest +): Promise => + invoke('plugin:livekit-mobile|report_system_call_answered_elsewhere', { payload: request }); + +export const reportSystemCallDeclinedElsewhere = ( + request: ReportDeclinedElsewhereRequest +): Promise => + invoke('plugin:livekit-mobile|report_system_call_declined_elsewhere', { payload: request }); + +export const reportSystemCallUnanswered = (request: ReportUnansweredRequest): Promise => + invoke('plugin:livekit-mobile|report_system_call_unanswered', { payload: request }); + +export const declineSystemCall = (request: DeclineSystemCallRequest): Promise => + invoke('plugin:livekit-mobile|decline_system_call', { payload: request }); + +export const onSystemCallAction = ( + handler: (action: SystemCallAction) => void +): Promise => addPluginListener('livekit-mobile', 'callkit_event', handler); diff --git a/src/app/features/call/livekitProvisioning.test.ts b/src/app/features/call/livekitProvisioning.test.ts new file mode 100644 index 000000000..186514f5c --- /dev/null +++ b/src/app/features/call/livekitProvisioning.test.ts @@ -0,0 +1,159 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { IOpenIDToken, MatrixClient } from '$types/matrix-sdk'; + +const fetchMock = vi.hoisted(() => vi.fn()); + +vi.mock('$utils/fetch', () => ({ fetch: fetchMock })); + +import { + getPreferredLivekitTransport, + provisionLivekitToken, + useStickyMemberships, +} from './livekitProvisioning'; + +const openidToken: IOpenIDToken = { + access_token: 'openid-secret', + token_type: 'Bearer', + matrix_server_name: 'example.org', + expires_in: 3600, +}; + +const response = (status: number, body: unknown): Response => + new Response(JSON.stringify(body), { status }); + +type TestClient = Pick; + +const client = (overrides: Partial = {}): TestClient => ({ + getOpenIdToken: vi.fn().mockResolvedValue(openidToken), + _unstable_getRTCTransports: vi + .fn() + .mockResolvedValue([]), + ...overrides, +}); + +describe('getPreferredLivekitTransport', () => { + it('prefers the SDK LiveKit transport over discovery', async () => { + const mx = client({ + _unstable_getRTCTransports: vi + .fn() + .mockResolvedValue([{ type: 'livekit', livekit_service_url: 'https://sdk.example' }]), + }); + + await expect( + getPreferredLivekitTransport(mx, { + 'org.matrix.msc4143.rtc_foci': [ + { type: 'livekit', livekit_service_url: 'https://discovery.example' }, + ], + }) + ).resolves.toEqual({ + type: 'livekit', + livekit_service_url: 'https://sdk.example', + }); + }); + + it('falls back to discovery when SDK transport discovery fails', async () => { + const mx = client({ + _unstable_getRTCTransports: vi + .fn() + .mockRejectedValue(new Error('unsupported')), + }); + + await expect( + getPreferredLivekitTransport(mx, { + 'org.matrix.msc4143.rtc_foci': [ + { type: 'livekit', livekit_service_url: 'https://discovery.example' }, + ], + }) + ).resolves.toEqual({ + type: 'livekit', + livekit_service_url: 'https://discovery.example', + }); + }); +}); + +describe('provisionLivekitToken', () => { + beforeEach(() => { + fetchMock.mockReset(); + }); + + const options = { + mx: client(), + roomId: '!room:example.org', + slotId: 'm.call#real-slot', + deviceId: 'DEVICE', + serviceUrl: 'https://sfu.example///', + memberId: 'member-id', + userId: '@alice:example.org', + }; + + it('provisions through the endpoint that matches the advertised membership format', async () => { + fetchMock.mockResolvedValueOnce( + response(200, { url: 'wss://livekit.example', jwt: 'jwt-secret' }) + ); + + await expect(provisionLivekitToken(options)).resolves.toEqual({ + url: 'wss://livekit.example', + jwt: 'jwt-secret', + }); + + expect(fetchMock).toHaveBeenCalledOnce(); + const [url, request] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe( + useStickyMemberships ? 'https://sfu.example/get_token' : 'https://sfu.example/sfu/get' + ); + expect(JSON.parse(request.body as string)).toEqual( + useStickyMemberships + ? { + room_id: '!room:example.org', + slot_id: 'm.call#real-slot', + openid_token: openidToken, + member: { + id: 'member-id', + claimed_user_id: '@alice:example.org', + claimed_device_id: 'DEVICE', + }, + } + : { + room: '!room:example.org', + openid_token: openidToken, + device_id: 'DEVICE', + } + ); + }); + + it('never tries the other endpoint, whose identity convention would not match', async () => { + fetchMock.mockResolvedValueOnce(response(404, { error: 'not found' })); + + await expect(provisionLivekitToken(options)).rejects.toThrow( + 'LiveKit token provisioning failed' + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('rejects an invalid response without retrying', async () => { + fetchMock.mockResolvedValueOnce(response(200, { url: 'wss://livekit.example' })); + + await expect(provisionLivekitToken(options)).rejects.toThrow( + 'LiveKit token provisioning failed' + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('does not retry after a server error', async () => { + fetchMock.mockResolvedValueOnce(response(500, { error: 'boom' })); + + await expect(provisionLivekitToken(options)).rejects.toThrow( + 'LiveKit token provisioning failed' + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('does not expose token values in errors', async () => { + fetchMock.mockRejectedValue(new Error('request failed: jwt-secret openid-secret')); + const provisioning = provisionLivekitToken(options); + + await expect(provisioning).rejects.toThrow('LiveKit token provisioning failed'); + await expect(provisioning).rejects.not.toThrow('openid-secret'); + await expect(provisioning).rejects.not.toThrow('jwt-secret'); + }); +}); diff --git a/src/app/features/call/livekitProvisioning.ts b/src/app/features/call/livekitProvisioning.ts new file mode 100644 index 000000000..23536b2ed --- /dev/null +++ b/src/app/features/call/livekitProvisioning.ts @@ -0,0 +1,147 @@ +import type { AutoDiscoveryInfo } from '../../cs-api'; +import { getLivekitTransports } from '../../cs-api'; +import { fetch as appFetch } from '$utils/fetch'; +import { trimTrailingSlash } from '$utils/common'; +import type { + IOpenIDToken, + LivekitTransportConfig, + MatrixClient, + Transport, +} from '$types/matrix-sdk'; + +export type LivekitProvisioningOptions = { + mx: Pick; + roomId: string; + slotId: string; + deviceId: string; + serviceUrl: string; + memberId?: string; + userId?: string; +}; + +export type LivekitProvisioningResult = { + url: string; + jwt: string; +}; + +/** + * Membership format for MatrixRTC. It also picks the JWT endpoint below, + * because the two cannot be chosen independently: `/get_token` grants a LiveKit + * identity hashed from the sticky membership, `/sfu/get` grants + * `${userId}:${deviceId}`, and LiveKit binds E2EE keys strictly to the identity + * the JWT carries. Sticky memberships need delayed events, which this + * homeserver does not support, so both stay on the legacy side. + */ +export const useStickyMemberships = false; + +export const isLivekitTransportConfig = ( + transport: Transport +): transport is LivekitTransportConfig => + transport.type === 'livekit' && typeof transport.livekit_service_url === 'string'; + +const isProvisioningResult = (value: unknown): value is LivekitProvisioningResult => + typeof value === 'object' && + value !== null && + 'url' in value && + typeof value.url === 'string' && + value.url.length > 0 && + 'jwt' in value && + typeof value.jwt === 'string' && + value.jwt.length > 0; + +export const getPreferredLivekitTransport = async ( + mx: Pick, + discovery?: Pick +): Promise => { + const transports = await mx['_unstable_getRTCTransports']().catch(() => undefined); + const livekitTransport = transports?.find(isLivekitTransportConfig); + if (livekitTransport) return livekitTransport; + + return getLivekitTransports(discovery)[0]; +}; + +type ModernProvisioningRequest = { + room_id: string; + slot_id: string; + openid_token: IOpenIDToken; + member?: { + id: string; + claimed_user_id: string; + claimed_device_id: string; + }; +}; + +type LegacyProvisioningRequest = { + room: string; + openid_token: IOpenIDToken; + device_id: string; +}; + +const requestLivekitToken = async ( + endpoint: string, + body: ModernProvisioningRequest | LegacyProvisioningRequest +): Promise => { + const response = await appFetch(endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + + if (response.status < 200 || response.status >= 300) { + throw new Error(`LiveKit provisioning request failed with status ${response.status}`); + } + + const data = (await response.json()) as unknown; + + if (!isProvisioningResult(data)) { + throw new Error('LiveKit provisioning response was invalid'); + } + + return { url: data.url, jwt: data.jwt }; +}; + +export const provisionLivekitToken = async ({ + mx, + roomId, + slotId, + deviceId, + serviceUrl, + memberId, + userId, +}: LivekitProvisioningOptions): Promise => { + let openidToken: IOpenIDToken; + try { + openidToken = await mx.getOpenIdToken(); + } catch { + throw new Error('Unable to obtain an OpenID token for LiveKit provisioning'); + } + + const endpoint = trimTrailingSlash(serviceUrl); + + try { + if (useStickyMemberships) { + const modernRequest: ModernProvisioningRequest = { + room_id: roomId, + slot_id: slotId, + openid_token: openidToken, + }; + if (memberId && userId && deviceId) { + modernRequest.member = { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }; + } + return await requestLivekitToken(`${endpoint}/get_token`, modernRequest); + } + + const legacyRequest: LegacyProvisioningRequest = { + room: roomId, + openid_token: openidToken, + device_id: deviceId, + }; + return await requestLivekitToken(`${endpoint}/sfu/get`, legacyRequest); + } catch { + throw new Error('LiveKit token provisioning failed'); + } +}; diff --git a/src/app/features/call/matrixRtcCallLifecycle.test.ts b/src/app/features/call/matrixRtcCallLifecycle.test.ts new file mode 100644 index 000000000..2132312f1 --- /dev/null +++ b/src/app/features/call/matrixRtcCallLifecycle.test.ts @@ -0,0 +1,359 @@ +import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; +import type { + MatrixClient, + MatrixRTCSession, + CallMembership, + JoinSessionConfig, + Room, +} from '$types/matrix-sdk'; +import { MatrixRTCSessionEvent } from '$types/matrix-sdk'; +import { joinAndProvisionMatrixRTC, leaveMatrixRTCOnPageHide } from './matrixRtcCallLifecycle'; +import type { + LivekitProvisioningResult, + getPreferredLivekitTransport, + provisionLivekitToken, +} from './livekitProvisioning'; + +type SessionHandler = (...args: unknown[]) => void; + +type TestSession = MatrixRTCSession & { + handlers: Map; +}; + +const makeSession = (): TestSession => { + const handlers = new Map(); + const session = { + handlers, + memberships: [] as CallMembership[], + slotId: 'm.call#slot', + on: vi + .fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>() + .mockImplementation((event, handler) => { + handlers.set(event, handler); + }), + removeListener: vi + .fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>() + .mockImplementation((event, handler) => { + if (handlers.get(event) === handler) { + handlers.delete(event); + } + }), + joinRTCSession: vi.fn<(identity: unknown, transports: unknown[], ..._: unknown[]) => void>(), + getOldestMembership: vi.fn<() => CallMembership | undefined>().mockReturnValue(undefined), + leaveRoomSession: vi.fn().mockResolvedValue(true), + } as unknown as TestSession; + return session; +}; + +const emitMembershipManagerError = (session: TestSession): void => { + session.handlers.get(MatrixRTCSessionEvent.MembershipManagerError)?.(); +}; + +const makeClient = (overrides: Partial = {}): MatrixClient => + ({ + getDeviceId: () => 'ALICEDEVICE', + getSafeUserId: () => '@alice:example.org', + getStateEvent: vi + .fn<() => Promise>() + .mockResolvedValue(undefined) as unknown as MatrixClient['getStateEvent'], + ...overrides, + }) as unknown as MatrixClient; + +const provisioned: LivekitProvisioningResult = { url: 'wss://livekit.example', jwt: 'jwt' }; + +const makeTransport = () => ({ + type: 'livekit' as const, + livekit_service_url: 'https://sfu.example', +}); + +describe('joinAndProvisionMatrixRTC', () => { + beforeEach(() => { + // vi.useFakeTimers() would interfere with setInterval; use real timers. + }); + + const callOpts = (overrides: Record = {}) => ({ + mx: makeClient(), + room: { + roomId: '!room:example.org', + loadMembersIfNeeded: () => Promise.resolve(), + } as unknown as Room, + session: makeSession(), + callIntent: 'audio' as const, + getPreferredTransport: vi + .fn() + .mockResolvedValue(makeTransport()), + provisionToken: vi.fn().mockResolvedValue(provisioned), + ...overrides, + }); + + it('resolves when MembershipsChanged fires with own membership', async () => { + const session = makeSession(); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + // wait for listener registration in the microtask/event loop + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + + // fire the membership event + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + + const result = await promise; + expect(result.provisioned).toEqual(provisioned); + expect(opts.getPreferredTransport).toHaveBeenCalledOnce(); + }); + + it('advertises the transport with a livekit alias and a bounded membership expiry', async () => { + const session = makeSession(); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => expect(session.joinRTCSession).toHaveBeenCalled()); + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + + const [, transports, , joinConfig] = (session.joinRTCSession as Mock).mock.calls[0] as [ + unknown, + unknown[], + unknown, + JoinSessionConfig, + ]; + expect(transports).toEqual([{ ...makeTransport(), livekit_alias: '!room:example.org' }]); + expect(joinConfig.membershipEventExpiryMs).toBe(30 * 60 * 1000); + expect(joinConfig.unstableSendStickyEvents).toBeUndefined(); + }); + + it('provisions against the oldest membership transport, not our own preference', async () => { + const session = makeSession(); + const oldestTransport = { type: 'livekit' as const, livekit_service_url: 'https://oldest.sfu' }; + const oldest = { + userId: '@bob:example.org', + deviceId: 'BOBDEVICE', + getTransport: () => oldestTransport, + } as unknown as CallMembership; + (session.getOldestMembership as Mock<() => CallMembership | undefined>).mockReturnValue(oldest); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => expect(session.on).toHaveBeenCalled()); + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + + expect(opts.provisionToken).toHaveBeenCalledWith( + expect.objectContaining({ serviceUrl: 'https://oldest.sfu' }) + ); + }); + + it('falls back to the preferred transport when the oldest membership has none', async () => { + const session = makeSession(); + const oldest = { + userId: '@bob:example.org', + deviceId: 'BOBDEVICE', + getTransport: () => undefined, + } as unknown as CallMembership; + (session.getOldestMembership as Mock<() => CallMembership | undefined>).mockReturnValue(oldest); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => expect(session.on).toHaveBeenCalled()); + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + + expect(opts.provisionToken).toHaveBeenCalledWith( + expect.objectContaining({ serviceUrl: 'https://sfu.example' }) + ); + }); + + it('rejects on MembershipManagerError', async () => { + const session = makeSession(); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipManagerError, + expect.any(Function) + ) + ); + + emitMembershipManagerError(session); + + await expect(promise).rejects.toThrow('MatrixRTC membership publication failed'); + }); + + it('rejects on timeout (30s) when no membership event fires', async () => { + vi.useFakeTimers(); + try { + const session = makeSession(); + const mx = makeClient({ + getStateEvent: vi.fn().mockResolvedValue(undefined as never), + }); + const opts = callOpts({ session, mx }); + const promise = joinAndProvisionMatrixRTC(opts); + // Handled by the assertion below; attached now so the rejection that + // lands during the timer flush is never seen as unhandled. + promise.catch(() => {}); + + await vi.runAllTimersAsync(); + await expect(promise).rejects.toThrow('MatrixRTC membership publication timed out'); + } finally { + vi.useRealTimers(); + } + }); + + it('resolves via server-side fallback when membership is filtered locally', async () => { + const session = makeSession(); + const mx = makeClient({ + getStateEvent: vi + .fn() + // first poll: no event yet + .mockResolvedValueOnce(undefined as never) + // second poll: membership found on server + .mockResolvedValueOnce({}), + }); + const opts = callOpts({ session, mx }); + const promise = joinAndProvisionMatrixRTC(opts); + + // memberships listener is registered, but never fires own membership + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + + const result = await promise; + expect(result.provisioned).toEqual(provisioned); + expect(opts.getPreferredTransport).toHaveBeenCalledOnce(); + // fallback polled twice + expect(mx.getStateEvent).toHaveBeenCalledTimes(2); + }); + + it('stops fallback polling on cancel', async () => { + vi.useFakeTimers(); + try { + const session = makeSession(); + const mx = makeClient({ + getStateEvent: vi.fn().mockResolvedValue(undefined as never), + }); + let cancelMembership!: (() => void) | undefined; + const opts = callOpts({ + session, + mx, + onMembershipWait: (cancel: (() => void) | undefined) => { + cancelMembership = cancel; + }, + }); + const promise = joinAndProvisionMatrixRTC(opts); + promise.catch(() => {}); + + await vi.waitFor(() => expect(session.on).toHaveBeenCalled()); + expect(cancelMembership).toBeDefined(); + + cancelMembership!(); + await vi.runAllTimersAsync(); + await expect(promise).rejects.toThrow('MatrixRTC membership wait cancelled'); + // on cancel, getStateEvent should never have been polled with fake timers + // (interval hasn't ticked yet, and cancel cleared it) + } finally { + vi.useRealTimers(); + } + }); + + it('rejects on SDK error path even with fallback available', async () => { + // MembershipManagerError wins: membership on server does not matter + const session = makeSession(); + const mx = makeClient({ + getStateEvent: vi.fn().mockResolvedValue({}), + }); + const opts = callOpts({ session, mx }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipManagerError, + expect.any(Function) + ) + ); + + emitMembershipManagerError(session); + + await expect(promise).rejects.toThrow('MatrixRTC membership publication failed'); + // fallback may have polled, but error settles first + }); + + it('cleans up listeners after resolution', async () => { + const session = makeSession(); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + + await promise; + + expect(session.removeListener).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ); + expect(session.removeListener).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipManagerError, + expect.any(Function) + ); + }); +}); + +const firePageHide = (persisted: boolean): void => { + const event = new Event('pagehide') as PageTransitionEvent; + Object.defineProperty(event, 'persisted', { value: persisted }); + window.dispatchEvent(event); +}; + +describe('leaveMatrixRTCOnPageHide', () => { + it('leaves the session when the page is torn down for good', () => { + const session = makeSession(); + const remove = leaveMatrixRTCOnPageHide(session); + + firePageHide(false); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + + remove(); + firePageHide(false); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + }); + + it('stays in the call when the page is only frozen', () => { + const session = makeSession(); + const remove = leaveMatrixRTCOnPageHide(session); + + firePageHide(true); + + expect(session.leaveRoomSession).not.toHaveBeenCalled(); + remove(); + }); +}); diff --git a/src/app/features/call/matrixRtcCallLifecycle.ts b/src/app/features/call/matrixRtcCallLifecycle.ts new file mode 100644 index 000000000..7b0e7c90b --- /dev/null +++ b/src/app/features/call/matrixRtcCallLifecycle.ts @@ -0,0 +1,314 @@ +import type { AutoDiscoveryInfo } from '../../cs-api'; +import { + EventType, + MatrixRTCSessionEvent, + type CallMembership, + type JoinSessionConfig, + type MatrixClient, + type MatrixRTCSession, + type Room, +} from '$types/matrix-sdk'; +import { + getPreferredLivekitTransport, + isLivekitTransportConfig, + provisionLivekitToken, + useStickyMemberships, +} from './livekitProvisioning'; +import type { LivekitProvisioningResult } from './livekitProvisioning'; +import { createDebugLogger } from '$utils/debugLogger'; + +const debugLog = createDebugLogger('matrixRtcCallLifecycle'); + +// Without delayed events nothing on the server retracts our membership when the +// app dies, so the expiry is what bounds a ghost participant. The SDK refreshes +// the event 5s before it lapses, so half an hour is far more headroom than a +// live call needs while cutting the stale window down from the 4h default. +const membershipEventExpiryMs = 30 * 60 * 1000; + +const membershipWaitTimeoutMs = 30_000; +const fallbackPollIntervalMs = 1_000; +const callMemberEventType = EventType.RTCMembership; +const legacyCallMemberEventType = EventType.GroupCallMemberPrefix; + +export type MatrixRTCJoinProvisionOptions = { + mx: MatrixClient; + room: Room; + session: MatrixRTCSession; + discovery?: Pick; + getPreferredTransport?: typeof getPreferredLivekitTransport; + provisionToken?: typeof provisionLivekitToken; + callIntent: JoinSessionConfig['callIntent']; + notificationType?: JoinSessionConfig['notificationType']; + manageMediaKeys?: boolean; + isCancelled?: () => boolean; + onStage?: (stage: 'joining-matrix' | 'provisioning') => void; + onMembershipWait?: (cancel: (() => void) | undefined) => void; + onJoinStarted?: () => void; +}; + +export type MatrixRTCJoinProvisionResult = { + ownMembership: CallMembership | undefined; + provisioned: LivekitProvisioningResult; +}; + +type MembershipWait = { + promise: Promise; + cancel: () => void; +}; + +const waitForOwnMembership = ( + session: MatrixRTCSession, + userId: string, + deviceId: string, + mx: MatrixClient, + roomId: string +): MembershipWait => { + let resolveWait!: () => void; + let rejectWait!: (reason?: unknown) => void; + let settled = false; + let timeout: ReturnType | undefined; + let fallbackTimer: ReturnType | undefined; + let membershipsListenerInstalled = false; + let membershipErrorListenerInstalled = false; + + const stateKey = `_${userId}_${deviceId}_m.call`; + + const handleMembershipsChanged = ( + _oldMemberships: CallMembership[], + memberships: CallMembership[] + ): void => { + debugLog.info( + 'call', + `membership changed: n=${memberships.length} want=${userId}:${deviceId} have=${memberships.map((m) => `${m.userId}:${m.deviceId}`).join(',')}` + ); + + if ( + memberships.some( + (membership) => membership.userId === userId && membership.deviceId === deviceId + ) + ) { + settle(resolveWait); + } + }; + + const handleMembershipManagerError = (): void => { + settle(() => rejectWait(new Error('MatrixRTC membership publication failed'))); + }; + + const removeListeners = (): void => { + if (membershipsListenerInstalled) { + try { + session.removeListener(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChanged); + } catch {} + membershipsListenerInstalled = false; + } + if (membershipErrorListenerInstalled) { + try { + session.removeListener( + MatrixRTCSessionEvent.MembershipManagerError, + handleMembershipManagerError + ); + } catch {} + membershipErrorListenerInstalled = false; + } + }; + + const stopFallback = (): void => { + if (fallbackTimer !== undefined) { + clearInterval(fallbackTimer); + fallbackTimer = undefined; + } + }; + + const settle = (settlePromise: () => void): void => { + if (settled) return; + settled = true; + if (timeout !== undefined) clearTimeout(timeout); + stopFallback(); + removeListeners(); + settlePromise(); + }; + + const promise = new Promise((resolve, reject) => { + resolveWait = resolve; + rejectWait = reject; + }); + // joinRTCSession can throw before the promise is awaited; keep the cancel + // rejection from surfacing as an unhandled rejection in that window. + promise.catch(() => {}); + + try { + session.on(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChanged); + membershipsListenerInstalled = true; + session.on(MatrixRTCSessionEvent.MembershipManagerError, handleMembershipManagerError); + membershipErrorListenerInstalled = true; + timeout = setTimeout( + () => settle(() => rejectWait(new Error('MatrixRTC membership publication timed out'))), + membershipWaitTimeoutMs + ); + + // Server-side fallback: poll mx.getStateEvent so we detect the membership + // even when the local session filters it out during sync gaps. + fallbackTimer = setInterval(() => { + if (settled) return; + debugLog.info( + 'call', + `fallback poll: checking server-side membership ${stateKey} in ${roomId}` + ); + mx.getStateEvent(roomId, callMemberEventType, stateKey) + .then((event) => { + if (event && !settled) { + debugLog.info('call', `fallback resolved: found server-side membership ${stateKey}`); + settle(resolveWait); + } + }) + .catch(() => + // Try the legacy event type as fallback + mx + .getStateEvent(roomId, legacyCallMemberEventType, stateKey) + .then((event) => { + if (event && !settled) { + debugLog.info( + 'call', + `fallback resolved (legacy): found server-side membership ${stateKey}` + ); + settle(resolveWait); + } + }) + .catch(() => { + // next poll will retry + }) + ); + }, fallbackPollIntervalMs); + } catch { + settle(() => rejectWait(new Error('MatrixRTC membership listener setup failed'))); + } + + return { + promise, + cancel: () => settle(() => rejectWait(new Error('MatrixRTC membership wait cancelled'))), + }; +}; + +export const joinAndProvisionMatrixRTC = async ({ + mx, + room, + session, + discovery, + getPreferredTransport = getPreferredLivekitTransport, + provisionToken = provisionLivekitToken, + callIntent, + notificationType, + manageMediaKeys = false, + isCancelled, + onStage, + onMembershipWait, + onJoinStarted, +}: MatrixRTCJoinProvisionOptions): Promise => { + const deviceId = mx.getDeviceId(); + if (!deviceId) throw new Error('MatrixRTC device unavailable'); + + const preferredTransport = await getPreferredTransport(mx, discovery); + if (!preferredTransport) throw new Error('No LiveKit transport available'); + + // Element Call builds that predate Matrix 2.0 read `livekit_alias` off the + // advertised transport, so the legacy path keeps carrying it. + const advertisedTransport = useStickyMemberships + ? preferredTransport + : { livekit_alias: room.roomId, ...preferredTransport }; + + const userId = mx.getSafeUserId(); + const identity = { userId, deviceId, memberId: `${userId}:${deviceId}` }; + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + + // The SDK discards RTC memberships for anyone it cannot see joined, and + // sliding sync only ships `m.room.member/$ME`, so without the roster even our + // own membership is filtered out and `session.memberships` stays empty. Not + // fatal: the membership wait falls back to polling server-side state, so a + // failure here costs the roster UI rather than the call. + await room.loadMembersIfNeeded().catch(() => undefined); + + const membershipWait = waitForOwnMembership( + session, + identity.userId, + identity.deviceId, + mx, + room.roomId + ); + onMembershipWait?.(membershipWait.cancel); + onStage?.('joining-matrix'); + + try { + const joinConfig: JoinSessionConfig = { + callIntent, + membershipEventExpiryMs, + ...(notificationType ? { notificationType } : {}), + ...(manageMediaKeys ? { manageMediaKeys: true } : {}), + ...(useStickyMemberships ? { unstableSendStickyEvents: true } : {}), + }; + onJoinStarted?.(); + session.joinRTCSession(identity, [advertisedTransport], undefined, joinConfig); + await membershipWait.promise; + } catch (error) { + membershipWait.cancel(); + throw error; + } finally { + onMembershipWait?.(undefined); + } + + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + const slotId = session.slotId; + if (!slotId) throw new Error('MatrixRTC slot was not assigned'); + const ownMembership = session.memberships?.find( + (membership) => + membership.userId === identity.userId && membership.deviceId === identity.deviceId + ); + + // We pass no multiSfuFocus, so the SDK advertises + // `focus_selection: "oldest_membership"`: every participant owes media to the + // oldest membership's transport. Publishing to our own preference instead + // puts us on an SFU the others never connect to. + const oldestMembership = session.getOldestMembership(); + const oldestTransport = oldestMembership?.getTransport(oldestMembership); + const callTransport = + oldestTransport && isLivekitTransportConfig(oldestTransport) + ? oldestTransport + : preferredTransport; + + onStage?.('provisioning'); + const provisioned = await provisionToken({ + mx, + roomId: room.roomId, + slotId, + deviceId, + serviceUrl: callTransport.livekit_service_url, + memberId: identity.memberId, + userId: identity.userId, + }); + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + + return { ownMembership, provisioned }; +}; + +export const leaveMatrixRTCOnPageHide = (session: MatrixRTCSession): (() => void) => { + const handlePageHide = (event: PageTransitionEvent): void => { + // A persisted page is only frozen (mobile app switch, back/forward cache) + // and the call is still ours when it resumes; only a real teardown leaves. + if (event.persisted) return; + void session.leaveRoomSession().catch(() => undefined); + }; + window.addEventListener('pagehide', handlePageHide); + return () => window.removeEventListener('pagehide', handlePageHide); +}; + +export const disconnectLivekitThenLeaveMatrixRTC = async ( + disconnect: () => Promise, + session: MatrixRTCSession +): Promise => { + try { + await disconnect(); + } catch {} + try { + await session.leaveRoomSession(5000); + } catch {} +}; diff --git a/src/app/features/call/nativeCallController.test.ts b/src/app/features/call/nativeCallController.test.ts new file mode 100644 index 000000000..b79c71963 --- /dev/null +++ b/src/app/features/call/nativeCallController.test.ts @@ -0,0 +1,514 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { + MatrixRTCSessionEvent, + type CallMembership, + type CallMembershipIdentityParts, + type MatrixClient, + type MatrixRTCSession, + type Room, +} from '$types/matrix-sdk'; +import { + createNativeCallController, + type NativeCallControllerDependencies, +} from './nativeCallController'; +import type { NativeCallSnapshot } from './livekitMobileBridge'; +import type { NativeCallSession } from '$state/nativeCall'; +import { + acquireCallOwner, + getActiveCallOwner, + resetCallOwnerForTests, + type CallOwnerLease, +} from '$state/callOwner'; + +const OWN_IDENTITY = '@alice:example.org:DEVICE'; +const OWN_PARTS = { + userId: '@alice:example.org', + deviceId: 'DEVICE', +} as CallMembershipIdentityParts; +const REMOTE_PARTS = { + userId: '@bob:example.org', + deviceId: 'BOBDEVICE', +} as CallMembershipIdentityParts; +const room = { + roomId: '!room:example.org', + loadMembersIfNeeded: () => Promise.resolve(), +} as unknown as Room; +const transport = { type: 'livekit' as const, livekit_service_url: 'https://sfu.example' }; +const ownMembership = { + userId: '@alice:example.org', + deviceId: 'DEVICE', + rtcBackendIdentity: OWN_IDENTITY, +} as CallMembership; + +const idleSnapshot = (callId: string | null = null): NativeCallSnapshot => ({ + revision: 1, + callId, + connectionState: 'idle', + microphoneEnabled: false, + cameraEnabled: false, + participantCount: 0, +}); + +const connectedSnapshot = (callId: string): NativeCallSnapshot => ({ + revision: 1, + callId, + connectionState: 'connected', + microphoneEnabled: true, + cameraEnabled: false, + participantCount: 1, +}); + +type SessionHandler = (...args: unknown[]) => void; + +type TestSession = MatrixRTCSession & { + handlers: Map; +}; + +const makeSession = (order: string[] = []): TestSession => { + const handlers = new Map(); + const session = { + handlers, + memberships: [] as CallMembership[], + slotId: 'm.call#real-slot', + on: vi.fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>((event, handler) => { + if (event === MatrixRTCSessionEvent.EncryptionKeyChanged) order.push('attach-keys'); + handlers.set(event, handler); + }), + off: vi.fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>( + (event, handler) => { + if (handlers.get(event) === handler) handlers.delete(event); + } + ), + removeListener: vi.fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>( + (event, handler) => { + if (handlers.get(event) === handler) handlers.delete(event); + } + ), + reemitEncryptionKeys: vi.fn<() => void>(), + joinRTCSession: vi.fn<() => void>(() => order.push('join')), + getOldestMembership: vi.fn<() => CallMembership | undefined>(() => undefined), + leaveRoomSession: vi.fn<() => Promise>(async () => { + order.push('leave'); + return true; + }), + } as unknown as TestSession; + return session; +}; + +const emitOwnMembership = (session: TestSession): void => { + session.memberships = [ownMembership]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)?.([], [ownMembership]); +}; + +const emitKey = ( + session: TestSession, + key: number[], + keyIndex: number, + identity: string, + parts: CallMembershipIdentityParts = OWN_PARTS +): void => { + session.handlers.get(MatrixRTCSessionEvent.EncryptionKeyChanged)?.( + new Uint8Array(key), + keyIndex, + parts, + identity + ); +}; + +const waitForMembershipListener = async (session: TestSession): Promise => { + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); +}; + +const flushMicrotasks = async (ticks = 20): Promise => { + for (let index = 0; index < ticks; index += 1) { + // oxlint-disable-next-line no-await-in-loop -- flushing chained microtasks one tick at a time + await Promise.resolve(); + } +}; + +const makeClient = (session: MatrixRTCSession): MatrixClient => + ({ + getDeviceId: () => 'DEVICE', + getSafeUserId: () => '@alice:example.org', + matrixRTC: { getRoomSession: () => session }, + }) as unknown as MatrixClient; + +type Harness = { + dependencies: NativeCallControllerDependencies; + setSession: ReturnType void>>; + connectCall: ReturnType; + disconnectCall: ReturnType; + setCamera: ReturnType; + setEncryptionKey: ReturnType; + emitSnapshot: (snapshot: NativeCallSnapshot) => void; + unlisten: ReturnType void>>; +}; + +const makeDependencies = (order: string[]): Harness => { + const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); + const connectCall = vi.fn<() => Promise>(async () => { + order.push('connect'); + return connectedSnapshot('call-id'); + }); + const disconnectCall = vi.fn<() => Promise>(async () => { + order.push('native-disconnect'); + return idleSnapshot(); + }); + const setCamera = vi.fn<() => Promise>(async () => + connectedSnapshot('call-id') + ); + const setMicrophone = vi.fn<() => Promise>(async () => + connectedSnapshot('call-id') + ); + const setEncryptionKey = vi.fn<() => Promise>(async () => + connectedSnapshot('call-id') + ); + const unlisten = vi.fn<() => void>(); + let snapshotHandler: ((snapshot: NativeCallSnapshot) => void) | undefined; + + return { + setSession, + connectCall, + disconnectCall, + setCamera, + setEncryptionKey, + unlisten, + emitSnapshot: (snapshot) => snapshotHandler?.(snapshot), + dependencies: { + setSession, + connectCall, + disconnectCall, + setCamera, + setMicrophone, + setEncryptionKey, + listenSnapshot: async (handler) => { + snapshotHandler = handler; + return unlisten; + }, + getPreferredTransport: async () => transport, + provisionToken: async () => ({ url: 'wss://livekit.example', jwt: 'jwt' }), + createCallId: () => 'call-id', + }, + }; +}; + +const startOptions = (session: MatrixRTCSession, video = false, microphone = true) => ({ + mx: makeClient(session), + room, + dm: false, + video, + microphone, + ongoing: false, +}); + +const lastSession = ( + setSession: ReturnType void>> +): NativeCallSession | undefined => setSession.mock.calls.at(-1)?.[0]; + +beforeEach(() => { + resetCallOwnerForTests(); +}); + +describe('native call controller', () => { + it('attaches the key forwarder before the Matrix join with managed media keys', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, connectCall } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + expect(order).toEqual(['attach-keys', 'join']); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + expect(session.joinRTCSession).toHaveBeenCalledWith( + { userId: '@alice:example.org', deviceId: 'DEVICE', memberId: '@alice:example.org:DEVICE' }, + [{ ...transport, livekit_alias: room.roomId }], + undefined, + { + callIntent: 'audio', + membershipEventExpiryMs: 30 * 60 * 1000, + notificationType: 'notification', + manageMediaKeys: true, + } + ); + expect(connectCall).toHaveBeenCalled(); + }); + + it('gates the native connect on the own-identity key and caches other keys', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, connectCall, setEncryptionKey } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + await vi.waitFor(() => expect(session.joinRTCSession).toHaveBeenCalled()); + emitKey(session, [9, 9], 0, 'remote-backend', REMOTE_PARTS); + await Promise.resolve(); + expect(connectCall).not.toHaveBeenCalled(); + + emitKey(session, [1, 2, 3, 4], 1, OWN_IDENTITY); + await startPromise; + + expect(connectCall).toHaveBeenCalledWith({ + callId: 'call-id', + url: 'wss://livekit.example', + token: 'jwt', + microphoneEnabled: true, + encryptionKeys: [ + { identity: 'remote-backend', keyIndex: 0, key: 'CQk=' }, + { identity: OWN_IDENTITY, keyIndex: 1, key: 'AQIDBA==' }, + ], + }); + expect(lastSession(dependencies.setSession as Harness['setSession'])).toMatchObject({ + backend: 'livekit-mobile', + roomId: room.roomId, + callId: 'call-id', + lifecycle: 'connected', + }); + expect(setEncryptionKey).not.toHaveBeenCalled(); + }); + + it('joins muted when the prescreen microphone was off', async () => { + const session = makeSession(); + const { dependencies, connectCall } = makeDependencies([]); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session, false, false)); + await waitForMembershipListener(session); + emitOwnMembership(session); + await vi.waitFor(() => expect(session.joinRTCSession).toHaveBeenCalled()); + emitKey(session, [1, 2, 3, 4], 1, OWN_IDENTITY); + await startPromise; + + expect(connectCall).toHaveBeenCalledWith(expect.objectContaining({ microphoneEnabled: false })); + }); + + it('rotates keys through the set command only after connect resolves', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, connectCall, setEncryptionKey } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [9, 9], 0, 'remote-backend', REMOTE_PARTS); + emitKey(session, [1], 0, OWN_IDENTITY); + expect(setEncryptionKey).not.toHaveBeenCalled(); + await startPromise; + expect(setEncryptionKey).not.toHaveBeenCalled(); + + emitKey(session, [5, 6, 7, 8], 4, 'remote-backend', REMOTE_PARTS); + expect(connectCall).toHaveBeenCalledTimes(1); + expect(setEncryptionKey).toHaveBeenCalledWith({ + callId: 'call-id', + identity: 'remote-backend', + keyIndex: 4, + key: 'BQYHCA==', + }); + + setEncryptionKey.mockClear(); + emitKey(session, [5, 6, 7, 8], 4, 'remote-backend', REMOTE_PARTS); + emitKey(session, [8, 8], 2, 'remote-backend', REMOTE_PARTS); + expect(setEncryptionKey).not.toHaveBeenCalled(); + }); + + it('disconnects natively before leaving Matrix, detaches, and releases the owner once', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const release = vi.fn<() => void>(); + harness.dependencies.acquireOwner = (): CallOwnerLease => ({ + kind: 'livekit-mobile', + roomId: room.roomId, + release, + }); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + order.length = 0; + + const connected = lastSession(harness.setSession); + expect(connected?.lifecycle).toBe('connected'); + await connected?.hangup(); + await connected?.hangup(); + + expect(order).toEqual(['native-disconnect', 'leave']); + expect(session.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function) + ); + expect(harness.unlisten).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledTimes(1); + expect(lastSession(harness.setSession)).toBeUndefined(); + }); + + it('enables the camera after connect only when video was requested', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, setCamera } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session, true)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + expect(order).toEqual(['attach-keys', 'join', 'connect']); + expect(setCamera).toHaveBeenCalledWith({ callId: 'call-id', enabled: true }); + + // Release the owner lease so the audio-path controller can start. + await lastSession(dependencies.setSession as Harness['setSession'])?.hangup(); + + const audioOrder: string[] = []; + const audioSession = makeSession(audioOrder); + const audioHarness = makeDependencies(audioOrder); + const audioController = createNativeCallController(audioHarness.dependencies); + const audioStart = audioController.start(startOptions(audioSession)); + await waitForMembershipListener(audioSession); + emitOwnMembership(audioSession); + emitKey(audioSession, [1], 0, OWN_IDENTITY); + await audioStart; + expect(audioHarness.setCamera).not.toHaveBeenCalled(); + }); + + it('tracks lifecycle and media flags from snapshots and ends on a terminal snapshot', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + harness.emitSnapshot({ + ...connectedSnapshot('call-id'), + connectionState: 'reconnecting', + microphoneEnabled: false, + cameraEnabled: true, + }); + expect(lastSession(harness.setSession)).toMatchObject({ + lifecycle: 'reconnecting', + microphoneEnabled: false, + cameraEnabled: true, + }); + + order.length = 0; + harness.emitSnapshot(idleSnapshot(null)); + await vi.waitFor(() => expect(session.leaveRoomSession).toHaveBeenCalled()); + expect(order).toEqual(['native-disconnect', 'leave']); + // A normal end clears the session; it is not an error the user must dismiss. + expect(lastSession(harness.setSession)).toBeUndefined(); + expect(getActiveCallOwner()).toBeUndefined(); + }); + + it('treats a failed snapshot for the current call as a call failure', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + harness.emitSnapshot({ + ...idleSnapshot('call-id'), + connectionState: 'failed', + lastError: { code: 'disconnected', message: 'disconnected' }, + }); + await vi.waitFor(() => expect(harness.disconnectCall).toHaveBeenCalled()); + // The failure code is mapped to user-facing copy, not passed through raw. + expect(lastSession(harness.setSession)).toMatchObject({ + lifecycle: 'error', + error: 'The connection to the call was lost.', + }); + }); + + it('cleans up and hides secrets when the own-key wait times out', async () => { + vi.useFakeTimers(); + try { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await flushMicrotasks(); + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ); + emitOwnMembership(session); + await flushMicrotasks(); + await vi.advanceTimersByTimeAsync(10_000); + await startPromise; + + expect(harness.connectCall).not.toHaveBeenCalled(); + expect(order).toEqual(['attach-keys', 'join', 'native-disconnect', 'leave']); + const failed = lastSession(harness.setSession); + expect(failed).toMatchObject({ lifecycle: 'error' }); + expect(failed?.error).toContain('Could not connect to the call.'); + // The surfaced message must never carry the provisioned token. + expect(failed?.error).not.toContain('jwt'); + expect(getActiveCallOwner()).toBeUndefined(); + } finally { + vi.useRealTimers(); + } + }); + + it('does not start while another call owns the owner lease', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const onCleanup = vi.fn<() => void>(); + harness.dependencies.onCleanup = onCleanup; + const elementLease = acquireCallOwner('element', '!other:example.org'); + + const controller = createNativeCallController(harness.dependencies); + await controller.start(startOptions(session)); + + expect(session.joinRTCSession).not.toHaveBeenCalled(); + expect(harness.setSession).not.toHaveBeenCalled(); + expect(onCleanup).toHaveBeenCalled(); + elementLease?.release(); + }); + + it('ignores a second start while a native call is active', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + const secondSession = makeSession(); + await controller.start(startOptions(secondSession)); + expect(secondSession.joinRTCSession).not.toHaveBeenCalled(); + expect(harness.connectCall).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/app/features/call/nativeCallController.ts b/src/app/features/call/nativeCallController.ts new file mode 100644 index 000000000..b153cf95a --- /dev/null +++ b/src/app/features/call/nativeCallController.ts @@ -0,0 +1,536 @@ +import type { AutoDiscoveryInfo } from '../../cs-api'; +import type { MatrixClient, MatrixRTCSession, Room } from '$types/matrix-sdk'; +import { + connectNativeCall, + disconnectNativeCall, + drainPendingSystemCallActions, + endSystemCall, + fulfillAnswerCall, + fulfillEndCall, + getAudioRoutes, + getNativeCallState, + listenNativeCallSnapshot, + onSystemCallAction, + reportSystemCallConnected, + setAudioRoute, + setNativeCallCameraEnabled, + setNativeCallEncryptionKey, + setNativeCallMicrophoneEnabled, + setNativeCallPiPEnabled, + setSystemCallMuted, + startSystemCall, + switchNativeCallCamera, + updateCallDisplay, + type NativeCallAudioRoute, + type NativeCallEncryptionKeyPayload, + type NativeCallFailureCode, + type NativeCallSnapshot, + type SystemCallAction, +} from './livekitMobileBridge'; +import { getPreferredLivekitTransport, provisionLivekitToken } from './livekitProvisioning'; +import { + createNativeCallKeyForwarder, + type NativeCallKeyForwarder, +} from './nativeCallKeyForwarder'; +import { + joinAndProvisionMatrixRTC, + disconnectLivekitThenLeaveMatrixRTC, + leaveMatrixRTCOnPageHide, +} from './matrixRtcCallLifecycle'; +import { acquireCallOwner, type CallOwnerLease } from '$state/callOwner'; +import type { NativeCallLifecycle, NativeCallSession } from '$state/nativeCall'; +import { createDebugLogger } from '$utils/debugLogger'; + +const debugLog = createDebugLogger('nativeCallController'); + +type SetupStage = 'joining the call' | 'authorizing' | 'connecting'; + +const setupErrorMessage = (stage: SetupStage): string => { + if (stage === 'joining the call') return 'Could not join the call.'; + if (stage === 'authorizing') return 'Could not get permission to join the call.'; + return 'Could not connect to the call.'; +}; + +const failureMessages: Record = { + invalid_request: 'Could not start the call.', + busy: 'Another call is already in progress.', + permission_denied: 'Microphone or camera access was denied.', + connect_failed: 'Could not connect to the call.', + media_failed: 'Your microphone or camera stopped working.', + disconnected: 'The connection to the call was lost.', + cancelled: 'The call was cancelled.', + unavailable: 'Calls are not available on this device.', + unexpected: 'The call ended unexpectedly.', +}; + +const nativeFailureMessage = (code: NativeCallFailureCode | undefined): string => + (code && failureMessages[code]) || 'The call ended unexpectedly.'; + +const failedMessage = 'Native call failed:'; + +const logFailure = (message: string): void => { + debugLog.error('call', message); +}; + +export type NativeCallStartOptions = { + mx: MatrixClient; + room: Room; + discovery?: Pick; + dm: boolean; + video: boolean; + microphone: boolean; + ongoing: boolean; +}; + +type NativeCallRecord = { + roomId: string; + callId: string; + session: MatrixRTCSession; + forwarder: NativeCallKeyForwarder; + ownerLease: CallOwnerLease; + video: boolean; + cancelled: boolean; + connectResolved: boolean; + sentKeyIndices: Map; + cancelMembershipWait?: () => void; + removePageHideListener?: () => void; + snapshotUnlistenPromise?: Promise<() => void>; + cleanupPromise?: Promise; +}; + +export type NativeCallController = { + start: (options: NativeCallStartOptions) => Promise; + setMicrophoneEnabled: (enabled: boolean) => Promise; + setCameraEnabled: (enabled: boolean) => Promise; + switchCamera: () => Promise; + listAudioRoutes: () => Promise; + selectAudioRoute: (routeId: string) => Promise; +}; + +export type NativeCallControllerDependencies = { + setSession: (session: NativeCallSession | undefined) => void; + connectCall?: typeof connectNativeCall; + disconnectCall?: typeof disconnectNativeCall; + setMicrophone?: typeof setNativeCallMicrophoneEnabled; + setCamera?: typeof setNativeCallCameraEnabled; + setEncryptionKey?: typeof setNativeCallEncryptionKey; + updateDisplay?: typeof updateCallDisplay; + listenSnapshot?: typeof listenNativeCallSnapshot; + createKeyForwarder?: () => NativeCallKeyForwarder; + getPreferredTransport?: typeof getPreferredLivekitTransport; + provisionToken?: typeof provisionLivekitToken; + createCallId?: () => string; + acquireOwner?: typeof acquireCallOwner; + onCleanup?: () => void; +}; + +const noMediaControls = { + setMicrophoneEnabled: async (): Promise => {}, + setCameraEnabled: async (): Promise => {}, + switchCamera: async (): Promise => {}, + listAudioRoutes: async (): Promise => [], + selectAudioRoute: async (): Promise => {}, +}; + +const toLifecycle = ( + state: NativeCallSnapshot['connectionState'] +): NativeCallLifecycle | undefined => { + if (state === 'connecting' || state === 'connected' || state === 'reconnecting') return state; + return undefined; +}; + +export const createNativeCallController = ( + dependencies: Pick & + Omit +): NativeCallController => { + const deps = { + ...dependencies, + connectCall: dependencies.connectCall ?? connectNativeCall, + disconnectCall: dependencies.disconnectCall ?? disconnectNativeCall, + setMicrophone: dependencies.setMicrophone ?? setNativeCallMicrophoneEnabled, + setCamera: dependencies.setCamera ?? setNativeCallCameraEnabled, + setEncryptionKey: dependencies.setEncryptionKey ?? setNativeCallEncryptionKey, + updateDisplay: dependencies.updateDisplay ?? updateCallDisplay, + listenSnapshot: dependencies.listenSnapshot ?? listenNativeCallSnapshot, + createKeyForwarder: dependencies.createKeyForwarder ?? createNativeCallKeyForwarder, + getPreferredTransport: dependencies.getPreferredTransport ?? getPreferredLivekitTransport, + provisionToken: dependencies.provisionToken ?? provisionLivekitToken, + createCallId: dependencies.createCallId ?? (() => crypto.randomUUID()), + acquireOwner: dependencies.acquireOwner ?? acquireCallOwner, + }; + + // Register the system-call (CallKit) listener once at controller creation. + // Events (answer, end, mute from lock-screen/system-UI) flow from the + // native side regardless of call state. + let pendingSystemUuid: string | undefined; + + // When the WebView resumes (phone unlocked, app foregrounded), the JS + // state may be stale: snapshot events emitted while JS was suspended are + // lost. Poll the native state to resync. + const handleVisibilityChange = () => { + if (document.visibilityState === 'visible' && activeRecord && !activeRecord.cancelled) { + void getNativeCallState() + .then((snapshot) => handleSnapshot(activeRecord!, snapshot)) + .catch(() => undefined); + } + }; + document.addEventListener('visibilitychange', handleVisibilityChange); + + void onSystemCallAction((action: SystemCallAction) => { + if (action.action === 'end') { + // System UI ended the call: hang up if active, then fulfill the + // pending CXEndCallAction so the system UI dismisses immediately. + if (activeRecord && !activeRecord.cancelled) { + void cleanup(activeRecord, undefined, true).finally(() => { + if (action.uuid) void fulfillEndCall(action.uuid).catch(() => undefined); + }); + } else if (action.uuid) { + void fulfillEndCall(action.uuid).catch(() => undefined); + } + void drainPendingSystemCallActions().catch(() => undefined); + } else if (action.action === 'answer') { + // System UI answered an incoming call: store the uuid for the + // start path so it can map back to the system call. + pendingSystemUuid = action.uuid; + } else if (action.action === 'mute') { + // System UI mute toggle: push to LiveKit. + if (activeRecord && !activeRecord.cancelled && activeRecord.connectResolved) { + void deps + .setMicrophone({ callId: activeRecord.callId, enabled: !action.muted }) + .catch(() => undefined); + } + } + }).catch(() => undefined); + + let activeRecord: NativeCallRecord | undefined; + let displayedRecord: NativeCallRecord | undefined; + + const isCurrent = (record: NativeCallRecord): boolean => + activeRecord === record && !record.cancelled; + + const setSetupError = (roomId: string, callId: string, error: string): void => { + try { + deps.setSession({ + backend: 'livekit-mobile', + roomId, + callId, + lifecycle: 'error', + error, + microphoneEnabled: false, + cameraEnabled: false, + ...noMediaControls, + hangup: async () => { + try { + deps.setSession(undefined); + } catch {} + }, + }); + } catch {} + }; + + const publish = ( + record: NativeCallRecord, + lifecycle: NativeCallLifecycle, + media?: { microphoneEnabled: boolean; cameraEnabled: boolean }, + error?: string + ): void => { + if (!isCurrent(record)) return; + deps.setSession({ + backend: 'livekit-mobile', + roomId: record.roomId, + callId: record.callId, + lifecycle, + ...(error ? { error } : {}), + microphoneEnabled: media?.microphoneEnabled ?? true, + cameraEnabled: media?.cameraEnabled ?? false, + setMicrophoneEnabled, + setCameraEnabled, + switchCamera, + listAudioRoutes, + selectAudioRoute, + hangup: () => cleanup(record, undefined, true), + }); + displayedRecord = record; + }; + + const forwardKey = (record: NativeCallRecord, key: NativeCallEncryptionKeyPayload): void => { + if (!record.connectResolved || record.cancelled) return; + const sentIndex = record.sentKeyIndices.get(key.identity) ?? -1; + if (key.keyIndex <= sentIndex) return; + record.sentKeyIndices.set(key.identity, key.keyIndex); + void deps + .setEncryptionKey({ + callId: record.callId, + identity: key.identity, + keyIndex: key.keyIndex, + key: key.key, + }) + .catch(() => undefined); + }; + + const cleanup = async ( + record: NativeCallRecord, + failure: string | undefined, + clear: boolean + ): Promise => { + if (record.cleanupPromise) { + await record.cleanupPromise; + if (clear && displayedRecord === record) { + displayedRecord = undefined; + try { + deps.setSession(undefined); + } catch {} + } + return; + } + + record.cancelled = true; + record.cancelMembershipWait?.(); + record.cancelMembershipWait = undefined; + record.removePageHideListener?.(); + record.removePageHideListener = undefined; + record.forwarder.setOnKey(undefined); + if (failure && activeRecord === record) { + try { + deps.setSession({ + backend: 'livekit-mobile', + roomId: record.roomId, + callId: record.callId, + lifecycle: 'error', + error: failure, + microphoneEnabled: false, + cameraEnabled: false, + ...noMediaControls, + hangup: () => cleanup(record, undefined, true), + }); + } catch {} + displayedRecord = record; + } + + record.cleanupPromise = (async () => { + // End the system call so CallKit dismisses the active-call UI. + void endSystemCall({ callId: record.callId, remoteEnded: true }).catch(() => undefined); + // Disable native PiP on cleanup. + void setNativeCallPiPEnabled({ callId: record.callId, enabled: false }).catch( + () => undefined + ); + await disconnectLivekitThenLeaveMatrixRTC( + () => deps.disconnectCall({ callId: record.callId }).then(() => undefined), + record.session + ); + await Promise.allSettled([ + record.snapshotUnlistenPromise?.then((unlisten) => unlisten()) ?? Promise.resolve(), + ]); + record.forwarder.detach(); + record.ownerLease.release(); + if (activeRecord === record) activeRecord = undefined; + if (clear && displayedRecord === record) { + displayedRecord = undefined; + try { + deps.setSession(undefined); + } catch {} + } + deps.onCleanup?.(); + })(); + await record.cleanupPromise; + }; + + const handleSnapshot = (record: NativeCallRecord, snapshot: NativeCallSnapshot): void => { + if (!isCurrent(record)) return; + if (snapshot.connectionState === 'idle' || snapshot.connectionState === 'failed') { + if (snapshot.callId === null && !record.connectResolved) return; + if (snapshot.callId !== null && snapshot.callId !== record.callId) return; + if (snapshot.connectionState === 'idle') { + // The call ended normally, usually because the other side hung up. Clear the + // session rather than publishing an error the user has to dismiss. + void cleanup(record, undefined, true); + return; + } + logFailure(`${failedMessage} ${snapshot.lastError?.code ?? 'unknown'}`); + void cleanup(record, nativeFailureMessage(snapshot.lastError?.code), false); + return; + } + if (snapshot.callId !== record.callId) return; + const lifecycle = toLifecycle(snapshot.connectionState); + if (!lifecycle) return; + publish(record, lifecycle, { + microphoneEnabled: snapshot.microphoneEnabled, + cameraEnabled: snapshot.cameraEnabled, + }); + }; + + const start = async ({ + mx, + room, + discovery, + dm, + video, + microphone, + ongoing, + }: NativeCallStartOptions) => { + if (activeRecord) { + deps.onCleanup?.(); + return; + } + const ownerLease = deps.acquireOwner('livekit-mobile', room.roomId); + if (!ownerLease) { + deps.onCleanup?.(); + return; + } + + let record: NativeCallRecord | undefined; + let stage: SetupStage = 'joining the call'; + try { + const callId = deps.createCallId(); + const session = mx.matrixRTC.getRoomSession(room); + const forwarder = deps.createKeyForwarder(); + record = { + roomId: room.roomId, + callId, + session, + forwarder, + ownerLease, + video, + cancelled: false, + connectResolved: false, + sentKeyIndices: new Map(), + }; + const currentRecord = record; + activeRecord = record; + publish(record, 'starting'); + + record.snapshotUnlistenPromise = deps.listenSnapshot((snapshot) => + handleSnapshot(currentRecord, snapshot) + ); + await record.snapshotUnlistenPromise; + + forwarder.attach(session, { userId: mx.getSafeUserId(), deviceId: mx.getDeviceId() }); + + const joined = await joinAndProvisionMatrixRTC({ + mx, + room, + session, + discovery, + getPreferredTransport: deps.getPreferredTransport, + provisionToken: deps.provisionToken, + callIntent: video ? 'video' : 'audio', + ...(ongoing ? {} : { notificationType: dm ? 'ring' : 'notification' }), + manageMediaKeys: true, + isCancelled: () => !isCurrent(currentRecord), + onStage: (joinStage) => { + stage = joinStage === 'joining-matrix' ? 'joining the call' : 'authorizing'; + }, + onMembershipWait: (cancel) => { + currentRecord.cancelMembershipWait = cancel; + }, + onJoinStarted: () => { + currentRecord.removePageHideListener = leaveMatrixRTCOnPageHide(session); + }, + }); + if (!isCurrent(currentRecord)) return; + + stage = 'connecting'; + await forwarder.waitForOwnKey(); + if (!isCurrent(currentRecord)) return; + + const encryptionKeys = forwarder.getKeys(); + encryptionKeys.forEach((key) => { + currentRecord.sentKeyIndices.set(key.identity, key.keyIndex); + }); + const snapshot = await deps.connectCall({ + callId, + url: joined.provisioned.url, + token: joined.provisioned.jwt, + microphoneEnabled: microphone, + encryptionKeys, + }); + currentRecord.connectResolved = true; + forwarder.setOnKey((key) => forwardKey(currentRecord, key)); + forwarder.getKeys().forEach((key) => forwardKey(currentRecord, key)); + handleSnapshot(currentRecord, snapshot); + + // Report the outgoing system call so CallKit shows the active-call UI. + // Use the pending uuid (from an answer action) if present; otherwise + // generate a new uuid. The native side maps callId ↔ uuid internally. + const systemUuid = pendingSystemUuid ?? crypto.randomUUID(); + const isIncomingAnswer = pendingSystemUuid !== undefined; + pendingSystemUuid = undefined; + const callerName = room.name || room.roomId; + void startSystemCall({ callId, uuid: systemUuid, callerName }).catch(() => undefined); + // Update the system call display with the room name and video flag so + // CallKit shows the correct caller info. + void deps + .updateDisplay({ + callId, + callerName, + hasVideo: video, + }) + .catch(() => undefined); + // Report the call as connected to CallKit so the system UI updates. + void reportSystemCallConnected(systemUuid).catch(() => undefined); + // Enable native PiP when the call connects. + void setNativeCallPiPEnabled({ callId, enabled: true }).catch(() => undefined); + // For system-initiated incoming answers: fulfill the deferred answer action. + if (isIncomingAnswer) { + void fulfillAnswerCall(systemUuid).catch(() => undefined); + } + + if (video && isCurrent(currentRecord)) { + await deps.setCamera({ callId, enabled: true }).catch(() => undefined); + } + } catch (cause) { + const detail = cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause); + logFailure(`Native call setup failed during ${stage}. ${detail}`); + if (record) { + await cleanup(record, setupErrorMessage(stage), false); + } else { + ownerLease.release(); + setSetupError(room.roomId, '', setupErrorMessage(stage)); + deps.onCleanup?.(); + } + } + }; + + const setMicrophoneEnabled = async (enabled: boolean): Promise => { + const record = activeRecord; + if (!record || record.cancelled || !record.connectResolved) return; + await deps.setMicrophone({ callId: record.callId, enabled }).catch(() => undefined); + // Push mute state back to CallKit for UI consistency. + void setSystemCallMuted({ callId: record.callId, muted: !enabled }).catch(() => undefined); + }; + + const setCameraEnabled = async (enabled: boolean): Promise => { + const record = activeRecord; + if (!record || record.cancelled || !record.connectResolved) return; + await deps.setCamera({ callId: record.callId, enabled }).catch(() => undefined); + }; + + const switchCamera = async (): Promise => { + const record = activeRecord; + if (!record || record.cancelled || !record.connectResolved) return; + await switchNativeCallCamera({ callId: record.callId }).catch(() => undefined); + }; + + const listAudioRoutes = async (): Promise => { + const record = activeRecord; + if (!record || record.cancelled || !record.connectResolved) return []; + return getAudioRoutes({ callId: record.callId }) + .then((result) => result.routes) + .catch(() => []); + }; + + const selectAudioRoute = async (routeId: string): Promise => { + const record = activeRecord; + if (!record || record.cancelled || !record.connectResolved) return; + await setAudioRoute({ callId: record.callId, routeId }).catch(() => undefined); + }; + + return { + start, + setMicrophoneEnabled, + setCameraEnabled, + switchCamera, + listAudioRoutes, + selectAudioRoute, + }; +}; diff --git a/src/app/features/call/nativeCallKeyForwarder.test.ts b/src/app/features/call/nativeCallKeyForwarder.test.ts new file mode 100644 index 000000000..5b5742205 --- /dev/null +++ b/src/app/features/call/nativeCallKeyForwarder.test.ts @@ -0,0 +1,177 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + MatrixRTCSessionEvent, + type CallMembershipIdentityParts, + type MatrixRTCSession, +} from '$types/matrix-sdk'; +import { + createNativeCallKeyForwarder, + OWN_KEY_UNAVAILABLE_ERROR, + OWN_KEY_WAIT_CANCELLED_ERROR, + ownKeyWaitTimeoutMs, +} from './nativeCallKeyForwarder'; + +type EncryptionKeyHandler = ( + key: Uint8Array, + encryptionKeyIndex: number, + membership: CallMembershipIdentityParts, + rtcBackendIdentity: string +) => void; + +const localIdentity = { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }; +const localParts = localIdentity as CallMembershipIdentityParts; +const remoteParts = { + userId: '@bob:example.org', + deviceId: 'BOBDEVICE', +} as CallMembershipIdentityParts; + +const makeSession = () => { + const handlers = new Map(); + const session = { + on: vi.fn<(event: MatrixRTCSessionEvent, handler: EncryptionKeyHandler) => void>( + (event, handler) => { + handlers.set(event, handler); + } + ), + off: vi.fn<(event: MatrixRTCSessionEvent, handler: EncryptionKeyHandler) => void>( + (event, handler) => { + if (handlers.get(event) === handler) handlers.delete(event); + } + ), + reemitEncryptionKeys: vi.fn<() => void>(), + } as unknown as MatrixRTCSession; + return { + session, + handlers, + emitKey: ( + key: number[], + keyIndex: number, + identity: string, + parts: CallMembershipIdentityParts = remoteParts + ) => + handlers.get(MatrixRTCSessionEvent.EncryptionKeyChanged)?.( + new Uint8Array(key) as Uint8Array, + keyIndex, + parts, + identity + ), + }; +}; + +describe('native call key forwarder', () => { + beforeEach(() => { + vi.useRealTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('subscribes to key changes and re-emits on attach, unsubscribes on detach', () => { + const { session, handlers } = makeSession(); + const forwarder = createNativeCallKeyForwarder(); + + forwarder.attach(session, localIdentity); + + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function) + ); + expect(session.reemitEncryptionKeys).toHaveBeenCalled(); + + forwarder.detach(); + expect(session.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function) + ); + expect(handlers.has(MatrixRTCSessionEvent.EncryptionKeyChanged)).toBe(false); + }); + + it('caches raw keys as standard padded base64 per identity', () => { + const { session, emitKey } = makeSession(); + const forwarder = createNativeCallKeyForwarder(); + forwarder.attach(session, localIdentity); + + emitKey([1, 2, 3, 4], 0, 'backend-a'); + emitKey([0, 1, 2], 2, 'backend-b'); + + expect(forwarder.getKeys()).toEqual([ + { identity: 'backend-a', keyIndex: 0, key: 'AQIDBA==' }, + { identity: 'backend-b', keyIndex: 2, key: 'AAEC' }, + ]); + }); + + it('keeps the latest key index per identity and ignores stale events', () => { + const { session, emitKey } = makeSession(); + const forwarder = createNativeCallKeyForwarder(); + forwarder.attach(session, localIdentity); + + emitKey([1, 2, 3, 4], 2, 'backend-a'); + emitKey([9, 9], 1, 'backend-a'); + emitKey([5, 6], 2, 'backend-a'); + + expect(forwarder.getKeys()).toEqual([{ identity: 'backend-a', keyIndex: 2, key: 'AQIDBA==' }]); + }); + + it('notifies the key listener only for accepted rotations', () => { + const { session, emitKey } = makeSession(); + const forwarder = createNativeCallKeyForwarder(); + forwarder.attach(session, localIdentity); + const onKey = vi.fn<(entry: { identity: string; keyIndex: number; key: string }) => void>(); + forwarder.setOnKey(onKey); + + emitKey([1, 2, 3, 4], 0, 'backend-a'); + emitKey([1, 2, 3, 4], 0, 'backend-a'); + emitKey([7, 7, 7], 1, 'backend-a'); + + expect(onKey).toHaveBeenCalledTimes(2); + expect(onKey).toHaveBeenLastCalledWith({ identity: 'backend-a', keyIndex: 1, key: 'BwcH' }); + }); + + it('resolves the own-key wait on the key whose membership is ours', async () => { + const { session, emitKey } = makeSession(); + const forwarder = createNativeCallKeyForwarder(); + forwarder.attach(session, localIdentity); + + const wait = forwarder.waitForOwnKey(); + emitKey([1], 0, 'other-backend'); + + emitKey([2], 0, 'own-backend', localParts); + await expect(wait).resolves.toBeUndefined(); + }); + + it('resolves the own-key wait for a key that arrived before the wait started', async () => { + const { session, emitKey } = makeSession(); + const forwarder = createNativeCallKeyForwarder(); + forwarder.attach(session, localIdentity); + emitKey([2], 0, 'own-backend', localParts); + + await expect(forwarder.waitForOwnKey()).resolves.toBeUndefined(); + }); + + it('rejects the own-key wait after the timeout', async () => { + vi.useFakeTimers(); + const { session } = makeSession(); + const forwarder = createNativeCallKeyForwarder(); + forwarder.attach(session, localIdentity); + + const waitError = forwarder.waitForOwnKey().catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(ownKeyWaitTimeoutMs); + const timeoutError = await waitError; + expect(timeoutError).toBeInstanceOf(Error); + expect((timeoutError as Error).message).toBe(OWN_KEY_UNAVAILABLE_ERROR); + }); + + it('rejects a pending own-key wait on detach', async () => { + const { session } = makeSession(); + const forwarder = createNativeCallKeyForwarder(); + forwarder.attach(session, localIdentity); + + const waitError = forwarder.waitForOwnKey().catch((error: unknown) => error); + forwarder.detach(); + const detachError = await waitError; + expect(detachError).toBeInstanceOf(Error); + expect((detachError as Error).message).toBe(OWN_KEY_WAIT_CANCELLED_ERROR); + expect(forwarder.getKeys()).toEqual([]); + }); +}); diff --git a/src/app/features/call/nativeCallKeyForwarder.ts b/src/app/features/call/nativeCallKeyForwarder.ts new file mode 100644 index 000000000..787c6ab31 --- /dev/null +++ b/src/app/features/call/nativeCallKeyForwarder.ts @@ -0,0 +1,132 @@ +import { + MatrixRTCSessionEvent, + type CallMembershipIdentityParts, + type MatrixRTCSession, +} from '$types/matrix-sdk'; +import type { NativeCallEncryptionKeyPayload } from './livekitMobileBridge'; +import type { LocalCallIdentity } from './livekitCallIdentity'; +import { createDebugLogger } from '$utils/debugLogger'; + +const debugLog = createDebugLogger('nativeCallKeyForwarder'); + +export const ownKeyWaitTimeoutMs = 10_000; + +export const OWN_KEY_UNAVAILABLE_ERROR = 'Native call own encryption key unavailable'; +export const OWN_KEY_WAIT_CANCELLED_ERROR = 'Native call own key wait cancelled'; + +export type NativeCallKeyForwarder = { + attach: (session: MatrixRTCSession, localIdentity: LocalCallIdentity) => void; + detach: () => void; + waitForOwnKey: () => Promise; + getKeys: () => NativeCallEncryptionKeyPayload[]; + setOnKey: (listener: ((key: NativeCallEncryptionKeyPayload) => void) | undefined) => void; +}; + +const toBase64 = (key: Uint8Array): string => { + let binary = ''; + key.forEach((byte) => { + binary += String.fromCharCode(byte); + }); + return btoa(binary); +}; + +export const createNativeCallKeyForwarder = (): NativeCallKeyForwarder => { + let rtcSession: MatrixRTCSession | undefined; + const keys = new Map(); + let localIdentity: LocalCallIdentity | null = null; + let localOutboundIdentity: string | null = null; + let onKey: ((key: NativeCallEncryptionKeyPayload) => void) | undefined; + + let waitResolve: (() => void) | undefined; + let waitReject: ((error: Error) => void) | undefined; + let waitTimeout: ReturnType | undefined; + + const settleWait = (settle: () => void): void => { + waitResolve = undefined; + waitReject = undefined; + if (waitTimeout !== undefined) clearTimeout(waitTimeout); + waitTimeout = undefined; + settle(); + }; + + const hasOwnKey = (): boolean => + localOutboundIdentity !== null && keys.has(localOutboundIdentity); + + const maybeResolveOwnKey = (): void => { + const resolve = waitResolve; + if (resolve && hasOwnKey()) settleWait(resolve); + }; + + const onEncryptionKeyChanged = ( + encryptionKey: Uint8Array, + encryptionKeyIndex: number, + membershipParts: CallMembershipIdentityParts, + rtcBackendIdentity: string + ): void => { + debugLog.debug( + 'call', + `key changed identity=${rtcBackendIdentity} index=${encryptionKeyIndex} ownIdentity=${localOutboundIdentity ?? 'unset'}` + ); + if ( + membershipParts.userId === localIdentity?.userId && + membershipParts.deviceId === localIdentity?.deviceId + ) { + localOutboundIdentity = rtcBackendIdentity; + } + const accepted = keys.get(rtcBackendIdentity); + if (accepted && encryptionKeyIndex <= accepted.keyIndex) return; + + const entry: NativeCallEncryptionKeyPayload = { + identity: rtcBackendIdentity, + keyIndex: encryptionKeyIndex, + key: toBase64(encryptionKey), + }; + keys.set(rtcBackendIdentity, entry); + onKey?.(entry); + maybeResolveOwnKey(); + }; + + const detach = (): void => { + if (rtcSession) { + rtcSession.off(MatrixRTCSessionEvent.EncryptionKeyChanged, onEncryptionKeyChanged); + rtcSession = undefined; + } + keys.clear(); + localIdentity = null; + localOutboundIdentity = null; + onKey = undefined; + const reject = waitReject; + if (reject) settleWait(() => reject(new Error(OWN_KEY_WAIT_CANCELLED_ERROR))); + }; + + const attach = (session: MatrixRTCSession, identity: LocalCallIdentity): void => { + detach(); + rtcSession = session; + localIdentity = identity; + session.on(MatrixRTCSessionEvent.EncryptionKeyChanged, onEncryptionKeyChanged); + session.reemitEncryptionKeys(); + }; + + const waitForOwnKey = (): Promise => { + if (hasOwnKey()) return Promise.resolve(); + return new Promise((resolve, reject) => { + waitResolve = resolve; + waitReject = reject; + waitTimeout = setTimeout( + () => settleWait(() => reject(new Error(OWN_KEY_UNAVAILABLE_ERROR))), + ownKeyWaitTimeoutMs + ); + maybeResolveOwnKey(); + }); + }; + + return { + attach, + detach, + waitForOwnKey, + getKeys: () => [...keys.values()], + setOnKey: (listener) => { + onKey = listener; + }, + }; +}; diff --git a/src/app/features/call/nativeCallManager.test.ts b/src/app/features/call/nativeCallManager.test.ts new file mode 100644 index 000000000..47799a531 --- /dev/null +++ b/src/app/features/call/nativeCallManager.test.ts @@ -0,0 +1,144 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createStore } from 'jotai'; +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import { createNativeCallManager, getNativeCallManager } from './nativeCallManager'; +import type { + NativeCallController, + NativeCallControllerDependencies, + NativeCallStartOptions, +} from './nativeCallController'; +import { nativeCallAtom, type NativeCallSession } from '$state/nativeCall'; +import { callEmbedAtom } from '$state/callEmbed'; +import type { CallEmbed } from '$plugins/call'; +import { resetCallOwnerForTests } from '$state/callOwner'; + +const room = { roomId: '!room:example.org' } as Room; + +const makeClient = (memberships: unknown[] = []) => + ({ + matrixRTC: { + getRoomSession: () => ({ memberships }), + }, + }) as unknown as MatrixClient; + +const makeNativeSession = (lifecycle: NativeCallSession['lifecycle']): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId: room.roomId, + callId: 'call-id', + lifecycle, + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, +}); + +type FakeController = { + controller: NativeCallController; + start: ReturnType Promise>>; + setSession?: (session: NativeCallSession | undefined) => void; +}; + +const makeFakeControllerFactory = () => { + const fake: FakeController = { + controller: undefined as unknown as NativeCallController, + start: vi.fn<(options: NativeCallStartOptions) => Promise>(async () => undefined), + }; + fake.controller = { start: fake.start } as unknown as NativeCallController; + const createController = ( + dependencies: Pick + ): NativeCallController => { + fake.setSession = dependencies.setSession; + return fake.controller; + }; + return { fake, createController }; +}; + +beforeEach(() => { + resetCallOwnerForTests(); +}); + +describe('createNativeCallManager', () => { + it('starts the controller with the computed call options', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + const manager = createNativeCallManager(store, createController); + + manager.start({ + mx: makeClient([{ userId: '@other:example.org' }]), + room, + dm: true, + video: true, + microphone: false, + }); + + expect(fake.start).toHaveBeenCalledWith({ + mx: expect.anything(), + room, + dm: true, + video: true, + microphone: false, + ongoing: true, + }); + }); + + it('defaults audio/direct-message flags', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + const manager = createNativeCallManager(store, createController); + + manager.start({ mx: makeClient(), room }); + + expect(fake.start).toHaveBeenCalledWith( + expect.objectContaining({ dm: false, video: false, microphone: true, ongoing: false }) + ); + }); + + it('publishes controller sessions into the native call atom', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + createNativeCallManager(store, createController); + + const session = makeNativeSession('connected'); + fake.setSession?.(session); + expect(store.get(nativeCallAtom)).toBe(session); + + fake.setSession?.(undefined); + expect(store.get(nativeCallAtom)).toBeUndefined(); + }); + + it('does not start while a native call session is active', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + const manager = createNativeCallManager(store, createController); + + store.set(nativeCallAtom, makeNativeSession('connected')); + manager.start({ mx: makeClient(), room }); + + expect(fake.start).not.toHaveBeenCalled(); + }); + + it('does not start while a failed native session is cleared but Element Call is active', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + const manager = createNativeCallManager(store, createController); + + store.set(callEmbedAtom, { + roomId: '!other:example.org', + dispose: vi.fn<() => void>(), + } as unknown as CallEmbed); + manager.start({ mx: makeClient(), room }); + + expect(fake.start).not.toHaveBeenCalled(); + }); +}); + +describe('getNativeCallManager', () => { + it('reuses one manager per store', () => { + const store = createStore(); + expect(getNativeCallManager(store)).toBe(getNativeCallManager(store)); + }); +}); diff --git a/src/app/features/call/nativeCallManager.ts b/src/app/features/call/nativeCallManager.ts new file mode 100644 index 000000000..22533eaaa --- /dev/null +++ b/src/app/features/call/nativeCallManager.ts @@ -0,0 +1,70 @@ +import type { useStore } from 'jotai'; + +type Store = ReturnType; +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import type { AutoDiscoveryInfo } from '../../cs-api'; +import { callEmbedAtom } from '$state/callEmbed'; +import { isLivekitJsCallActive, livekitJsCallAtom } from '$state/livekitJsCall'; +import { isNativeCallActive, nativeCallAtom } from '$state/nativeCall'; +import { + createNativeCallController, + type NativeCallController, + type NativeCallControllerDependencies, +} from './nativeCallController'; + +export type NativeCallManagerStartOptions = { + mx: MatrixClient; + room: Room; + discovery?: Pick; + dm?: boolean; + video?: boolean; + microphone?: boolean; +}; + +export type NativeCallManager = { + start: (options: NativeCallManagerStartOptions) => void; +}; + +export const createNativeCallManager = ( + store: Pick, + createController: ( + dependencies: Pick + ) => NativeCallController = createNativeCallController +): NativeCallManager => { + const controller = createController({ + setSession: (session) => store.set(nativeCallAtom, session), + }); + + return { + start: ({ mx, room, discovery, dm, video, microphone }) => { + if ( + store.get(callEmbedAtom) || + isLivekitJsCallActive(store.get(livekitJsCallAtom)) || + isNativeCallActive(store.get(nativeCallAtom)) + ) { + return; + } + void controller + .start({ + mx, + room, + discovery, + dm: dm ?? false, + video: video ?? false, + microphone: microphone ?? true, + ongoing: mx.matrixRTC.getRoomSession(room).memberships.length > 0, + }) + .catch(() => undefined); + }, + }; +}; + +const managers = new WeakMap, NativeCallManager>(); + +export const getNativeCallManager = (store: Pick): NativeCallManager => { + const cached = managers.get(store); + if (cached) return cached; + const manager = createNativeCallManager(store); + managers.set(store, manager); + return manager; +}; diff --git a/src/app/features/call/nativeCallProbe.test.ts b/src/app/features/call/nativeCallProbe.test.ts new file mode 100644 index 000000000..3c31ed89a --- /dev/null +++ b/src/app/features/call/nativeCallProbe.test.ts @@ -0,0 +1,67 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { getNativeCallAvailability, resetNativeCallAvailabilityForTests } from './nativeCallProbe'; +import { isMobileTauri } from '$utils/platform'; +import { getNativeCallCapabilities, type NativeCallCapabilities } from './livekitMobileBridge'; + +vi.mock('$utils/platform', () => ({ + isMobileTauri: vi.fn<() => boolean>(), +})); + +vi.mock('./livekitMobileBridge', () => ({ + getNativeCallCapabilities: vi.fn<() => Promise>(), +})); + +const allCapabilities: NativeCallCapabilities = { + supported: true, + microphone: true, + backgroundAudio: true, + nativeRoom: true, + camera: true, + nativeVideoOverlay: false, + callKit: true, +}; + +beforeEach(() => { + resetNativeCallAvailabilityForTests(); + vi.mocked(isMobileTauri).mockReturnValue(false); + vi.mocked(getNativeCallCapabilities).mockResolvedValue(allCapabilities); +}); + +describe('getNativeCallAvailability', () => { + it('is unavailable outside Tauri mobile even with new calls enabled', async () => { + vi.mocked(isMobileTauri).mockReturnValue(false); + + await expect(getNativeCallAvailability(true)).resolves.toBe(false); + expect(getNativeCallCapabilities).not.toHaveBeenCalled(); + }); + + it('is unavailable with new calls disabled on Tauri mobile', async () => { + vi.mocked(isMobileTauri).mockReturnValue(true); + + await expect(getNativeCallAvailability(false)).resolves.toBe(false); + expect(getNativeCallCapabilities).not.toHaveBeenCalled(); + }); + + it('is available on Tauri mobile with new calls enabled and supporting capabilities', async () => { + vi.mocked(isMobileTauri).mockReturnValue(true); + + await expect(getNativeCallAvailability(true)).resolves.toBe(true); + }); + + it('is unavailable when the native plugin does not support calls', async () => { + vi.mocked(isMobileTauri).mockReturnValue(true); + vi.mocked(getNativeCallCapabilities).mockResolvedValue({ + ...allCapabilities, + supported: false, + }); + + await expect(getNativeCallAvailability(true)).resolves.toBe(false); + }); + + it('is unavailable when the capabilities request fails', async () => { + vi.mocked(isMobileTauri).mockReturnValue(true); + vi.mocked(getNativeCallCapabilities).mockRejectedValue(new Error('plugin missing')); + + await expect(getNativeCallAvailability(true)).resolves.toBe(false); + }); +}); diff --git a/src/app/features/call/nativeCallProbe.ts b/src/app/features/call/nativeCallProbe.ts new file mode 100644 index 000000000..83cf6c222 --- /dev/null +++ b/src/app/features/call/nativeCallProbe.ts @@ -0,0 +1,19 @@ +import { isMobileTauri } from '$utils/platform'; +import { getNativeCallCapabilities, type NativeCallCapabilities } from './livekitMobileBridge'; + +const supportsNativeCall = (capabilities: NativeCallCapabilities): boolean => + capabilities.supported && capabilities.microphone; + +let availabilityPromise: Promise | undefined; + +// Native calls are gated solely by the new-call setting; platform and +// capability checks still apply on top of it. +export const getNativeCallAvailability = (newCallsEnabled: boolean): Promise => { + if (!newCallsEnabled || !isMobileTauri()) return Promise.resolve(false); + availabilityPromise ??= getNativeCallCapabilities().then(supportsNativeCall, () => false); + return availabilityPromise; +}; + +export const resetNativeCallAvailabilityForTests = (): void => { + availabilityPromise = undefined; +}; diff --git a/src/app/features/call/useCallSignaling.ts b/src/app/features/call/useCallSignaling.ts index a652d21ab..36dffec86 100644 --- a/src/app/features/call/useCallSignaling.ts +++ b/src/app/features/call/useCallSignaling.ts @@ -1,7 +1,12 @@ import { useCallback, useEffect, useRef } from 'react'; import * as Sentry from '@sentry/react'; import { useAtomValue, useSetAtom, useStore } from 'jotai'; -import type { RoomEventHandlerMap, MatrixEvent, Room } from '$types/matrix-sdk'; +import { + EventType, + type RoomEventHandlerMap, + type MatrixEvent, + type Room, +} from '$types/matrix-sdk'; import { MatrixRTCSessionManagerEvents, RoomEvent } from '$types/matrix-sdk'; import { mDirectAtom } from '$state/mDirectList'; import { @@ -11,6 +16,8 @@ import { mutedCallRoomIdAtom, type IncomingCall, } from '$state/callEmbed'; +import { livekitJsCallAtom } from '$state/livekitJsCall'; +import { nativeCallAtom, selectActiveCallSessionIncludingNative } from '$state/nativeCall'; import { settingsAtom } from '$state/settings'; import { parseIncomingRtcNotification, @@ -46,12 +53,16 @@ import { createDebugLogger } from '$utils/debugLogger'; const debugLog = createDebugLogger('CallSignaling'); const canSenderStartCalls = (room: Room, senderId: string): boolean => - room.currentState?.maySendStateEvent('org.matrix.msc3401.call.member', senderId) ?? false; + room.currentState?.maySendStateEvent(EventType.RTCMembership, senderId) || + room.currentState?.maySendStateEvent(EventType.GroupCallMemberPrefix, senderId) || + false; export function useIncomingCallSignaling() { const mx = useMatrixClient(); const store = useStore(); const callEmbed = useAtomValue(callEmbedAtom); + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const nativeCall = useAtomValue(nativeCallAtom); const mDirects = useAtomValue(mDirectAtom); const settings = useAtomValue(settingsAtom); const incomingCall = useAtomValue(incomingCallAtom); @@ -89,6 +100,8 @@ export function useIncomingCallSignaling() { type SignalingHandlerRefs = { callEmbed: typeof callEmbed; + livekitJsCall: typeof livekitJsCall; + nativeCall: typeof nativeCall; mDirects: typeof mDirects; outgoingRingbackAllowed: boolean; handleIncomingCall: (incoming: IncomingCall) => void; @@ -117,7 +130,7 @@ export function useIncomingCallSignaling() { hasCallBeenActiveRef.current = false; outgoingRingRoomIdRef.current = null; outgoingStartRef.current = null; - }, [callEmbed]); + }, [callEmbed?.roomId, livekitJsCall?.roomId, nativeCall?.roomId]); useEffect(() => { ringtoneManager.syncSources( @@ -147,7 +160,12 @@ export function useIncomingCallSignaling() { const handleOutgoingDecline = useCallback( (decline: OutgoingDeclineEvent) => { - if (!callEmbed || callEmbed.roomId !== decline.roomId) { + const activeCall = selectActiveCallSessionIncludingNative( + callEmbed, + livekitJsCall, + nativeCall + ); + if (!activeCall || activeCall.roomId !== decline.roomId) { return; } @@ -207,8 +225,10 @@ export function useIncomingCallSignaling() { Sentry.metrics.count('sable.call.outgoing.declined', 1); stopOutgoingRing(); - void callEmbed - .hangup() + const hangup = + selectActiveCallSessionIncludingNative(callEmbed, livekitJsCall, nativeCall)?.hangup() ?? + Promise.resolve(); + void hangup .catch((error) => { debugLog.warn('call', 'Failed to hang up after outgoing decline', { roomId: decline.roomId, @@ -219,12 +239,11 @@ export function useIncomingCallSignaling() { .finally(() => { window.setTimeout(() => { const activeEmbed = store.get(callEmbedAtom); - if (activeEmbed !== callEmbed) return; - setCallEmbed(undefined); + if (activeEmbed === callEmbed) setCallEmbed(undefined); }, OUTGOING_DECLINE_EMBED_CLEAR_MS); }); }, - [callEmbed, mDirects, mx, setCallEmbed, stopOutgoingRing, store] + [callEmbed, livekitJsCall, nativeCall, mDirects, mx, setCallEmbed, stopOutgoingRing, store] ); const callAudioAllowed = canPlayCallAudio({ @@ -283,6 +302,8 @@ export function useIncomingCallSignaling() { signalingHandlerRefs.current = { callEmbed, + livekitJsCall, + nativeCall, mDirects, outgoingRingbackAllowed, handleIncomingCall, @@ -421,7 +442,12 @@ export function useIncomingCallSignaling() { if (!senderId || !eventId) return; if (senderId === myUserId) { - if (type === RTC_NOTIFICATION_EVENT_TYPE && handlers().callEmbed?.roomId === room.roomId) { + const activeCall = selectActiveCallSessionIncludingNative( + handlers().callEmbed, + handlers().livekitJsCall, + handlers().nativeCall + ); + if (type === RTC_NOTIFICATION_EVENT_TYPE && activeCall?.roomId === room.roomId) { activeOutgoingNotificationIdRef.current = eventId; } return; @@ -436,8 +462,12 @@ export function useIncomingCallSignaling() { // Only inspect declines for the active outgoing call room. Cleartext declines are // cheap; encrypted events are decrypted only when they might be RTC declines. - const activeEmbed = handlers().callEmbed; - if (!activeEmbed || activeEmbed.roomId !== room.roomId) { + const activeCall = selectActiveCallSessionIncludingNative( + handlers().callEmbed, + handlers().livekitJsCall, + handlers().nativeCall + ); + if (!activeCall || activeCall.roomId !== room.roomId) { return; } if (event.isDecryptionFailure()) { @@ -496,7 +526,11 @@ export function useIncomingCallSignaling() { let outgoingRingTimeoutId: number | undefined; const evaluateOutgoingFallback = () => { - const activeCallRoomId = handlers().callEmbed?.roomId; + const activeCallRoomId = selectActiveCallSessionIncludingNative( + handlers().callEmbed, + handlers().livekitJsCall, + handlers().nativeCall + )?.roomId; const stop = () => { handlers().stopOutgoingRing(); diff --git a/src/app/features/call/useCallStartCapabilities.test.tsx b/src/app/features/call/useCallStartCapabilities.test.tsx new file mode 100644 index 000000000..01e9bb193 --- /dev/null +++ b/src/app/features/call/useCallStartCapabilities.test.tsx @@ -0,0 +1,142 @@ +import type { ReactNode } from 'react'; +import { act, render } from '@testing-library/react'; +import { createStore, Provider as JotaiProvider } from 'jotai'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import type { CallEmbed } from '$plugins/call'; +import { MatrixClientProvider } from '$hooks/useMatrixClient'; +import { AutoDiscoveryInfoProvider } from '$hooks/useAutoDiscoveryInfo'; +import { nativeCallAtom, type NativeCallSession } from '$state/nativeCall'; +import type { CallStartCapabilities } from './callStartCapabilities'; +import { useCallStartCapabilities } from './useCallStartCapabilities'; + +const { webRtcSupportedMock, useCallEmbedMock } = vi.hoisted(() => ({ + webRtcSupportedMock: vi.fn<() => boolean>(() => true), + useCallEmbedMock: vi.fn<() => CallEmbed | undefined>(() => undefined), +})); + +vi.mock('$utils/rtc', () => ({ + webRTCSupported: () => webRtcSupportedMock(), +})); + +vi.mock('$hooks/useCallEmbed', () => ({ + useCallEmbed: () => useCallEmbedMock(), +})); + +const room = { + roomId: '!room:example.org', + currentState: { + maySendStateEvent: () => true, + }, +} as unknown as Room; + +const mx = { + getSafeUserId: () => '@me:example.org', + on: vi.fn<(...args: unknown[]) => void>(), + removeListener: vi.fn<(...args: unknown[]) => void>(), +} as unknown as MatrixClient; + +const makeNativeSession = ( + roomId: string, + lifecycle: NativeCallSession['lifecycle'] +): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId, + callId: 'call-id', + lifecycle, + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, +}); + +type Harness = { + store: ReturnType; + wrapper: ({ children }: { children?: ReactNode }) => ReactNode; + latest: () => CallStartCapabilities; +}; + +const createHarness = (): Harness => { + const store = createStore(); + let current: CallStartCapabilities | undefined; + const Probe = (): null => { + current = useCallStartCapabilities(room); + return null; + }; + const wrapper = ({ children }: { children?: ReactNode }) => ( + + + + {children ?? } + + + + ); + const latest = (): CallStartCapabilities => { + expect(current).toBeDefined(); + return current!; + }; + return { store, wrapper, latest }; +}; + +describe('useCallStartCapabilities', () => { + beforeEach(() => { + webRtcSupportedMock.mockReset().mockReturnValue(true); + useCallEmbedMock.mockReset().mockReturnValue(undefined); + }); + + it('blocks starting in another room while a native call is active there', () => { + const harness = createHarness(); + harness.store.set(nativeCallAtom, makeNativeSession('!other:example.org', 'connected')); + + render({undefined}); + + const capabilities = harness.latest(); + expect(capabilities.inAnotherCall).toBe(true); + expect(capabilities.blockers).toContain('already_in_another_call'); + expect(capabilities.canStart).toBe(false); + }); + + it('stops treating a native session as active once it reaches the error state', () => { + const harness = createHarness(); + render({undefined}); + + act(() => { + harness.store.set(nativeCallAtom, makeNativeSession('!other:example.org', 'starting')); + }); + expect(harness.latest().inAnotherCall).toBe(true); + + act(() => { + harness.store.set(nativeCallAtom, makeNativeSession('!other:example.org', 'error')); + }); + const capabilities = harness.latest(); + expect(capabilities.inAnotherCall).toBe(false); + expect(capabilities.blockers).not.toContain('already_in_another_call'); + expect(capabilities.canStart).toBe(true); + }); + + it('still counts an active Element call embed as an active call', () => { + const harness = createHarness(); + useCallEmbedMock.mockReturnValue({ + roomId: '!other:example.org', + } as CallEmbed); + + render({undefined}); + + const capabilities = harness.latest(); + expect(capabilities.inAnotherCall).toBe(true); + expect(capabilities.blockers).toContain('already_in_another_call'); + }); +}); diff --git a/src/app/features/call/useCallStartCapabilities.ts b/src/app/features/call/useCallStartCapabilities.ts index bad149ca5..beff9f078 100644 --- a/src/app/features/call/useCallStartCapabilities.ts +++ b/src/app/features/call/useCallStartCapabilities.ts @@ -2,6 +2,9 @@ import { useCallback, useMemo } from 'react'; import type { Room } from '$types/matrix-sdk'; import { EventType } from '$types/matrix-sdk'; import { useCallEmbed } from '$hooks/useCallEmbed'; +import { useAtomValue } from 'jotai'; +import { isLivekitJsCallActive, livekitJsCallAtom } from '$state/livekitJsCall'; +import { isNativeCallActive, nativeCallAtom } from '$state/nativeCall'; import { useLivekitSupport } from '$hooks/useLivekitSupport'; import { useMatrixClient } from '$hooks/useMatrixClient'; import { useStateEventCallback } from '$hooks/useStateEventCallback'; @@ -15,10 +18,18 @@ import { export const useCallStartCapabilities = (room: Room): CallStartCapabilities => { const mx = useMatrixClient(); const callEmbed = useCallEmbed(); + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const nativeCall = useAtomValue(nativeCallAtom); const livekitSupported = useLivekitSupport(); const rtcSupported = webRTCSupported(); const myUserId = mx.getSafeUserId(); const [updateCount, forceUpdate] = useForceUpdate(); + // Terminal sessions stay in their atom for display, but they are not active + // calls and must not block starting a later one. + const activeLivekitJsRoomId = isLivekitJsCallActive(livekitJsCall) + ? livekitJsCall?.roomId + : undefined; + const activeNativeCallRoomId = isNativeCallActive(nativeCall) ? nativeCall?.roomId : undefined; useStateEventCallback( mx, @@ -42,9 +53,18 @@ export const useCallStartCapabilities = (room: Room): CallStartCapabilities => { return evaluateCallStartCapabilities({ room, myUserId, - activeCallRoomId: callEmbed?.roomId, + activeCallRoomId: callEmbed?.roomId ?? activeLivekitJsRoomId ?? activeNativeCallRoomId, livekitSupported, rtcSupported, }); - }, [room, myUserId, callEmbed?.roomId, livekitSupported, rtcSupported, updateCount]); + }, [ + room, + myUserId, + callEmbed?.roomId, + activeLivekitJsRoomId, + activeNativeCallRoomId, + livekitSupported, + rtcSupported, + updateCount, + ]); }; diff --git a/src/app/features/settings/experimental/Experimental.test.tsx b/src/app/features/settings/experimental/Experimental.test.tsx new file mode 100644 index 000000000..9a2313184 --- /dev/null +++ b/src/app/features/settings/experimental/Experimental.test.tsx @@ -0,0 +1,94 @@ +import type { ReactNode } from 'react'; +import type * as SettingsModule from '$state/settings'; +import { fireEvent, render, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { Experimental } from './Experimental'; + +const { mockSetNewCallsEnabled, mockUseSetting } = vi.hoisted(() => ({ + mockSetNewCallsEnabled: vi.fn<(value: boolean) => void>(), + mockUseSetting: + vi.fn<(_atom: unknown, key: string) => readonly [boolean, (value: boolean) => void]>(), +})); + +vi.mock('$state/settings', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, settingsAtom: {} }; +}); + +vi.mock('$state/hooks/settings', () => ({ + useSetting: mockUseSetting, +})); + +vi.mock('$components/page', () => ({ + PageContent: ({ children }: { children: ReactNode }) =>
{children}
, + SettingsSectionPage: ({ children }: { children: ReactNode }) =>
{children}
, +})); + +vi.mock('$components/info-card', () => ({ + InfoCard: () =>
Experimental warning
, +})); + +vi.mock('$components/setting-tile', () => ({ + SettingToggle: ({ + title, + description, + focusId, + value, + onChange, + }: { + title: string; + description: ReactNode; + focusId: string; + value: boolean; + onChange: (value: boolean) => void; + }) => ( +
+ {title} + {description} + +
+ ), +})); + +vi.mock('./BandwithSavingEmojis', () => ({ BandwidthSavingEmojis: () => null })); +vi.mock('./MSC4268HistoryShare', () => ({ MSC4268HistoryShare: () => null })); +vi.mock('./MSC4274MediaGalleries', () => ({ MSC4274MediaGalleries: () => null })); + +vi.mock('folds', () => ({ + Box: ({ children }: { children: ReactNode }) =>
{children}
, + Scroll: ({ children }: { children: ReactNode }) =>
{children}
, + Text: ({ children }: { children: ReactNode }) => {children}, +})); + +beforeEach(() => { + mockSetNewCallsEnabled.mockReset(); + mockUseSetting.mockImplementation((_atom: unknown, key: string) => { + if (key === 'newCallsEnabled') return [false, mockSetNewCallsEnabled]; + return [false, vi.fn<() => void>()]; + }); +}); + +describe('Experimental new calls setting', () => { + it('shows one new calls toggle', () => { + render( {}} />); + + expect(screen.getByText('New calls')).toBeInTheDocument(); + expect(screen.getByText('Enable new calls')).toBeInTheDocument(); + expect( + screen.getByText( + 'Uses LiveKit JS on web and desktop, and native LiveKit on supported mobile devices. Element Call remains the fallback.' + ) + ).toBeInTheDocument(); + expect(screen.queryByText(/manual media|connection probe/i)).not.toBeInTheDocument(); + }); + + it('persists the opt-in through the settings hook', () => { + render( {}} />); + + fireEvent.click(screen.getByRole('button', { name: 'new-calls' })); + + expect(mockSetNewCallsEnabled).toHaveBeenCalledWith(true); + }); +}); diff --git a/src/app/features/settings/experimental/Experimental.tsx b/src/app/features/settings/experimental/Experimental.tsx index 5d38fb06b..b89ec3c6e 100644 --- a/src/app/features/settings/experimental/Experimental.tsx +++ b/src/app/features/settings/experimental/Experimental.tsx @@ -29,6 +29,23 @@ function PersonaToggle() { ); } +function NewCallsToggle() { + const [newCallsEnabled, setNewCallsEnabled] = useSetting(settingsAtom, 'newCallsEnabled'); + + return ( + + New calls + + + ); +} + type ExperimentalProps = { requestBack?: () => void; requestClose: () => void; @@ -56,6 +73,7 @@ export function Experimental({ requestBack, requestClose }: Readonly + diff --git a/src/app/features/settings/settingsLink.ts b/src/app/features/settings/settingsLink.ts index ef4d0d1d9..1a13b3b70 100644 --- a/src/app/features/settings/settingsLink.ts +++ b/src/app/features/settings/settingsLink.ts @@ -234,6 +234,7 @@ export const settingsLinkFocusIdsBySection: Record { return invoke('activate_call_audio_session'); } +export async function buildDiagnosticsArchive(params: types.BuildDiagnosticsArchiveParams): Promise { + return invoke('build_diagnostics_archive', params); +} + export async function clearMediaSession(): Promise { return invoke('clear_media_session'); } @@ -66,6 +70,10 @@ export async function saveDownload(params: types.SaveDownloadParams): Promise { + return invoke('save_media_to_photos', params); +} + export async function setMediaEncryption(params: types.SetMediaEncryptionParams): Promise { return invoke('set_media_encryption', params); } diff --git a/src/app/generated/tauri/events.ts b/src/app/generated/tauri/events.ts index 5d4703b02..1d611b14e 100644 --- a/src/app/generated/tauri/events.ts +++ b/src/app/generated/tauri/events.ts @@ -1,7 +1,7 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-29T09:24:56.415138+00:00 + * Generated at: 2026-07-30T16:36:15.051835+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate diff --git a/src/app/generated/tauri/index.ts b/src/app/generated/tauri/index.ts index c15cf28d9..e62a83936 100644 --- a/src/app/generated/tauri/index.ts +++ b/src/app/generated/tauri/index.ts @@ -1,7 +1,7 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-29T09:24:56.415651+00:00 + * Generated at: 2026-07-30T16:36:15.052085+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate diff --git a/src/app/generated/tauri/types.ts b/src/app/generated/tauri/types.ts index 83aef83d8..9a11ec402 100644 --- a/src/app/generated/tauri/types.ts +++ b/src/app/generated/tauri/types.ts @@ -1,7 +1,7 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-29T09:24:56.409726+00:00 + * Generated at: 2026-07-30T16:36:15.048780+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate @@ -73,6 +73,11 @@ export interface AbortNativeUploadParams { [key: string]: unknown; } +export interface BuildDiagnosticsArchiveParams { + frontendLogs?: string | null; + [key: string]: unknown; +} + export interface ExportDiagnosticsParams { frontendLogs?: string | null; [key: string]: unknown; @@ -108,6 +113,13 @@ export interface SaveDownloadParams { [key: string]: unknown; } +export interface SaveMediaToPhotosParams { + bytes: number[]; + filename: string; + mimeType: string; + [key: string]: unknown; +} + export interface SetMediaEncryptionParams { url: string; key: string; @@ -171,3 +183,4 @@ export interface UploadWriteChunkParams { chunk: string; [key: string]: unknown; } + diff --git a/src/app/hooks/useAutoJoinCall.ts b/src/app/hooks/useAutoJoinCall.ts index 0f95d9485..c4804cb96 100644 --- a/src/app/hooks/useAutoJoinCall.ts +++ b/src/app/hooks/useAutoJoinCall.ts @@ -26,6 +26,8 @@ export function useAutoJoinCall() { microphone: callPreferences.microphone, video: autoJoinIntent.video, sound: callPreferences.sound, + audioDeviceId: callPreferences.audioDeviceId, + videoDeviceId: callPreferences.videoDeviceId, }); setAutoJoinIntent(null); } @@ -38,6 +40,8 @@ export function useAutoJoinCall() { mDirects, callPreferences.microphone, callPreferences.sound, + callPreferences.audioDeviceId, + callPreferences.videoDeviceId, startDirectCall, startRoomCall, ]); diff --git a/src/app/hooks/useCallEmbed.ts b/src/app/hooks/useCallEmbed.ts index c55cfe831..a3a4721f5 100644 --- a/src/app/hooks/useCallEmbed.ts +++ b/src/app/hooks/useCallEmbed.ts @@ -1,7 +1,7 @@ import type { RefObject } from 'react'; import { createContext, useCallback, useContext, useEffect, useState } from 'react'; import type { MatrixClient, Room } from '$types/matrix-sdk'; -import { useSetAtom } from 'jotai'; +import { useSetAtom, useStore } from 'jotai'; import * as Sentry from '@sentry/react'; import type { ElementCallThemeKind } from '../plugins/call'; import { CallEmbed, ElementWidgetActions, useClientWidgetApiEvent } from '../plugins/call'; @@ -15,6 +15,14 @@ import type { CallPreferences } from '../state/callPreferences'; import { createDebugLogger } from '../utils/debugLogger'; import { useClientConfig } from './useClientConfig'; import { callEmbedStartErrorAtom } from '$state/callEmbed'; +import { settingsAtom } from '$state/settings'; +import { useSetting } from '$state/hooks/settings'; +import { acquireCallOwner } from '$state/callOwner'; +import { selectCallStartOwner } from '$features/call/callStartSelection'; +import { useLivekitJsCallManager } from '$features/call/livekitJsCallManager'; +import { getNativeCallAvailability } from '$features/call/nativeCallProbe'; +import { getNativeCallManager } from '$features/call/nativeCallManager'; +import { useAutoDiscoveryInfo } from './useAutoDiscoveryInfo'; const debugLog = createDebugLogger('useCallEmbed'); @@ -66,12 +74,54 @@ export const useCallStart = (dm = false) => { const setCallEmbed = useSetAtom(callEmbedAtom); const setCallEmbedStartError = useSetAtom(callEmbedStartErrorAtom); const callEmbedRef = useCallEmbedRef(); + const store = useStore(); + const discovery = useAutoDiscoveryInfo(); + const [newCallsEnabled] = useSetting(settingsAtom, 'newCallsEnabled'); + const livekitJsCallManager = useLivekitJsCallManager(); const startCall = useCallback( (room: Room, pref?: CallPreferences) => { + if (newCallsEnabled) { + if (!livekitJsCallManager) { + throw new Error('LiveKit JS call manager is not provided!'); + } + // Resolved rather than cached in state so the first tap cannot race the + // native capability probe and fall through to the JS backend. + void getNativeCallAvailability(true).then((nativeCallAvailable) => { + if (selectCallStartOwner({ newCallsEnabled, nativeCallAvailable }) === 'livekit-mobile') { + getNativeCallManager(store).start({ + mx, + room, + discovery, + dm, + video: pref?.video, + microphone: pref?.microphone, + }); + return; + } + livekitJsCallManager.start({ + room, + dm, + video: pref?.video, + microphone: pref?.microphone, + sound: pref?.sound, + audioDeviceId: pref?.audioDeviceId, + videoDeviceId: pref?.videoDeviceId, + }); + }); + return; + } + const ownerLease = acquireCallOwner('element', room.roomId); + if (!ownerLease) { + debugLog.warn('call', 'Failed to start call: another call is already active', { + roomId: room.roomId, + }); + return; + } const container = callEmbedRef.current; if (!container) { - debugLog.error('call', 'Failed to start call — no embed container', { + ownerLease.release(); + debugLog.error('call', 'Failed to start call: no embed container', { roomId: room.roomId, }); Sentry.metrics.count('sable.call.start.error', 1, { @@ -96,6 +146,7 @@ export const useCallStart = (dm = false) => { ); setCallEmbed(callEmbed); } catch (err) { + ownerLease.release(); debugLog.error('call', 'Call embed creation failed', { roomId: room.roomId, error: err instanceof Error ? err.message : String(err), @@ -106,7 +157,19 @@ export const useCallStart = (dm = false) => { throw err; } }, - [mx, dm, theme, setCallEmbed, callEmbedRef, clientConfig.elementCallUrl, setCallEmbedStartError] + [ + mx, + dm, + theme, + setCallEmbed, + callEmbedRef, + store, + discovery, + clientConfig.elementCallUrl, + setCallEmbedStartError, + newCallsEnabled, + livekitJsCallManager, + ] ); return startCall; diff --git a/src/app/pages/CallStatusRenderer.test.tsx b/src/app/pages/CallStatusRenderer.test.tsx new file mode 100644 index 000000000..f7b7c2723 --- /dev/null +++ b/src/app/pages/CallStatusRenderer.test.tsx @@ -0,0 +1,149 @@ +import { render, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createStore, Provider } from 'jotai'; +import type { ReactNode } from 'react'; +import { livekitJsCallAtom, type LivekitJsCallSession } from '../state/livekitJsCall'; +import { nativeCallAtom, type NativeCallSession } from '../state/nativeCall'; +import { CallStatusRenderer } from './CallStatusRenderer'; + +const mocks = vi.hoisted(() => ({ + callEmbed: undefined as { roomId: string } | undefined, + selectedRoom: undefined as string | undefined, + screenSize: 'Desktop', +})); + +vi.mock('../hooks/useCallEmbed', () => ({ useCallEmbed: () => mocks.callEmbed })); +vi.mock('../hooks/router/useSelectedRoom', () => ({ + useSelectedRoom: () => mocks.selectedRoom, +})); +vi.mock('../hooks/useScreenSize', () => ({ + ScreenSize: { Mobile: 'Mobile', Desktop: 'Desktop' }, + useScreenSizeContext: () => mocks.screenSize, +})); +vi.mock('../features/call-status', () => ({ + CallStatus: () =>
, +})); +vi.mock('../features/call-status/LivekitCallStatus', () => ({ + LivekitCallStatus: () =>
, +})); +vi.mock('../features/call-status/NativeCallStatus', () => ({ + NativeCallStatus: () =>
, +})); + +const session = (lifecycle: LivekitJsCallSession['lifecycle']): LivekitJsCallSession => ({ + roomId: '!room:example.org', + lifecycle, + failure: null, + e2eeReady: true, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, +}); + +const renderWith = (children: ReactNode, store = createStore()) => + render({children}); + +beforeEach(() => { + mocks.callEmbed = undefined; + mocks.selectedRoom = undefined; + mocks.screenSize = 'Desktop'; +}); + +describe('CallStatusRenderer', () => { + it('shows the status bar during a LiveKit call', () => { + const store = createStore(); + store.set(livekitJsCallAtom, session('active')); + + renderWith(, store); + + expect(screen.getByTestId('livekit-call-status')).toBeInTheDocument(); + }); + + it('keeps showing it while the call is still connecting', () => { + const store = createStore(); + store.set(livekitJsCallAtom, session('joining-matrix')); + + renderWith(, store); + + expect(screen.getByTestId('livekit-call-status')).toBeInTheDocument(); + }); + + it('hides it once the call has failed', () => { + const store = createStore(); + store.set(livekitJsCallAtom, session('failed')); + + renderWith(, store); + + expect(screen.queryByTestId('livekit-call-status')).not.toBeInTheDocument(); + }); + + it('hides it on mobile while the call room is open, as the embed path does', () => { + mocks.screenSize = 'Mobile'; + mocks.selectedRoom = '!room:example.org'; + const store = createStore(); + store.set(livekitJsCallAtom, session('active')); + + renderWith(, store); + + expect(screen.queryByTestId('livekit-call-status')).not.toBeInTheDocument(); + }); + + it('prefers the Element Call bar when an embed is active', () => { + mocks.callEmbed = { roomId: '!other:example.org' }; + const store = createStore(); + store.set(livekitJsCallAtom, session('active')); + + renderWith(, store); + + expect(screen.getByTestId('element-call-status')).toBeInTheDocument(); + expect(screen.queryByTestId('livekit-call-status')).not.toBeInTheDocument(); + }); + + it('shows the status bar during a native call', () => { + const store = createStore(); + store.set(nativeCallAtom, { + backend: 'livekit-mobile', + roomId: '!room:example.org', + callId: 'call-id', + lifecycle: 'connected', + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, + } satisfies NativeCallSession); + + renderWith(, store); + + expect(screen.getByTestId('native-call-status')).toBeInTheDocument(); + }); + + it('hides the native bar once the call has failed', () => { + const store = createStore(); + store.set(nativeCallAtom, { + backend: 'livekit-mobile', + roomId: '!room:example.org', + callId: 'call-id', + lifecycle: 'error', + microphoneEnabled: false, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, + } satisfies NativeCallSession); + + renderWith(, store); + + expect(screen.queryByTestId('native-call-status')).not.toBeInTheDocument(); + }); + + it('renders nothing with no call at all', () => { + const { container } = renderWith(); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/src/app/pages/CallStatusRenderer.tsx b/src/app/pages/CallStatusRenderer.tsx index d0fb4732e..dbd76bde7 100644 --- a/src/app/pages/CallStatusRenderer.tsx +++ b/src/app/pages/CallStatusRenderer.tsx @@ -1,17 +1,36 @@ +import { useAtomValue } from 'jotai'; import { useCallEmbed } from '../hooks/useCallEmbed'; import { CallStatus } from '../features/call-status'; +import { LivekitCallStatus } from '../features/call-status/LivekitCallStatus'; +import { NativeCallStatus } from '../features/call-status/NativeCallStatus'; +import { isLivekitJsCallActive, livekitJsCallAtom } from '../state/livekitJsCall'; +import { isNativeCallActive, nativeCallAtom } from '../state/nativeCall'; import { useSelectedRoom } from '../hooks/router/useSelectedRoom'; import { ScreenSize, useScreenSizeContext } from '../hooks/useScreenSize'; export function CallStatusRenderer() { const callEmbed = useCallEmbed(); + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const nativeCall = useAtomValue(nativeCallAtom); const selectedRoom = useSelectedRoom(); const screenSize = useScreenSizeContext(); + const mobileInCallRoom = screenSize === ScreenSize.Mobile; - if (!callEmbed) return null; + if (callEmbed) { + if (mobileInCallRoom && callEmbed.roomId === selectedRoom) return null; + return ; + } - if (screenSize === ScreenSize.Mobile && callEmbed.roomId === selectedRoom) return null; + if (isLivekitJsCallActive(livekitJsCall) && livekitJsCall) { + if (mobileInCallRoom && livekitJsCall.roomId === selectedRoom) return null; + return ; + } - return ; + if (isNativeCallActive(nativeCall) && nativeCall) { + if (mobileInCallRoom && nativeCall.roomId === selectedRoom) return null; + return ; + } + + return null; } diff --git a/src/app/state/callEmbed.ts b/src/app/state/callEmbed.ts index 3dcc6c745..6a8112454 100644 --- a/src/app/state/callEmbed.ts +++ b/src/app/state/callEmbed.ts @@ -2,6 +2,7 @@ import { atom } from 'jotai'; import * as Sentry from '@sentry/react'; import type { CallEmbed } from '../plugins/call'; import type { CallEmbedStartError } from '$plugins/call/callEmbedError'; +import { releaseCallOwner } from './callOwner'; const baseCallEmbedAtom = atom(undefined); const baseCallEmbedStartErrorAtom = atom(null); @@ -25,6 +26,7 @@ export const callEmbedAtom = atom resetCallOwnerForTests()); + +describe('call owner guard', () => { + it('allows one owner and rejects competing owners until release', () => { + const element = acquireCallOwner('element', '!room:example.org'); + + expect(element).toBeDefined(); + expect(acquireCallOwner('livekit-js', '!room:example.org')).toBeUndefined(); + expect(acquireCallOwner('livekit-mobile', '!room:example.org')).toBeUndefined(); + expect(getActiveCallOwner()).toMatchObject({ kind: 'element' }); + + element?.release(); + expect(acquireCallOwner('livekit-js', '!room:example.org')).toBeDefined(); + }); + + it('blocks competing owners while the native transport owns the call', () => { + const native = acquireCallOwner('livekit-mobile', '!room:example.org'); + + expect(native).toBeDefined(); + expect(acquireCallOwner('element', '!room:example.org')).toBeUndefined(); + expect(acquireCallOwner('livekit-js', '!room:example.org')).toBeUndefined(); + expect(getActiveCallOwner()).toMatchObject({ kind: 'livekit-mobile' }); + + native?.release(); + expect(acquireCallOwner('element', '!room:example.org')).toBeDefined(); + }); + + it('makes release idempotent and cannot release a replacement lease', () => { + const first = acquireCallOwner('element', '!room:example.org'); + first?.release(); + const second = acquireCallOwner('livekit-js', '!room:example.org'); + + first?.release(); + expect(getActiveCallOwner()).toMatchObject({ kind: 'livekit-js' }); + second?.release(); + expect(getActiveCallOwner()).toBeUndefined(); + }); +}); diff --git a/src/app/state/callOwner.ts b/src/app/state/callOwner.ts new file mode 100644 index 000000000..a6c95fac7 --- /dev/null +++ b/src/app/state/callOwner.ts @@ -0,0 +1,40 @@ +export type CallOwnerKind = 'element' | 'livekit-js' | 'livekit-mobile'; + +export type CallOwnerLease = { + kind: CallOwnerKind; + roomId: string; + release: () => void; +}; + +let activeOwner: CallOwnerLease | undefined; + +export const acquireCallOwner = ( + kind: CallOwnerKind, + roomId: string +): CallOwnerLease | undefined => { + if (activeOwner) return undefined; + + let released = false; + const lease: CallOwnerLease = { + kind, + roomId, + release: () => { + if (released || activeOwner !== lease) return; + released = true; + activeOwner = undefined; + }, + }; + activeOwner = lease; + return lease; +}; + +export const getActiveCallOwner = (): Pick | undefined => + activeOwner; + +export const releaseCallOwner = (kind: CallOwnerKind, roomId: string): void => { + if (activeOwner?.kind === kind && activeOwner.roomId === roomId) activeOwner.release(); +}; + +export const resetCallOwnerForTests = (): void => { + activeOwner = undefined; +}; diff --git a/src/app/state/callPreferences.ts b/src/app/state/callPreferences.ts index db452f0b5..3c082b2f0 100644 --- a/src/app/state/callPreferences.ts +++ b/src/app/state/callPreferences.ts @@ -9,6 +9,8 @@ export type CallPreferences = { microphone: boolean; video: boolean; sound: boolean; + audioDeviceId?: string; + videoDeviceId?: string; }; const CALL_PREFERENCES = 'callPreferences'; diff --git a/src/app/state/hooks/callPreferences.ts b/src/app/state/hooks/callPreferences.ts index 3bc5f20ce..5350fa06a 100644 --- a/src/app/state/hooks/callPreferences.ts +++ b/src/app/state/hooks/callPreferences.ts @@ -18,6 +18,8 @@ export const useCallPreferences = (): CallPreferences & { toggleMicrophone: () => void; toggleVideo: () => void; toggleSound: () => void; + setAudioDeviceId: (deviceId: string) => void; + setVideoDeviceId: (deviceId: string) => void; setPreferences: (prefs: CallPreferences) => void; } => { const callPrefAtom = useCallPreferencesAtom(); @@ -27,8 +29,8 @@ export const useCallPreferences = (): CallPreferences & { const microphone = !pref.microphone; setPref({ + ...pref, microphone, - video: pref.video, sound: !pref.sound && microphone ? true : pref.sound, }); }, [setPref, pref]); @@ -36,25 +38,33 @@ export const useCallPreferences = (): CallPreferences & { const toggleVideo = useCallback(() => { const video = !pref.video; - setPref({ - microphone: pref.microphone, - video, - sound: pref.sound, - }); + setPref({ ...pref, video }); }, [setPref, pref]); const toggleSound = useCallback(() => { const sound = !pref.sound; setPref({ + ...pref, microphone: !sound ? false : pref.microphone, - video: pref.video, sound, }); }, [setPref, pref]); + const setAudioDeviceId = useCallback( + (audioDeviceId: string) => setPref({ ...pref, audioDeviceId }), + [setPref, pref] + ); + + const setVideoDeviceId = useCallback( + (videoDeviceId: string) => setPref({ ...pref, videoDeviceId }), + [setPref, pref] + ); + return { ...pref, + setAudioDeviceId, + setVideoDeviceId, toggleMicrophone, toggleVideo, toggleSound, diff --git a/src/app/state/livekitJsCall.test.ts b/src/app/state/livekitJsCall.test.ts new file mode 100644 index 000000000..47f44cfab --- /dev/null +++ b/src/app/state/livekitJsCall.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { selectActiveCallSession } from './livekitJsCall'; +import { acquireCallOwner, getActiveCallOwner, resetCallOwnerForTests } from './callOwner'; + +describe('selectActiveCallSession', () => { + it('selects the JS owner when Element Call is absent', () => { + const livekitSession = { + roomId: '!room:example.org', + lifecycle: 'active' as const, + failure: null, + e2eeReady: true, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, + }; + + expect(selectActiveCallSession(undefined, livekitSession)).toBe(livekitSession); + }); + + it('preserves Element Call precedence over the JS owner', () => { + const element = { roomId: '!element:example.org' }; + const livekit = { + roomId: '!livekit:example.org', + lifecycle: 'active' as const, + failure: null, + e2eeReady: true, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, + }; + + expect(selectActiveCallSession(element, livekit)).toBe(element); + }); + + it('routes past a failed JS session to Element Call and releases its lease', async () => { + resetCallOwnerForTests(); + const lease = acquireCallOwner('element', '!room:example.org'); + const element = { + roomId: '!room:example.org', + hangup: async () => lease?.release(), + }; + const livekit = { + roomId: '!room:example.org', + lifecycle: 'failed' as const, + failure: 'setup-failed' as const, + e2eeReady: false, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, + }; + + await selectActiveCallSession(element, livekit)?.hangup(); + + expect(getActiveCallOwner()).toBeUndefined(); + }); +}); diff --git a/src/app/state/livekitJsCall.ts b/src/app/state/livekitJsCall.ts new file mode 100644 index 000000000..4d73d7c8f --- /dev/null +++ b/src/app/state/livekitJsCall.ts @@ -0,0 +1,51 @@ +import { atom } from 'jotai'; +import type { + LivekitJsControllerFailure, + LivekitJsControllerLifecycle, +} from '$features/call/livekitJsController'; +import type { Room as LivekitRoom } from 'livekit-client'; + +export type LivekitJsCallMedia = { + microphone: boolean; + camera: boolean; + sound: boolean; + audioDeviceId?: string; + videoDeviceId?: string; +}; + +export type LivekitJsCallSession = { + roomId: string; + /** What the user chose on the prescreen; applied once E2EE is ready. */ + initialMedia: LivekitJsCallMedia; + lifecycle: LivekitJsControllerLifecycle; + failure: LivekitJsControllerFailure | null; + room?: LivekitRoom; + e2eeReady: boolean; + hangup: () => Promise; +}; + +export const livekitJsCallAtom = atom(undefined); + +export const livekitJsCallSoundAtom = atom(true); + +/** + * Whether `initialMedia` has been published for the current call. The surface + * unmounts when the user navigates out of the room while the call keeps + * running, so this has to outlive it or coming back re-applies the prescreen + * choice and undoes anything toggled from the call bar. + */ +export const livekitJsCallInitialMediaAppliedAtom = atom(false); + +export const isLivekitJsCallActive = (session: LivekitJsCallSession | undefined): boolean => + session?.lifecycle !== undefined && + session.lifecycle !== 'idle' && + session.lifecycle !== 'failed'; + +export const selectActiveCallSession = ( + elementCall: Element | undefined, + livekitJsCall: LivekitJsCallSession | undefined +): Element | LivekitJsCallSession | undefined => { + if (elementCall) return elementCall; + if (isLivekitJsCallActive(livekitJsCall)) return livekitJsCall; + return undefined; +}; diff --git a/src/app/state/nativeCall.test.ts b/src/app/state/nativeCall.test.ts new file mode 100644 index 000000000..664965d1f --- /dev/null +++ b/src/app/state/nativeCall.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import { + isNativeCallActive, + selectActiveCallSessionIncludingNative, + type NativeCallSession, +} from './nativeCall'; +import type { LivekitJsCallSession } from './livekitJsCall'; + +const makeNativeSession = (lifecycle: NativeCallSession['lifecycle']): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId: '!room:example.org', + callId: 'call-id', + lifecycle, + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, +}); + +const livekitSession: LivekitJsCallSession = { + roomId: '!room:example.org', + lifecycle: 'active', + failure: null, + e2eeReady: true, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, +}; + +describe('isNativeCallActive', () => { + it('treats missing or errored sessions as inactive', () => { + expect(isNativeCallActive(undefined)).toBe(false); + expect(isNativeCallActive(makeNativeSession('error'))).toBe(false); + expect(isNativeCallActive(makeNativeSession('connected'))).toBe(true); + }); +}); + +describe('selectActiveCallSessionIncludingNative', () => { + it('prefers Element Call and LiveKit JS over the native session', () => { + const element = { roomId: '!room:example.org' }; + const native = makeNativeSession('connected'); + + expect(selectActiveCallSessionIncludingNative(element, livekitSession, native)).toBe(element); + expect(selectActiveCallSessionIncludingNative(undefined, livekitSession, native)).toBe( + livekitSession + ); + }); + + it('returns the native session only when nothing else is active', () => { + const native = makeNativeSession('connected'); + expect(selectActiveCallSessionIncludingNative(undefined, undefined, native)).toBe(native); + expect( + selectActiveCallSessionIncludingNative(undefined, undefined, makeNativeSession('error')) + ).toBeUndefined(); + expect(selectActiveCallSessionIncludingNative(undefined, undefined, undefined)).toBeUndefined(); + }); +}); diff --git a/src/app/state/nativeCall.ts b/src/app/state/nativeCall.ts new file mode 100644 index 000000000..459006160 --- /dev/null +++ b/src/app/state/nativeCall.ts @@ -0,0 +1,43 @@ +import { atom } from 'jotai'; +import type { NativeCallAudioRoute } from '$features/call/livekitMobileBridge'; +import { selectActiveCallSession, type LivekitJsCallSession } from './livekitJsCall'; + +export type NativeCallBackend = 'livekit-mobile'; + +export type NativeCallLifecycle = + | 'starting' + | 'connecting' + | 'connected' + | 'reconnecting' + | 'error'; + +export type NativeCallSession = { + backend: NativeCallBackend; + roomId: string; + callId: string; + lifecycle: NativeCallLifecycle; + error?: string; + microphoneEnabled: boolean; + cameraEnabled: boolean; + setMicrophoneEnabled: (enabled: boolean) => Promise; + setCameraEnabled: (enabled: boolean) => Promise; + switchCamera: () => Promise; + listAudioRoutes: () => Promise; + selectAudioRoute: (routeId: string) => Promise; + hangup: () => Promise; +}; + +export const nativeCallAtom = atom(undefined); + +export const isNativeCallActive = (session: NativeCallSession | undefined): boolean => + session?.lifecycle !== undefined && session.lifecycle !== 'error'; + +export const selectActiveCallSessionIncludingNative = ( + elementCall: Element | undefined, + livekitJsCall: LivekitJsCallSession | undefined, + nativeCall: NativeCallSession | undefined +): Element | LivekitJsCallSession | NativeCallSession | undefined => { + const selected = selectActiveCallSession(elementCall, livekitJsCall); + if (selected) return selected; + return isNativeCallActive(nativeCall) ? nativeCall : undefined; +}; diff --git a/src/app/state/settings.defaults.test.ts b/src/app/state/settings.defaults.test.ts index 2b798084d..c2d84bf66 100644 --- a/src/app/state/settings.defaults.test.ts +++ b/src/app/state/settings.defaults.test.ts @@ -12,6 +12,46 @@ beforeEach(() => { }); describe('mergePersistedSettings', () => { + it('defaults new calls off and persists the opt-in', () => { + expect(defaultSettings.newCallsEnabled).toBe(false); + + localStorage.setItem('settings', JSON.stringify({ newCallsEnabled: true })); + expect(mergePersistedSettings(localStorage.getItem('settings'), {}).newCallsEnabled).toBe(true); + }); + + it('enables new calls when either legacy experimental setting was on', () => { + localStorage.setItem('settings', JSON.stringify({ livekitJsCallsEnabled: true })); + const merged = mergePersistedSettings(localStorage.getItem('settings'), {}); + expect(merged.newCallsEnabled).toBe(true); + expect(merged).not.toHaveProperty('livekitJsCallsEnabled'); + + localStorage.setItem('settings', JSON.stringify({ livekitJsMediaTestEnabled: true })); + const mergedMedia = mergePersistedSettings(localStorage.getItem('settings'), {}); + expect(mergedMedia.newCallsEnabled).toBe(true); + expect(mergedMedia).not.toHaveProperty('livekitJsMediaTestEnabled'); + }); + + it('keeps new calls off when both legacy settings were off or absent', () => { + localStorage.setItem( + 'settings', + JSON.stringify({ livekitJsCallsEnabled: false, livekitJsMediaTestEnabled: false }) + ); + const merged = mergePersistedSettings(localStorage.getItem('settings'), {}); + expect(merged.newCallsEnabled).toBe(false); + expect(merged).not.toHaveProperty('livekitJsCallsEnabled'); + expect(merged).not.toHaveProperty('livekitJsMediaTestEnabled'); + }); + + it('does not override an explicit new calls preference with legacy keys', () => { + localStorage.setItem( + 'settings', + JSON.stringify({ newCallsEnabled: false, livekitJsCallsEnabled: true }) + ); + const merged = mergePersistedSettings(localStorage.getItem('settings'), {}); + expect(merged.newCallsEnabled).toBe(false); + expect(merged).not.toHaveProperty('livekitJsCallsEnabled'); + }); + it('layers deployer defaults over code defaults when localStorage is empty', () => { const merged = mergePersistedSettings(null, { twitterEmoji: false }); expect(merged.twitterEmoji).toBe(false); @@ -88,6 +128,19 @@ describe('sanitizeSettingsDefaults', () => { }); }); + it('accepts the new calls setting', () => { + expect(sanitizeSettingsDefaults({ newCallsEnabled: true })).toEqual({ + newCallsEnabled: true, + }); + expect(sanitizeSettingsDefaults({ newCallsEnabled: 'yes' })).toEqual({}); + }); + + it('drops the legacy LiveKit JS experimental settings', () => { + expect( + sanitizeSettingsDefaults({ livekitJsCallsEnabled: true, livekitJsMediaTestEnabled: true }) + ).toEqual({}); + }); + it('drops unknown keys', () => { expect(sanitizeSettingsDefaults({ notARealSetting: true, hour24Clock: true })).toEqual({ hour24Clock: true, diff --git a/src/app/state/settings.ts b/src/app/state/settings.ts index 04b609745..f5c34e0ef 100644 --- a/src/app/state/settings.ts +++ b/src/app/state/settings.ts @@ -180,6 +180,7 @@ export interface Settings { developerTools: boolean; enableMSC4268CMD: boolean; enableMediaGalleries: boolean; + newCallsEnabled: boolean; settingsSyncEnabled: boolean; // Cosmetics! @@ -337,6 +338,7 @@ export const defaultSettings: Settings = { enableMSC4268CMD: false, enableMediaGalleries: false, + newCallsEnabled: false, // Push notifications (SW/Sygnal): default on for mobile, opt-in on desktop. // In-app pill banner: default on for mobile (primary foreground alert), opt-in on desktop. @@ -520,6 +522,16 @@ function migrateParsedLocalStorage(parsed: Record): void { delete parsed.themeChatPreviewAnyUrl; delete parsed.themeChatPreviewApprovedCatalogOnly; + // Consolidate the legacy LiveKit JS experiments into the single new-call gate. + if ( + typeof parsed.newCallsEnabled !== 'boolean' && + (parsed.livekitJsCallsEnabled === true || parsed.livekitJsMediaTestEnabled === true) + ) { + parsed.newCallsEnabled = true; + } + delete parsed.livekitJsCallsEnabled; + delete parsed.livekitJsMediaTestEnabled; + if (typeof parsed.callRingtoneVolume === 'number' && Number.isFinite(parsed.callRingtoneVolume)) { parsed.callRingtoneVolume = clampPercent(parsed.callRingtoneVolume); } diff --git a/src/types/matrix-sdk.ts b/src/types/matrix-sdk.ts index f1bea4e09..7c2ec0653 100644 --- a/src/types/matrix-sdk.ts +++ b/src/types/matrix-sdk.ts @@ -55,5 +55,8 @@ export * from 'matrix-js-sdk/lib/common-crypto/CryptoBackend'; export * from 'matrix-js-sdk/lib/matrixrtc/CallMembership'; export * from 'matrix-js-sdk/lib/matrixrtc/MatrixRTCSession'; export * from 'matrix-js-sdk/lib/matrixrtc/MatrixRTCSessionManager'; +export { type CallMembershipIdentityParts } from 'matrix-js-sdk/lib/matrixrtc/EncryptionManager'; +export { type LivekitTransportConfig } from 'matrix-js-sdk/lib/matrixrtc/LivekitTransport'; +export { type Transport } from 'matrix-js-sdk/lib/matrixrtc/types'; export * from 'matrix-js-sdk/lib/models/thread';