From 58e51d6c1ad9c82d59452fc9ffe6e1172c4f79ea Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 14:42:07 -0800 Subject: [PATCH 01/20] chore(quest): claim quest/m0/browser-source.md Quest-Claim: 8a0da92c421c5b81 From d7cf5c4263562dfd66fa2159311c7d1573a77347 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 15:09:21 -0800 Subject: [PATCH 02/20] wip: browser sources through a CEF helper process --- Cargo.lock | 325 ++++++++++++++++ flake.nix | 77 ++++ rs/moq-scene/Cargo.toml | 14 + rs/moq-scene/src/browser/host.rs | 540 +++++++++++++++++++++++++++ rs/moq-scene/src/browser/import.rs | 59 +++ rs/moq-scene/src/browser/mod.rs | 247 ++++++++++++ rs/moq-scene/src/browser/protocol.rs | 283 ++++++++++++++ rs/moq-scene/src/browser/run.rs | 141 +++++++ rs/moq-scene/src/collection.rs | 32 +- rs/moq-scene/src/compositor.rs | 30 +- rs/moq-scene/src/lib.rs | 8 +- rs/moq-scene/src/source.rs | 62 ++- rs/moq-studio/Cargo.toml | 4 + rs/moq-studio/build.rs | 17 + rs/moq-studio/src/engine.rs | 16 + rs/moq-studio/src/interact.rs | 414 ++++++++++++++++++++ rs/moq-studio/src/main.rs | 13 +- rs/moq-studio/src/ui.rs | 91 ++++- rs/moq-studio/tests/browser.rs | 270 ++++++++++++++ 19 files changed, 2613 insertions(+), 30 deletions(-) create mode 100644 rs/moq-scene/src/browser/host.rs create mode 100644 rs/moq-scene/src/browser/import.rs create mode 100644 rs/moq-scene/src/browser/mod.rs create mode 100644 rs/moq-scene/src/browser/protocol.rs create mode 100644 rs/moq-scene/src/browser/run.rs create mode 100644 rs/moq-studio/build.rs create mode 100644 rs/moq-studio/src/interact.rs create mode 100644 rs/moq-studio/tests/browser.rs diff --git a/Cargo.lock b/Cargo.lock index c3f6f1d..51aec8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,12 +117,56 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -627,6 +671,15 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "calloop" version = "0.13.0" @@ -689,6 +742,40 @@ dependencies = [ "shlex 2.0.1", ] +[[package]] +name = "cef" +version = "156.3.0+156.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630eba7cb50b3cc5ee13ece8c922f36f72ace8a1db8e81b6f694b98c5cf373b1" +dependencies = [ + "ash", + "cef-dll-sys", + "libc", + "libloading 0.9.0", + "objc2 0.6.5", + "objc2-foundation 0.3.2", + "objc2-io-surface", + "objc2-metal 0.3.2", + "thiserror 2.0.21", + "tracing", + "wgpu", + "windows", + "windows-sys 0.61.2", +] + +[[package]] +name = "cef-dll-sys" +version = "156.3.0+156.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d194589b022eed4f975dc40fc8a6318ea7b42e2ff8b393f880f496790cafc4" +dependencies = [ + "anyhow", + "cc", + "cmake", + "download-cef", + "serde_json", +] + [[package]] name = "cexpr" version = "0.6.0" @@ -763,6 +850,46 @@ dependencies = [ "libloading 0.8.9", ] +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + [[package]] name = "clipboard-win" version = "5.4.3" @@ -816,6 +943,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.8" @@ -835,6 +968,18 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.16.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -856,12 +1001,41 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "cookie-factory" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap 2.14.2", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1310,6 +1484,26 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +[[package]] +name = "download-cef" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ee509499d14dc47da5f824e4bb1bd558ceb259adc5475956265d48787a80da" +dependencies = [ + "bzip2", + "clap", + "fs-err", + "indicatif", + "regex", + "semver", + "serde", + "serde_json", + "sha1_smol", + "tar", + "thiserror 2.0.21", + "ureq", +] + [[package]] name = "dpi" version = "0.1.2" @@ -1569,6 +1763,12 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "endi" version = "1.1.1" @@ -1745,6 +1945,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.14" @@ -2653,6 +2863,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "inotify" version = "0.11.5" @@ -2679,6 +2902,12 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.13.0" @@ -2931,6 +3160,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +[[package]] +name = "libbz2-rs-sys" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f454fd6831a29c0e827ed075bca2a15be201bd57f4168f0fd1041124eab17e" + [[package]] name = "libc" version = "0.2.191" @@ -3403,6 +3638,7 @@ name = "moq-scene" version = "0.1.0" dependencies = [ "bytemuck", + "cef", "hang", "humantime-serde", "image", @@ -3411,6 +3647,8 @@ dependencies = [ "moq-mux", "moq-net", "moq-video", + "postcard", + "rustix 1.1.5", "serde", "serde_with", "tempfile", @@ -4243,8 +4481,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ "bitflags 2.13.2", + "libc", "objc2 0.6.5", "objc2-core-foundation", + "objc2-foundation 0.3.2", ] [[package]] @@ -4295,6 +4535,7 @@ dependencies = [ "objc2 0.6.5", "objc2-core-foundation", "objc2-foundation 0.3.2", + "objc2-io-surface", ] [[package]] @@ -4452,6 +4693,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openh264" version = "0.9.9" @@ -5371,6 +5618,7 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -5777,6 +6025,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.11.1" @@ -6214,6 +6468,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -6669,6 +6934,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "unsafe-libopus" version = "0.2.0" @@ -6692,6 +6963,38 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7be76686bdf7198002062423ca0d8cf9b75a27c8e1de804513b97e97b555d9d" +dependencies = [ + "base64 0.23.1", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622dce33c252115466d83f63e9744a18f12b4406aae51a01c15ad8fcf1185eb3" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -6749,12 +7052,24 @@ dependencies = [ "usage-argv", ] +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.28.0" @@ -7787,6 +8102,16 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.5", +] + [[package]] name = "xcursor" version = "0.3.11" diff --git a/flake.nix b/flake.nix index 048ecb6..aa73122 100644 --- a/flake.nix +++ b/flake.nix @@ -82,6 +82,80 @@ ] ); + # CEF for browser sources (`cef` feature), at the version the `cef` + # crate in Cargo.lock binds: bump them together. cef-dll-sys builds + # its wrapper from this instead of downloading CEF, and a dev build + # loads libcef.so from it. Patched like nixpkgs' cef-binary, whose + # install step expects GL libraries this release no longer ships. + cefArchive = { + x86_64-linux = { + name = "cef_binary_156.0.15+gcf671c0+chromium-156.0.8078.80_linux64_minimal"; + hash = "sha256-aHP5dbsEcxZoDqGDmYDtW9TDPIzUzkV2HJQhGFZgU4g="; + sha1 = "09235427222e026a4f34399742df27c5d98a4f8a"; + }; + aarch64-linux = { + name = "cef_binary_156.0.15+gcf671c0+chromium-156.0.8078.80_linuxarm64_minimal"; + hash = "sha256-gGZjqeyiqPe+GFSAzn53nFhktc3VK9d0iQU72xkz/rg="; + sha1 = "85602db8e813ad824b45ae54218476aa3d91d561"; + }; + }; + cef = lib.optionalAttrs isLinux ( + let + archive = cefArchive.${system}; + in + pkgs.stdenv.mkDerivation { + pname = "cef"; + version = "156.0.15"; + src = pkgs.fetchurl { + url = "https://cef-builds.spotifycdn.com/${lib.escapeURL archive.name}.tar.bz2"; + inherit (archive) hash; + }; + nativeBuildInputs = [ pkgs.autoPatchelfHook ]; + buildInputs = with pkgs; [ + alsa-lib + at-spi2-atk + at-spi2-core + atk + cairo + cups + dbus + expat + glib + gtk3 + libdrm + libgbm + libx11 + libxcb + libxcomposite + libxdamage + libxext + libxfixes + libxkbcommon + libxrandr + libxshmfence + nspr + nss + pango + stdenv.cc.cc + ]; + # Opened at run time rather than linked. + runtimeDependencies = with pkgs; [ + libGL + udev + vulkan-loader + ]; + dontStrip = true; + installPhase = '' + runHook preInstall + mkdir $out + cp -r Release/. Resources/. CMakeLists.txt cmake include libcef_dll $out/ + # What cef-dll-sys checks the version against. + echo '{"type":"minimal","name":"${archive.name}.tar.bz2","sha1":"${archive.sha1}"}' > $out/archive.json + runHook postInstall + ''; + } + ); + # Where the shell's libasound looks for PCM plugins. ALSA's default # device is a plugin on any desktop running a sound server, and the # shell's alsa-lib only searches its own store path, so without this @@ -108,6 +182,8 @@ jq cmake nasm + # cef-dll-sys builds CEF's C++ wrapper with Ninja. + ninja pkg-config # libclang for the bindgen builds (pipewire, VAAPI). rustPlatform.bindgenHook @@ -156,6 +232,7 @@ env = lib.optionalAttrs isLinux { ALSA_PLUGIN_DIR = "${alsaPlugins}/lib/alsa-lib"; + CEF_PATH = "${cef}"; LD_LIBRARY_PATH = lib.makeLibraryPath runtimeLibs; }; }; diff --git a/rs/moq-scene/Cargo.toml b/rs/moq-scene/Cargo.toml index a89e534..a85b094 100644 --- a/rs/moq-scene/Cargo.toml +++ b/rs/moq-scene/Cargo.toml @@ -25,6 +25,11 @@ capture = ["moq-video/capture", "moq-audio/capture"] # moq-video's `pipewire`: it links libpipewire-0.3 at build time, so a Linux # package turns it on where the build host has it. pipewire = ["capture", "moq-video/pipewire", "moq-audio/pipewire"] +# Browser sources, rendered by CEF (the Chromium Embedded Framework) in a helper +# process, on Linux. Off by default: the build downloads the CEF runtime, or +# takes it from `CEF_PATH`, and compiles its C++ wrapper. Without it, a browser +# source reports that this build cannot render web pages. +cef = ["dep:cef", "dep:tempfile"] [dependencies] bytemuck = { version = "1", features = ["derive"] } @@ -47,6 +52,15 @@ tokio = { workspace = true, features = ["macros", "time"] } toml = { workspace = true } tracing = { workspace = true } +# The helper process a browser source runs CEF in, and the socket between them. +[target.'cfg(target_os = "linux")'.dependencies] +# `accelerated_osr` copies the page's shared texture into the studio's device. +cef = { version = "156.3", default-features = false, features = ["accelerated_osr"], optional = true } +postcard = { version = "1", default-features = false, features = ["use-std"] } +rustix = { version = "1", features = ["net", "stdio"] } +# The helper's CEF profile, which lives as long as the helper. +tempfile = { workspace = true, optional = true } + [dev-dependencies] tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util"] } diff --git a/rs/moq-scene/src/browser/host.rs b/rs/moq-scene/src/browser/host.rs new file mode 100644 index 0000000..fd8663a --- /dev/null +++ b/rs/moq-scene/src/browser/host.rs @@ -0,0 +1,540 @@ +//! The helper process: CEF's browser process, rendering one page offscreen +//! and handing each picture and sound to the studio over the socket on its +//! stdin. + +use std::cell::RefCell; +use std::os::fd::BorrowedFd; +use std::os::unix::net::UnixStream; +use std::process::ExitCode; +use std::sync::{Arc, Mutex, mpsc}; + +use cef::*; + +use super::protocol::{Channel, Command, Event, Format, Open, Plane, Texture}; +use super::{Button, Input, Modifiers}; + +/// Run one of CEF's subprocesses (renderer, GPU, utility, zygote). +pub(super) fn subprocess() -> ExitCode { + let _ = api_hash(sys::CEF_API_VERSION_LAST, 0); + let args = args::Args::new(); + let code = execute_process(Some(args.as_main_args()), None, std::ptr::null_mut()); + ExitCode::from(code.clamp(0, 255) as u8) +} + +/// Run the helper until the studio closes the socket or the page fails. +pub(super) fn run() -> ExitCode { + match serve() { + Ok(()) => ExitCode::SUCCESS, + Err(err) => { + eprintln!("browser source helper: {err}"); + ExitCode::FAILURE + } + } +} + +/// What every handler shares. +struct Shared { + open: Open, + /// Sends from the UI thread (pictures) and CEF's audio thread (sound). + writer: Mutex, + /// Each [`Command::Painted`], which lets a painted picture go back to CEF. + painted: Mutex>, +} + +impl Shared { + fn send(&self, event: &Event, fds: &[BorrowedFd<'_>]) -> bool { + self.writer.lock().unwrap().send(event, fds).is_ok() + } + + /// Send a picture and wait until the studio has copied it. + fn paint(&self, event: &Event, fds: &[BorrowedFd<'_>]) { + if self.send(event, fds) { + // Fails only once the studio is gone, and the reader is quitting. + let _ = self.painted.lock().unwrap().recv(); + } + } + + fn fail(&self, reason: String) { + self.send(&Event::Failed(reason), &[]); + } +} + +thread_local! { + /// The page, on CEF's UI thread, where input is delivered. + static BROWSER: RefCell> = const { RefCell::new(None) }; +} + +fn serve() -> Result<(), String> { + // The studio hands over the socket as stdin. Take it, and leave CEF's + // subprocesses a harmless stdin instead. + let socket = rustix::io::fcntl_dupfd_cloexec(rustix::stdio::stdin(), 3).map_err(|err| format!("stdin: {err}"))?; + let null = std::fs::File::open("/dev/null").map_err(|err| format!("/dev/null: {err}"))?; + rustix::stdio::dup2_stdin(&null).map_err(|err| format!("stdin: {err}"))?; + let reader = Channel::new(UnixStream::from(socket)); + let writer = reader.try_clone().map_err(|err| format!("socket: {err}"))?; + + let open = match reader.recv::() { + Ok(Some((Command::Open(open), _))) => open, + Ok(Some((command, _))) => return Err(format!("expected Open, got {command:?}")), + Ok(None) => return Ok(()), + Err(err) => return Err(format!("socket: {err}")), + }; + + let (painted_tx, painted) = mpsc::channel(); + let shared = Arc::new(Shared { + open, + writer: Mutex::new(writer), + painted: Mutex::new(painted), + }); + + let _ = api_hash(sys::CEF_API_VERSION_LAST, 0); + let args = args::Args::new(); + let mut app = HelperApp::new(shared.open.render_node.clone()); + // A profile of its own, since two CEF instances cannot share one, gone + // when the helper is. + let profile = tempfile::tempdir().map_err(|err| format!("profile: {err}"))?; + let settings = Settings { + windowless_rendering_enabled: 1, + root_cache_path: profile.path().to_string_lossy().as_ref().into(), + log_severity: LogSeverity::WARNING, + // Transparent, so an overlay shows what is under it. + background_color: 0, + ..Default::default() + }; + if initialize(Some(args.as_main_args()), Some(&settings), Some(&mut app), std::ptr::null_mut()) != 1 { + return Err("CEF failed to initialize".into()); + } + + let window = WindowInfo { + windowless_rendering_enabled: 1, + shared_texture_enabled: shared.open.accelerated.into(), + ..Default::default() + }; + let browser_settings = BrowserSettings { + windowless_frame_rate: shared.open.framerate as _, + background_color: 0, + ..Default::default() + }; + let mut client = HelperClient::new( + HelperRender::new(shared.clone()), + HelperAudio::new(shared.clone()), + HelperLoad::new(shared.clone()), + HelperRequest::new(shared.clone()), + HelperLife::new(), + ); + let url = CefString::from(shared.open.url.as_str()); + let browser = browser_host_create_browser_sync( + Some(&window), + Some(&mut client), + Some(&url), + Some(&browser_settings), + None, + None, + ) + .ok_or("CEF failed to create the page")?; + BROWSER.with_borrow_mut(|slot| *slot = Some(browser)); + + std::thread::Builder::new() + .name("browser-commands".into()) + .spawn(move || commands(reader, painted_tx)) + .map_err(|err| format!("thread: {err}"))?; + + run_message_loop(); + BROWSER.with_borrow_mut(|slot| *slot = None); + shutdown(); + drop(profile); + Ok(()) +} + +/// Read the studio's commands until it closes the socket, then close the page, +/// which ends the message loop. +fn commands(reader: Channel, painted: mpsc::Sender<()>) { + loop { + match reader.recv::() { + Ok(Some((Command::Painted, _))) => { + let _ = painted.send(()); + } + Ok(Some((Command::Input(input), _))) => post(move || deliver(input)), + Ok(Some((Command::Open(_), _))) | Ok(None) | Err(_) => break, + } + } + // Unblocks a paint waiting on the studio. + drop(painted); + post(|| { + BROWSER.with_borrow(|browser| { + if let Some(host) = browser.as_ref().and_then(|browser| browser.host()) { + host.close_browser(1); + } + }) + }); +} + +/// Hand one input event to the page. Runs on the UI thread. +fn deliver(input: Input) { + BROWSER.with_borrow(|browser| { + let Some(host) = browser.as_ref().and_then(|browser| browser.host()) else { + return; + }; + let mouse = |x: i32, y: i32, modifiers: Modifiers| MouseEvent { + x, + y, + modifiers: flags(modifiers), + }; + match input { + Input::Move { x, y, modifiers } => host.send_mouse_move_event(Some(&mouse(x, y, modifiers)), 0), + Input::Leave { modifiers } => host.send_mouse_move_event(Some(&mouse(0, 0, modifiers)), 1), + Input::Button { + x, + y, + button, + down, + clicks, + modifiers, + } => { + let button = match button { + Button::Left => MouseButtonType::LEFT, + Button::Middle => MouseButtonType::MIDDLE, + Button::Right => MouseButtonType::RIGHT, + }; + host.send_mouse_click_event( + Some(&mouse(x, y, modifiers)), + button, + (!down).into(), + clicks.min(3) as _, + ); + } + Input::Wheel { + x, + y, + dx, + dy, + modifiers, + } => host.send_mouse_wheel_event(Some(&mouse(x, y, modifiers)), dx, dy), + Input::Key { code, down, modifiers } => { + let event = KeyEvent { + type_: if down { + KeyEventType::RAWKEYDOWN + } else { + KeyEventType::KEYUP + }, + modifiers: flags(modifiers), + windows_key_code: code, + ..Default::default() + }; + host.send_key_event(Some(&event)); + } + Input::Text(text) => { + for unit in text.encode_utf16() { + let event = KeyEvent { + type_: KeyEventType::CHAR, + windows_key_code: unit.into(), + character: unit, + unmodified_character: unit, + ..Default::default() + }; + host.send_key_event(Some(&event)); + } + } + Input::Focus(focused) => host.set_focus(focused.into()), + } + }); +} + +fn flags(modifiers: Modifiers) -> u32 { + use sys::cef_event_flags_t as F; + [ + (modifiers.shift, F::EVENTFLAG_SHIFT_DOWN), + (modifiers.ctrl, F::EVENTFLAG_CONTROL_DOWN), + (modifiers.alt, F::EVENTFLAG_ALT_DOWN), + (modifiers.meta, F::EVENTFLAG_COMMAND_DOWN), + (modifiers.left, F::EVENTFLAG_LEFT_MOUSE_BUTTON), + (modifiers.middle, F::EVENTFLAG_MIDDLE_MOUSE_BUTTON), + (modifiers.right, F::EVENTFLAG_RIGHT_MOUSE_BUTTON), + ] + .into_iter() + .filter(|(held, _)| *held) + .fold(0, |flags, (_, flag)| flags | flag.0) +} + +/// Run `job` on CEF's UI thread. +fn post(job: impl FnOnce() + Send + 'static) { + let job: Job = Arc::new(Mutex::new(Some(Box::new(job)))); + let mut task = HelperTask::new(job); + post_task(ThreadId::UI, Some(&mut task)); +} + +type Job = Arc>>>; + +wrap_task! { + struct HelperTask { + job: Job, + } + + impl Task { + fn execute(&self) { + if let Some(job) = self.job.lock().unwrap().take() { + job(); + } + } + } +} + +wrap_app! { + struct HelperApp { + render_node: Option, + } + + impl App { + fn on_before_command_line_processing( + &self, + _process_type: Option<&CefString>, + command_line: Option<&mut CommandLine>, + ) { + let Some(command_line) = command_line else { + return; + }; + // An alert box plays its sound without anyone clicking the page. + command_line.append_switch_with_value( + Some(&"autoplay-policy".into()), + Some(&"no-user-gesture-required".into()), + ); + // Paint on the studio's GPU, which is the only one it imports from. + if let Some(node) = &self.render_node { + command_line.append_switch_with_value(Some(&"render-node-override".into()), Some(&node.as_str().into())); + } + } + } +} + +wrap_client! { + struct HelperClient { + render: RenderHandler, + audio: AudioHandler, + load: LoadHandler, + request: RequestHandler, + life: LifeSpanHandler, + } + + impl Client { + fn render_handler(&self) -> Option { + Some(self.render.clone()) + } + + fn audio_handler(&self) -> Option { + Some(self.audio.clone()) + } + + fn load_handler(&self) -> Option { + Some(self.load.clone()) + } + + fn request_handler(&self) -> Option { + Some(self.request.clone()) + } + + fn life_span_handler(&self) -> Option { + Some(self.life.clone()) + } + } +} + +wrap_render_handler! { + struct HelperRender { + shared: Arc, + } + + impl RenderHandler { + fn view_rect(&self, _browser: Option<&mut Browser>, rect: Option<&mut Rect>) { + if let Some(rect) = rect { + rect.width = self.shared.open.width as _; + rect.height = self.shared.open.height as _; + } + } + + fn on_paint( + &self, + _browser: Option<&mut Browser>, + type_: PaintElementType, + _dirty_rects: Option<&[Rect]>, + buffer: *const u8, + width: ::std::os::raw::c_int, + height: ::std::os::raw::c_int, + ) { + if type_ != PaintElementType::VIEW || buffer.is_null() || width <= 0 || height <= 0 { + return; + } + let len = width as usize * height as usize * 4; + // SAFETY: CEF hands over `width * height` BGRA pixels, valid for + // this call. + let data = unsafe { std::slice::from_raw_parts(buffer, len) }.to_vec(); + let event = Event::Pixels { + width: width as u32, + height: height as u32, + data, + }; + self.shared.paint(&event, &[]); + } + + fn on_accelerated_paint( + &self, + _browser: Option<&mut Browser>, + type_: PaintElementType, + _dirty_rects: Option<&[Rect]>, + info: Option<&AcceleratedPaintInfo>, + ) { + let Some(info) = info else { return }; + if type_ != PaintElementType::VIEW { + return; + } + let format = match info.format { + ColorType::RGBA_8888 => Format::Rgba, + ColorType::BGRA_8888 => Format::Bgra, + other => { + self.shared.fail(format!("CEF painted an unknown format {:?}", other.as_ref())); + return; + } + }; + let count = (info.plane_count.max(0) as usize).min(info.planes.len()); + let planes = &info.planes[..count]; + let texture = Texture { + width: info.extra.coded_size.width as u32, + height: info.extra.coded_size.height as u32, + format, + modifier: info.modifier, + planes: planes + .iter() + .map(|plane| Plane { + stride: plane.stride, + offset: plane.offset, + size: plane.size, + }) + .collect(), + }; + // SAFETY: CEF owns these descriptors and keeps them open for this + // call, which outlasts the send that duplicates them. + let fds: Vec<_> = planes.iter().map(|plane| unsafe { BorrowedFd::borrow_raw(plane.fd) }).collect(); + self.shared.paint(&Event::Texture(texture), &fds); + } + } +} + +wrap_audio_handler! { + struct HelperAudio { + shared: Arc, + } + + impl AudioHandler { + fn audio_parameters( + &self, + _browser: Option<&mut Browser>, + params: Option<&mut AudioParameters>, + ) -> ::std::os::raw::c_int { + let Some(params) = params else { return 0 }; + params.channel_layout = match self.shared.open.channels { + 1 => ChannelLayout::LAYOUT_MONO, + 2 => ChannelLayout::LAYOUT_STEREO, + _ => return 0, + }; + params.sample_rate = self.shared.open.sample_rate as _; + 1 + } + + fn on_audio_stream_started( + &self, + _browser: Option<&mut Browser>, + params: Option<&AudioParameters>, + channels: ::std::os::raw::c_int, + ) { + let rate = params.map(|params| params.sample_rate); + if channels as u32 != self.shared.open.channels || rate != Some(self.shared.open.sample_rate as _) { + self.shared.fail(format!( + "the page's sound arrived as {channels} channels at {rate:?} Hz, not the bus's {} at {} Hz", + self.shared.open.channels, self.shared.open.sample_rate + )); + } + } + + fn on_audio_stream_packet( + &self, + _browser: Option<&mut Browser>, + data: *mut *const f32, + frames: ::std::os::raw::c_int, + _pts: i64, + ) { + if data.is_null() || frames <= 0 { + return; + } + let channels = self.shared.open.channels as usize; + let frames = frames as usize; + // SAFETY: CEF hands over one plane of `frames` samples per + // channel it started the stream with, which is the bus's count. + let planes: Vec<&[f32]> = (0..channels) + .map(|channel| unsafe { std::slice::from_raw_parts(*data.add(channel), frames) }) + .collect(); + let samples = (0..frames) + .flat_map(|frame| planes.iter().map(move |plane| plane[frame])) + .collect(); + self.shared.send(&Event::Audio(samples), &[]); + } + + fn on_audio_stream_error(&self, _browser: Option<&mut Browser>, message: Option<&CefString>) { + let message = message.map(|message| message.to_string()).unwrap_or_default(); + self.shared.fail(format!("the page's sound failed: {message}")); + } + } +} + +wrap_load_handler! { + struct HelperLoad { + shared: Arc, + } + + impl LoadHandler { + fn on_load_error( + &self, + _browser: Option<&mut Browser>, + frame: Option<&mut Frame>, + error_code: Errorcode, + error_text: Option<&CefString>, + failed_url: Option<&CefString>, + ) { + // A subresource or an iframe failing leaves the page standing, and + // ABORTED is a navigation replaced by another. + if frame.is_none_or(|frame| frame.is_main() == 0) || error_code == Errorcode::ABORTED { + return; + } + let url = failed_url.map(|url| url.to_string()).unwrap_or_default(); + let text = error_text.map(|text| text.to_string()).unwrap_or_default(); + self.shared.fail(format!("cannot load {url}: {text}")); + } + } +} + +wrap_request_handler! { + struct HelperRequest { + shared: Arc, + } + + impl RequestHandler { + fn on_render_process_terminated( + &self, + _browser: Option<&mut Browser>, + _status: TerminationStatus, + error_code: ::std::os::raw::c_int, + error_string: Option<&CefString>, + ) { + let reason = error_string.map(|reason| reason.to_string()).unwrap_or_default(); + self.shared.fail(format!("the page's renderer stopped ({error_code}): {reason}")); + } + } +} + +wrap_life_span_handler! { + struct HelperLife; + + impl LifeSpanHandler { + fn on_before_close(&self, _browser: Option<&mut Browser>) { + quit_message_loop(); + } + } +} diff --git a/rs/moq-scene/src/browser/import.rs b/rs/moq-scene/src/browser/import.rs new file mode 100644 index 0000000..a8b272c --- /dev/null +++ b/rs/moq-scene/src/browser/import.rs @@ -0,0 +1,59 @@ +//! Copy a page's shared texture onto the studio's device. + +use std::os::fd::OwnedFd; + +use super::Helper; +use super::protocol::Texture; +use crate::wgpu; + +/// Whether the studio's device can import the helper's shared textures, so +/// the helper should paint into them. +pub(crate) fn supported(helper: &Helper) -> bool { + cfg!(feature = "cef") && helper.device.features().contains(wgpu::Features::VULKAN_EXTERNAL_MEMORY_DMA_BUF) +} + +/// Copy the DMA-BUF `texture` describes, whose plane descriptors are `fds`, +/// into a texture of the studio's own, waiting until the copy is done. +#[cfg(feature = "cef")] +pub(super) fn texture(helper: &Helper, texture: &Texture, fds: Vec) -> Result { + use std::os::fd::AsRawFd; + + use cef::osr_texture_import::SharedTextureHandle; + + use super::protocol::{Format, MAX_PLANES}; + + if fds.len() != texture.planes.len() || fds.is_empty() || fds.len() > MAX_PLANES { + return Err(format!( + "a shared texture came with {} planes and {} descriptors", + texture.planes.len(), + fds.len() + )); + } + let mut info = cef::AcceleratedPaintInfo::default(); + for ((slot, plane), fd) in info.planes.iter_mut().zip(&texture.planes).zip(&fds) { + slot.stride = plane.stride; + slot.offset = plane.offset; + slot.size = plane.size; + slot.fd = fd.as_raw_fd(); + } + info.plane_count = fds.len() as _; + info.modifier = texture.modifier; + info.format = match texture.format { + Format::Rgba => cef::ColorType::RGBA_8888, + Format::Bgra => cef::ColorType::BGRA_8888, + }; + info.extra.coded_size.width = texture.width as _; + info.extra.coded_size.height = texture.height as _; + + // SAFETY: the helper keeps the texture out of CEF's hands until it hears + // back, and `fds` stay open until this returns. The device is the + // studio's, opened by `moq_video::render::request_device` with DMA-BUF + // import, on the GPU CEF paints with. + unsafe { SharedTextureHandle::new(&info).import_texture(&helper.device, &helper.queue) } + .map_err(|err| format!("cannot import the page's texture: {err}")) +} + +#[cfg(not(feature = "cef"))] +pub(super) fn texture(_: &Helper, _: &Texture, _: Vec) -> Result { + Err("this build cannot import shared textures (enable the `cef` feature)".into()) +} diff --git a/rs/moq-scene/src/browser/mod.rs b/rs/moq-scene/src/browser/mod.rs new file mode 100644 index 0000000..50574f9 --- /dev/null +++ b/rs/moq-scene/src/browser/mod.rs @@ -0,0 +1,247 @@ +//! Browser sources: a web page, such as an overlay, alert box, or chat widget, +//! rendered offscreen by CEF (the Chromium Embedded Framework) and drawn by the +//! compositor. +//! +//! Each page runs in a helper process of its own, so a crash there fails that +//! source and nothing else. The helper is the studio's own executable started +//! again, which calls [`child`] first thing in `main` to become one; CEF's own +//! subprocesses start the same way. The helper paints into a shared GPU texture +//! the studio copies on its device, or into memory the studio uploads when +//! either side cannot share textures ([`Paint`] says which). The page's sound +//! joins the program bus at the bus's rate. +//! +//! Linux only for now, and only with the `cef` feature: elsewhere a browser +//! source reports that it is unsupported. + +use std::path::PathBuf; +use std::process::ExitCode; +use std::sync::{Arc, Mutex}; + +use serde::{Deserialize, Serialize}; + +use crate::wgpu; + +#[cfg(target_os = "linux")] +mod protocol; +#[cfg(target_os = "linux")] +mod run; + +#[cfg(all(target_os = "linux", feature = "cef"))] +mod host; +#[cfg(target_os = "linux")] +mod import; + +#[cfg(target_os = "linux")] +pub(crate) use import::supported as accelerated; +#[cfg(target_os = "linux")] +pub(crate) use protocol::Open; + +/// The render node CEF paints on, if the studio's device has one. +#[cfg(target_os = "linux")] +pub(crate) fn render_node(helper: &Helper) -> Option { + Some(helper.render_node.as_ref()?.to_str()?.to_string()) +} +#[cfg(target_os = "linux")] +pub(crate) use run::run; + +/// The longest side of a page, in pixels: room for a 4K canvas. +pub(crate) const MAX_SIDE: u32 = 4096; + +/// The argument that starts the studio's executable as a helper. +#[cfg_attr(not(target_os = "linux"), expect(dead_code, reason = "browser sources are Linux-only"))] +const ARG: &str = "--moq-browser-helper"; + +/// Become a browser source's helper, or one of CEF's subprocesses, if this +/// process was started as one; `None` for any other process. +/// +/// A binary that offers browser sources calls this before anything else in +/// `main`, and exits with the code it returns. +pub fn child() -> Option { + #[cfg(all(target_os = "linux", feature = "cef"))] + { + if std::env::args_os().nth(1).is_some_and(|arg| arg == ARG) { + return Some(host::run()); + } + if std::env::args_os().any(|arg| arg.as_encoded_bytes().starts_with(b"--type=")) { + return Some(host::subprocess()); + } + } + None +} + +/// What browser sources need from the studio: the executable that becomes +/// their helper, and the device their pictures are copied to. +#[derive(Clone)] +pub struct Helper { + program: PathBuf, + device: wgpu::Device, + queue: wgpu::Queue, + /// The device's DRM render node, which CEF must paint on for the studio to + /// import what it paints. + #[cfg_attr(not(target_os = "linux"), expect(dead_code, reason = "browser sources are Linux-only"))] + render_node: Option, +} + +impl Helper { + /// Start helpers from `program`, which calls [`child`] first thing in + /// `main`, usually the running executable, and copy their pictures to + /// `device`, opened from `adapter`. + pub fn new(program: impl Into, adapter: &wgpu::Adapter, device: &wgpu::Device, queue: &wgpu::Queue) -> Self { + // A machine with two GPUs would otherwise paint on whichever CEF + // picks, and the studio cannot import from another GPU. + let bus = adapter.get_info().device_pci_bus_id; + let render_node = (!bus.is_empty()) + .then(|| std::fs::canonicalize(format!("/dev/dri/by-path/pci-{bus}-render")).ok()) + .flatten(); + Self { + program: program.into(), + device: device.clone(), + queue: queue.clone(), + render_node, + } + } +} + +/// How a page's pictures reach the compositor. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Paint { + /// A shared texture, copied on the GPU. + Gpu, + /// Pixels in memory, uploaded every frame: the fallback when the page's + /// GPU process or the studio's device cannot share textures. + Cpu, +} + +/// A running browser source: how it paints, and the way to interact with it. +#[derive(Clone, Default)] +pub struct Page { + state: Arc>, +} + +#[derive(Default)] +struct PageState { + #[cfg(target_os = "linux")] + commands: Option, + paint: Option, +} + +impl Page { + /// How the newest picture arrived, once one has. + pub fn paint(&self) -> Option { + self.state.lock().unwrap().paint + } + + /// Interact with the page. Dropped while the page is starting or after it + /// stopped, as a click on a window that is not there yet would be. + pub fn send(&self, input: Input) { + #[cfg(target_os = "linux")] + { + let mut state = self.state.lock().unwrap(); + let sent = state + .commands + .as_ref() + .map(|commands| commands.send(&protocol::Command::Input(input), &[])); + // A failed send means the helper is gone, which its source reports. + if let Some(Err(_)) = sent { + state.commands = None; + } + } + #[cfg(not(target_os = "linux"))] + let _ = input; + } +} + +/// Interaction with a page. Positions are in page pixels from its top-left +/// corner. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum Input { + /// The pointer moved over the page. + Move { x: i32, y: i32, modifiers: Modifiers }, + /// The pointer left the page. + Leave { modifiers: Modifiers }, + /// A mouse button went down or up; `clicks` counts a double or triple + /// click. + Button { + x: i32, + y: i32, + button: Button, + down: bool, + clicks: u32, + modifiers: Modifiers, + }, + /// A scroll, in pixels. + Wheel { + x: i32, + y: i32, + dx: i32, + dy: i32, + modifiers: Modifiers, + }, + /// A key went down or up. `code` is the Windows virtual-key code, which is + /// what CEF takes on every platform. + Key { code: i32, down: bool, modifiers: Modifiers }, + /// Text typed, after the keyboard layout and any input method. + Text(String), + /// The page gained or lost keyboard focus. + Focus(bool), +} + +/// A mouse button. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Button { + Left, + Middle, + Right, +} + +/// The keys and buttons held during an [`Input`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Modifiers { + pub shift: bool, + pub ctrl: bool, + pub alt: bool, + /// Command on macOS, the Windows key elsewhere. + pub meta: bool, + pub left: bool, + pub middle: bool, + pub right: bool, +} + +/// The texture a page's pixels are uploaded to: premultiplied BGRA, as CEF +/// paints. +#[cfg_attr(not(target_os = "linux"), expect(dead_code, reason = "browser sources are Linux-only"))] +fn upload(device: &wgpu::Device, queue: &wgpu::Queue, width: u32, height: u32, data: &[u8]) -> Result { + let limit = device.limits().max_texture_dimension_2d; + if width == 0 || height == 0 || width > limit || height > limit { + return Err(format!("a {width}x{height} page does not fit this GPU's {limit}px texture limit")); + } + if data.len() != width as usize * height as usize * 4 { + return Err(format!("{} bytes is not a {width}x{height} picture", data.len())); + } + let size = wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }; + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene page"), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Bgra8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + queue.write_texture( + texture.as_image_copy(), + data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(width * 4), + rows_per_image: Some(height), + }, + size, + ); + Ok(texture) +} diff --git a/rs/moq-scene/src/browser/protocol.rs b/rs/moq-scene/src/browser/protocol.rs new file mode 100644 index 0000000..434898d --- /dev/null +++ b/rs/moq-scene/src/browser/protocol.rs @@ -0,0 +1,283 @@ +//! What a browser source says to its helper process, and back. +//! +//! One Unix stream socket carries both directions, each message a +//! little-endian `u32` length and that many bytes of postcard. A shared +//! texture's DMA-BUF descriptors ride with the first byte of its message as +//! `SCM_RIGHTS`, so they arrive with the message that describes them. + +use std::io::{self, IoSlice, IoSliceMut, Read, Write}; +use std::mem::MaybeUninit; +use std::os::fd::{AsFd, BorrowedFd, OwnedFd}; +use std::os::unix::net::UnixStream; + +use rustix::net::{ + RecvAncillaryBuffer, RecvAncillaryMessage, RecvFlags, SendAncillaryBuffer, SendAncillaryMessage, SendFlags, +}; +use serde::{Deserialize, Serialize}; +use serde::de::DeserializeOwned; + +use super::Input; + +/// The most planes a shared texture has, and so the most descriptors one +/// message carries: CEF's own limit. +pub(crate) const MAX_PLANES: usize = 4; + +/// The longest message either side accepts: the largest page's pixels, with +/// room for the rest of the message. Bounds what a misbehaving peer can make +/// the other allocate. +const MAX_MESSAGE: usize = (super::MAX_SIDE * super::MAX_SIDE * 4) as usize + 1024; + +/// What the studio asks of the helper. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(crate) enum Command { + /// The first message: the page to open and how to deliver it. + Open(Open), + /// The last picture is copied, so the helper may let CEF reuse it. + Painted, + /// Interaction with the page. + Input(Input), +} + +/// The page a helper opens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(crate) struct Open { + pub url: String, + /// The page's viewport, in pixels. + pub width: u32, + pub height: u32, + /// How often the page paints: the canvas's frame rate. + pub framerate: u32, + /// Whether to paint into shared textures the studio imports, rather than + /// into memory it uploads. + pub accelerated: bool, + /// The DRM render node to paint on: the studio's GPU. + pub render_node: Option, + /// The sound the page plays is delivered at the program bus's rate and + /// channel count, so it joins the bus without resampling. + pub sample_rate: u32, + pub channels: u32, +} + +/// What the helper tells the studio. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(crate) enum Event { + /// The page painted into a shared texture, whose plane descriptors ride + /// with this message. Valid until the studio answers + /// [`Command::Painted`]. + Texture(Texture), + /// The page painted into memory: `width * height` premultiplied BGRA + /// pixels, tightly packed. + Pixels { width: u32, height: u32, data: Vec }, + /// Interleaved samples at the [`Open`] rate and channel count. + Audio(Vec), + /// The page cannot be shown, and why. + Failed(String), +} + +/// A shared texture's layout, CEF's `cef_accelerated_paint_info_t` minus the +/// descriptors. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(crate) struct Texture { + pub width: u32, + pub height: u32, + pub format: Format, + /// The DRM format modifier: how the planes are tiled. + pub modifier: u64, + pub planes: Vec, +} + +/// The pixel layouts CEF paints. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) enum Format { + Rgba, + Bgra, +} + +/// One plane of a shared texture. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Plane { + pub stride: u32, + pub offset: u64, + pub size: u64, +} + +/// One end of the socket. +/// +/// Sending takes `&self`, so a caller sending from several threads holds a +/// lock around each [`send`](Self::send); receiving is for one thread. +pub(crate) struct Channel { + socket: UnixStream, +} + +impl Channel { + pub fn new(socket: UnixStream) -> Self { + Self { socket } + } + + /// Another handle on the same socket, so one thread can send while + /// another receives. + pub fn try_clone(&self) -> io::Result { + Ok(Self::new(self.socket.try_clone()?)) + } + + /// Send `message` with `fds` attached. + pub fn send(&self, message: &T, fds: &[BorrowedFd<'_>]) -> io::Result<()> { + assert!(fds.len() <= MAX_PLANES, "more descriptors than a message carries"); + let payload = postcard::to_stdvec(message).map_err(io::Error::other)?; + if payload.len() > MAX_MESSAGE { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "message too long")); + } + let mut bytes = Vec::with_capacity(4 + payload.len()); + bytes.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&payload); + + let mut space = [MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(MAX_PLANES))]; + let mut control = SendAncillaryBuffer::new(&mut space); + if !fds.is_empty() { + let pushed = control.push(SendAncillaryMessage::ScmRights(fds)); + assert!(pushed, "the control buffer holds every descriptor"); + } + // The descriptors go with the first byte sent; a short send leaves the + // rest for a plain write. + let sent = rustix::net::sendmsg(&self.socket, &[IoSlice::new(&bytes)], &mut control, SendFlags::NOSIGNAL)?; + (&self.socket).write_all(&bytes[sent..]) + } + + /// The next message and the descriptors that came with it, or `None` once + /// the peer has closed the socket. + pub fn recv(&self) -> io::Result)>> { + let mut length = [0u8; 4]; + let mut read = 0; + let mut fds = Vec::new(); + while read < length.len() { + let mut space = [MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(MAX_PLANES))]; + let mut control = RecvAncillaryBuffer::new(&mut space); + let received = rustix::net::recvmsg( + &self.socket, + &mut [IoSliceMut::new(&mut length[read..])], + &mut control, + RecvFlags::CMSG_CLOEXEC, + )?; + for message in control.drain() { + if let RecvAncillaryMessage::ScmRights(received) = message { + fds.extend(received); + } + } + if received.flags.contains(rustix::net::ReturnFlags::CTRUNC) { + return Err(invalid("more descriptors than a message carries")); + } + match received.bytes { + 0 if read == 0 => return Ok(None), + 0 => return Err(io::ErrorKind::UnexpectedEof.into()), + bytes => read += bytes, + } + } + + let length = u32::from_le_bytes(length) as usize; + if length > MAX_MESSAGE { + return Err(invalid("message too long")); + } + let mut payload = vec![0u8; length]; + (&self.socket).read_exact(&mut payload)?; + let message = postcard::from_bytes(&payload).map_err(invalid)?; + Ok(Some((message, fds))) + } +} + +impl AsFd for Channel { + fn as_fd(&self) -> BorrowedFd<'_> { + self.socket.as_fd() + } +} + +fn invalid(message: impl std::fmt::Display) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pair() -> (Channel, Channel) { + let (a, b) = UnixStream::pair().unwrap(); + (Channel::new(a), Channel::new(b)) + } + + /// Descriptors arrive with the message they were sent with, not the one + /// before or after, however the stream splits. + #[test] + fn descriptors_ride_with_their_message() { + let (studio, helper) = pair(); + let texture = Texture { + width: 1280, + height: 720, + format: Format::Bgra, + modifier: 0, + planes: vec![Plane { + stride: 1280 * 4, + offset: 0, + size: 1280 * 720 * 4, + }], + }; + let file = tempfile::tempfile().unwrap(); + helper.send(&Event::Audio(vec![0.5; 3]), &[]).unwrap(); + helper.send(&Event::Texture(texture.clone()), &[file.as_fd()]).unwrap(); + helper.send(&Event::Failed("gone".into()), &[]).unwrap(); + drop(helper); + + let (event, fds) = studio.recv::().unwrap().unwrap(); + assert_eq!(event, Event::Audio(vec![0.5; 3])); + assert!(fds.is_empty()); + + let (event, fds) = studio.recv::().unwrap().unwrap(); + assert_eq!(event, Event::Texture(texture)); + assert_eq!(fds.len(), 1); + let ours = rustix::fs::fstat(&fds[0]).unwrap(); + let theirs = rustix::fs::fstat(&file).unwrap(); + assert_eq!((ours.st_dev, ours.st_ino), (theirs.st_dev, theirs.st_ino), "the same file"); + + let (event, fds) = studio.recv::().unwrap().unwrap(); + assert_eq!(event, Event::Failed("gone".into())); + assert!(fds.is_empty()); + + assert!(studio.recv::().unwrap().is_none(), "closed"); + } + + /// A message larger than the socket buffer arrives whole. + #[test] + fn a_large_message_arrives_whole() { + let (studio, helper) = pair(); + let data: Vec = (0..1920 * 1080 * 4).map(|i| i as u8).collect(); + let sent = Event::Pixels { + width: 1920, + height: 1080, + data, + }; + let sender = { + let sent = sent.clone(); + std::thread::spawn(move || helper.send(&sent, &[]).unwrap()) + }; + let (received, _) = studio.recv::().unwrap().unwrap(); + sender.join().unwrap(); + assert_eq!(received, sent); + } + + /// A peer cannot make the other side allocate past the bound. + #[test] + fn refuses_an_oversized_length() { + let (studio, helper) = pair(); + (&helper.socket).write_all(&u32::MAX.to_le_bytes()).unwrap(); + let err = studio.recv::().unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + } + + /// A socket closed partway through a message is an error, not a clean end. + #[test] + fn a_truncated_message_is_an_error() { + let (studio, helper) = pair(); + (&helper.socket).write_all(&[8, 0]).unwrap(); + drop(helper); + let err = studio.recv::().unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof); + } +} diff --git a/rs/moq-scene/src/browser/run.rs b/rs/moq-scene/src/browser/run.rs new file mode 100644 index 0000000..21eb581 --- /dev/null +++ b/rs/moq-scene/src/browser/run.rs @@ -0,0 +1,141 @@ +//! The studio's side of a browser source: start its helper, then deliver what +//! the helper paints and plays until the source stops. + +use std::os::fd::OwnedFd; +use std::os::unix::net::UnixStream; +use std::process::{Child, Stdio}; + +use moq_audio::bus::Channel as Sound; + +use super::protocol::{Channel, Command, Event, Open}; +use super::{ARG, Helper, Page, Paint}; +use crate::Error; +use crate::source::{Mailbox, Picture, Status}; + +/// Run the page `open` describes until its helper stops, delivering pictures +/// to `picture` and sound to `sound`. Dropping the future stops the helper. +pub(crate) async fn run( + helper: &Helper, + open: Open, + page: &Page, + picture: &Mailbox, + sound: Option, + status: &kio::Producer, +) -> Result<(), Error> { + let (mut process, events) = spawn(helper) + .map_err(|err| Error::Browser(format!("cannot start {}: {err}", helper.program.display())))?; + let commands = events.try_clone().map_err(|err| Error::Browser(format!("socket: {err}")))?; + let end = match commands.send(&Command::Open(open), &[]) { + Ok(()) => { + page.state.lock().unwrap().commands = Some(commands); + let reader = { + let (helper, page, picture, status) = (helper.clone(), page.clone(), picture.clone(), status.clone()); + tokio::task::spawn_blocking(move || deliver(&helper, &events, &page, &picture, sound, &status)) + }; + let end = reader.await; + page.state.lock().unwrap().commands = None; + end.unwrap_or_else(|err| End::Failed(format!("the page's reader panicked: {err}"))) + } + Err(err) => closed(err), + }; + match end { + // The helper is exiting or gone, so its status says why. + End::Closed => { + let exit = process.child.wait().map_err(|err| Error::Browser(format!("the browser process: {err}")))?; + Err(Error::Browser(format!("the browser process exited: {exit}"))) + } + End::Failed(reason) => Err(Error::Browser(reason)), + } +} + +/// Why a page stopped. +enum End { + /// The helper closed the socket: it exited, or crashed. + Closed, + /// The helper reported a failure, or broke the protocol. + Failed(String), +} + +/// What a socket error says about the helper. +fn closed(err: std::io::Error) -> End { + use std::io::ErrorKind::*; + match err.kind() { + BrokenPipe | ConnectionReset | UnexpectedEof => End::Closed, + _ => End::Failed(format!("the browser process's socket: {err}")), + } +} + +/// The helper process, killed when dropped. +struct Process { + child: Child, +} + +impl Drop for Process { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Start a helper, handing it one end of a socket as its stdin. +fn spawn(helper: &Helper) -> std::io::Result<(Process, Channel)> { + let (ours, theirs) = UnixStream::pair()?; + let child = std::process::Command::new(&helper.program) + .arg(ARG) + .stdin(OwnedFd::from(theirs)) + .stdout(Stdio::null()) + .spawn()?; + Ok((Process { child }, Channel::new(ours))) +} + +/// Read the helper's events until it closes the socket or reports a failure. +fn deliver( + helper: &Helper, + events: &Channel, + page: &Page, + picture: &Mailbox, + mut sound: Option, + status: &kio::Producer, +) -> End { + let mut live = false; + loop { + let (event, fds) = match events.recv::() { + Ok(Some(received)) => received, + Ok(None) => return End::Closed, + Err(err) => return closed(err), + }; + let (texture, paint) = match event { + Event::Texture(texture) => (super::import::texture(helper, &texture, fds), Paint::Gpu), + Event::Pixels { width, height, data } => ( + super::upload(&helper.device, &helper.queue, width, height, &data), + Paint::Cpu, + ), + Event::Audio(samples) => { + if let Some(sound) = &mut sound + && let Err(err) = sound.write(bytemuck::cast_slice(&samples)) + { + return End::Failed(format!("the page's sound: {err}")); + } + continue; + } + Event::Failed(reason) => return End::Failed(reason), + }; + // The copy is done, so the helper may hand the source back to CEF. + if let Some(commands) = &page.state.lock().unwrap().commands + && let Err(err) = commands.send(&Command::Painted, &[]) + { + return closed(err); + } + let texture = match texture { + Ok(texture) => texture, + Err(reason) => return End::Failed(reason), + }; + page.state.lock().unwrap().paint = Some(paint); + *picture.lock().unwrap() = Some(Picture::Texture(texture)); + if !std::mem::replace(&mut live, true) + && let Ok(mut status) = status.write() + { + *status = Status::Live; + } + } +} diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs index 77a9182..0ee615c 100644 --- a/rs/moq-scene/src/collection.rs +++ b/rs/moq-scene/src/collection.rs @@ -120,6 +120,9 @@ impl Collection { if let Some(name) = self.sources.keys().find(|name| name.is_empty()) { return Err(Error::Invalid(format!("a source has an empty name: `{name}`"))); } + for (name, source) in &self.sources { + source.validate(name)?; + } for (name, audio) in &self.audio { if !self.sources.contains_key(name) { @@ -326,6 +329,18 @@ pub enum Source { /// The broadcast path on that relay. path: String, }, + /// A web page, such as an overlay, alert box, or chat widget, with its + /// sound. Desktop only: viewers cannot render one, so it reaches them + /// only as pixels. + Browser { + /// The page's address. + url: String, + /// The page's viewport width in pixels, independent of where it is + /// placed. + width: u32, + /// The page's viewport height in pixels. + height: u32, + }, } impl Source { @@ -338,9 +353,24 @@ impl Source { pub fn has_audio(&self) -> bool { matches!( self, - Self::Microphone { .. } | Self::SystemAudio | Self::Broadcast { .. } + Self::Microphone { .. } | Self::SystemAudio | Self::Broadcast { .. } | Self::Browser { .. } ) } + + fn validate(&self, name: &str) -> Result<(), Error> { + if let Self::Browser { url, width, height } = self { + if url.trim().is_empty() { + return Err(Error::Invalid(format!("browser source `{name}` has no url"))); + } + let max = crate::browser::MAX_SIDE; + if *width == 0 || *height == 0 || *width > max || *height > max { + return Err(Error::Invalid(format!( + "browser source `{name}` is {width}x{height}; each side must be 1 to {max} pixels" + ))); + } + } + Ok(()) + } } /// The gain and mute of one source in the program audio. diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index baa5721..a6f91eb 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -22,7 +22,8 @@ pub const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm; struct Uniform { /// Top-left x, top-left y, bottom-right x, bottom-right y, in clip space. rect: [f32; 4], - /// Opacity in `x`; the rest pads the struct to WGSL's 16-byte vectors. + /// Opacity in `x`, and in `y` 1 if the image's alpha is premultiplied; + /// the rest pads the struct to WGSL's 16-byte vectors. opacity: [f32; 4], } @@ -50,9 +51,11 @@ struct Varyings { return out; } +// Premultiplied out, so straight and premultiplied images blend alike. @fragment fn fs_main(in: Varyings) -> @location(0) vec4 { let color = textureSample(image, image_sampler, in.uv); - return vec4(color.rgb, color.a * item.opacity.x); + let rgb = select(color.rgb * color.a, color.rgb, item.opacity.y > 0.5); + return vec4(rgb, color.a) * item.opacity.x; } "#; @@ -114,6 +117,8 @@ struct Layer { renderer: Option, /// Samples the newest picture; `None` until the first arrives. bind: Option, + /// Whether that picture's alpha is premultiplied. + premultiplied: bool, /// Why the newest picture failed to draw, logged when it first does. error: Option, } @@ -216,10 +221,11 @@ impl Compositor { compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: FORMAT, - // Straight alpha over an opaque canvas, which stays opaque. + // Premultiplied alpha over an opaque canvas, which stays + // opaque. blend: Some(wgpu::BlendState { color: wgpu::BlendComponent { - src_factor: wgpu::BlendFactor::SrcAlpha, + src_factor: wgpu::BlendFactor::One, dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, operation: wgpu::BlendOperation::Add, }, @@ -357,7 +363,7 @@ impl Compositor { let mut draws = self.scene(collection, fade.from.as_deref()); draws.push(Draw { bind: self.incoming.bind.clone(), - uniform: uniform(Rect::fill(&self.canvas), self.canvas, progress), + uniform: uniform(Rect::fill(&self.canvas), self.canvas, progress, false), }); passes.push(Pass { view: &self.program.view, @@ -426,10 +432,10 @@ impl Compositor { .iter() .filter(|item| item.visible) .filter_map(|item| { - let bind = self.layers.get(&item.source)?.bind.clone()?; + let layer = self.layers.get(&item.source)?; Some(Draw { - bind, - uniform: uniform(item.rect, self.canvas, 1.0), + bind: layer.bind.clone()?, + uniform: uniform(item.rect, self.canvas, 1.0, layer.premultiplied), }) }) .collect() @@ -452,8 +458,10 @@ impl Compositor { source: source.clone(), renderer: None, bind: None, + premultiplied: false, error: None, }); + let premultiplied = matches!(picture, Picture::Texture(_)); let texture = match picture { Picture::Frame(frame) => { let renderer = match layer.renderer.as_mut() { @@ -466,6 +474,7 @@ impl Compositor { .map_err(Error::from) } Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels), + Picture::Texture(texture) => Ok(texture), }; match texture { @@ -473,6 +482,7 @@ impl Compositor { layer.error = None; let view = texture.create_view(&Default::default()); layer.bind = Some(image_bind(&self.device, &self.image_layout, &self.sampler, &view)); + layer.premultiplied = premultiplied; } Err(err) => { if layer.error.is_none() { @@ -486,13 +496,13 @@ impl Compositor { } /// Place `rect` (canvas pixels) in clip space. -fn uniform(rect: Rect, canvas: Canvas, opacity: f32) -> Uniform { +fn uniform(rect: Rect, canvas: Canvas, opacity: f32, premultiplied: bool) -> Uniform { let (width, height) = (canvas.width as f32, canvas.height as f32); let x = |px: f32| px / width * 2.0 - 1.0; let y = |px: f32| 1.0 - px / height * 2.0; Uniform { rect: [x(rect.x), y(rect.y), x(rect.x + rect.width), y(rect.y + rect.height)], - opacity: [opacity, 0.0, 0.0, 0.0], + opacity: [opacity, f32::from(u8::from(premultiplied)), 0.0, 0.0], } } diff --git a/rs/moq-scene/src/lib.rs b/rs/moq-scene/src/lib.rs index 79614db..f592b7a 100644 --- a/rs/moq-scene/src/lib.rs +++ b/rs/moq-scene/src/lib.rs @@ -8,7 +8,8 @@ //! credentials. //! - [`source::Sources`] runs each source: a camera, display, window, //! microphone, or system-audio capture (with the `capture` feature), a still -//! image, a solid color, or a broadcast subscribed over MoQ. Pictures go to the +//! image, a solid color, a broadcast subscribed over MoQ, or a web page +//! rendered by a browser engine (the [`browser`] module). Pictures go to the //! compositor and sound goes to a `moq_audio::bus::Bus`, the program audio. //! - [`Compositor`] draws the scene in preview and the scene on program into //! textures on the caller's `wgpu` device, cutting or fading between them. @@ -20,6 +21,7 @@ //! source that stalls repeats its last picture rather than holding up the //! canvas. +pub mod browser; mod collection; mod compositor; pub mod source; @@ -76,4 +78,8 @@ pub enum Error { /// The broadcast subscription failed. #[error(transparent)] Net(#[from] moq_net::Error), + + /// A browser source's page failed, or its helper process stopped. + #[error("browser: {0}")] + Browser(String), } diff --git a/rs/moq-scene/src/source.rs b/rs/moq-scene/src/source.rs index 835e7c4..999801c 100644 --- a/rs/moq-scene/src/source.rs +++ b/rs/moq-scene/src/source.rs @@ -16,6 +16,7 @@ use moq_audio::bus::{Bus, Strip}; use moq_mux::catalog::Stream; use crate::Error; +use crate::browser::Page; use crate::collection::{Collection, Source}; /// A source's newest picture, waiting for the compositor to draw it. @@ -24,11 +25,14 @@ pub(crate) enum Picture { Frame(moq_video::Frame), /// Tightly packed RGBA with straight alpha, such as a still image. Rgba { size: moq_video::Size, pixels: Vec }, + /// A texture on the compositor's device with premultiplied alpha, such as + /// a web page. + Texture(moq_video::render::wgpu::Texture), } /// The one picture a source holds for the compositor. A newer picture replaces /// an undrawn one, so a slow canvas skips pictures instead of queuing them. -type Mailbox = Arc>>; +pub(crate) type Mailbox = Arc>>; /// One instance of a broadcast being decoded. type Playing<'a> = Pin> + 'a>>; @@ -60,6 +64,8 @@ pub struct Context { /// What cameras, screens, and broadcasts deliver video as: /// `moq_video::render::output` of the compositor's device. pub output: moq_video::Output, + /// What browser sources run in; without it they fail as unsupported. + pub browser: Option, } /// The running sources of a collection. @@ -95,7 +101,7 @@ impl Sources { let mut drivers = Vec::new(); for (name, source) in &collection.sources { if !self.feeds.contains_key(name) { - let (feed, driver) = open(name, source, &self.context); + let (feed, driver) = open(name, source, &self.context, collection.canvas.framerate); self.feeds.insert(name.clone(), feed); drivers.push(driver); } @@ -126,6 +132,7 @@ pub struct Feed { picture: Mailbox, strip: Option, status: kio::Consumer, + page: Option, } impl Feed { @@ -144,6 +151,11 @@ impl Feed { self.strip.as_ref() } + /// A browser source's page: how it paints, and interaction with it. + pub fn page(&self) -> Option<&Page> { + self.page.as_ref() + } + /// The newest picture not yet drawn. pub(crate) fn take(&self) -> Option { self.picture.lock().unwrap().take() @@ -159,6 +171,9 @@ pub struct Driver { picture: Mailbox, strip: Option, status: kio::Producer, + page: Option, + /// The canvas's, which a browser source paints at. + framerate: u32, } impl Driver { @@ -200,6 +215,7 @@ impl Driver { std::future::pending().await } Source::Broadcast { path } => self.broadcast(path).await, + Source::Browser { url, width, height } => self.browser(url, *width, *height).await, device => self.device(device).await, } } @@ -221,7 +237,7 @@ impl Driver { Source::Window { id } => Video::Window(id.clone()), Source::Microphone { id } => return self.listen(Audio::Microphone(id.clone())).await, Source::SystemAudio => return self.listen(Audio::System).await, - Source::Color { .. } | Source::Image { .. } | Source::Broadcast { .. } => { + Source::Color { .. } | Source::Image { .. } | Source::Broadcast { .. } | Source::Browser { .. } => { unreachable!("not a device source") } }; @@ -253,6 +269,39 @@ impl Driver { )) } + /// Render the page at `url` until its helper stops, or fail at once if the + /// studio offers no helper. + #[cfg(target_os = "linux")] + async fn browser(&self, url: &str, width: u32, height: u32) -> Result<(), Error> { + let Some(helper) = &self.context.browser else { + return Err(Error::Unsupported( + "this build cannot render web pages (enable the `cef` feature)".into(), + )); + }; + let page = self.page.as_ref().expect("browser sources have a page"); + let bus = self.context.bus.config().clone(); + let open = crate::browser::Open { + url: url.to_string(), + width, + height, + framerate: self.framerate, + accelerated: crate::browser::accelerated(helper), + render_node: crate::browser::render_node(helper), + sample_rate: bus.sample_rate, + channels: bus.layout.channels(), + }; + let mut input = moq_audio::encode::Input::new(bus.sample_rate, bus.layout); + input.format = moq_audio::Format::F32; + let strip = self.strip.clone().expect("browser sources have a strip"); + let sound = self.context.bus.channel(input, strip)?; + crate::browser::run(helper, open, page, &self.picture, Some(sound), &self.status).await + } + + #[cfg(not(target_os = "linux"))] + async fn browser(&self, _: &str, _: u32, _: u32) -> Result<(), Error> { + Err(Error::Unsupported("browser sources run on Linux only for now".into())) + } + /// Follow the broadcast at `path` for as long as the feed lives, playing /// each instance of it that comes online. async fn broadcast(&self, path: &str) -> Result<(), Error> { @@ -417,16 +466,18 @@ fn pick<'a, C: Reference + Clone + 'a>( } /// Start the feed for `source`, and the driver that fills it. -fn open(name: &str, source: &Source, context: &Context) -> (Feed, Driver) { +fn open(name: &str, source: &Source, context: &Context, framerate: u32) -> (Feed, Driver) { let picture = Mailbox::default(); let strip = source.has_audio().then(Strip::default); let status = kio::Producer::new(Status::Starting); + let page = matches!(source, Source::Browser { .. }).then(Page::default); let feed = Feed { source: source.clone(), picture: picture.clone(), strip: strip.clone(), status: status.consume(), + page: page.clone(), }; let driver = Driver { name: name.to_string(), @@ -435,6 +486,8 @@ fn open(name: &str, source: &Source, context: &Context) -> (Feed, Driver) { picture, strip, status, + page, + framerate, }; (feed, driver) } @@ -472,6 +525,7 @@ pub(crate) mod test { origin: origin.consume(), dir: dir.to_path_buf(), output: moq_video::Output::Cpu, + browser: None, }; (context, origin) } diff --git a/rs/moq-studio/Cargo.toml b/rs/moq-studio/Cargo.toml index 8a0312d..f428840 100644 --- a/rs/moq-studio/Cargo.toml +++ b/rs/moq-studio/Cargo.toml @@ -32,6 +32,10 @@ vaapi = ["moq-video/vaapi"] # Wayland screens and windows are not reachable yet: display and window sources # capture X11 only. pipewire = ["capture", "moq-scene/pipewire"] +# Browser sources, rendered by CEF in a helper process (Linux). Off by default: +# the build downloads the CEF runtime (or takes it from `CEF_PATH`) and copies +# it beside the binary, where the binary looks for it when it starts. +cef = ["moq-scene/cef"] [dependencies] anyhow = { workspace = true, features = ["backtrace"] } diff --git a/rs/moq-studio/build.rs b/rs/moq-studio/build.rs new file mode 100644 index 0000000..f64ff44 --- /dev/null +++ b/rs/moq-studio/build.rs @@ -0,0 +1,17 @@ +//! With `cef` on Linux, the binary loads `libcef.so` when it starts: from +//! beside itself, where a package installs the CEF runtime, or else from +//! `CEF_PATH` when the build has one, as the dev shell does. + +use std::path::Path; + +fn main() { + println!("cargo::rerun-if-env-changed=CEF_PATH"); + let cef = std::env::var_os("CARGO_FEATURE_CEF").is_some(); + if !cef || std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("linux") { + return; + } + println!("cargo::rustc-link-arg-bins=-Wl,-rpath,$ORIGIN"); + if let Some(dir) = std::env::var_os("CEF_PATH").filter(|dir| Path::new(dir).join("libcef.so").exists()) { + println!("cargo::rustc-link-arg-bins=-Wl,-rpath,{}", Path::new(&dir).display()); + } +} diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs index 9cceccc..94287bf 100644 --- a/rs/moq-studio/src/engine.rs +++ b/rs/moq-studio/src/engine.rs @@ -214,6 +214,7 @@ impl Engine { origin: subscribe.consume(), dir, output: moq_video::render::output(&gpu.device), + browser: browser(&gpu)?, }); let mut compositor = Compositor::new(&gpu.device, &gpu.queue, collection.canvas)?; compositor.set_preview(collection.scenes.first().map(|scene| scene.name.clone())); @@ -573,6 +574,21 @@ fn encode_frames( Ok(()) } +/// What browser sources run in: this executable, which `main` hands to +/// `moq_scene::browser::child` when it is started as their helper. +fn browser(gpu: &Gpu) -> anyhow::Result> { + if !cfg!(feature = "cef") { + return Ok(None); + } + let program = std::env::current_exe().context("cannot find the studio's executable")?; + Ok(Some(moq_scene::browser::Helper::new( + program, + &gpu.adapter, + &gpu.device, + &gpu.queue, + ))) +} + /// Run a source on its own thread until its feed is dropped. fn spawn(driver: Driver) { let name = format!("source-{}", driver.name()); diff --git a/rs/moq-studio/src/interact.rs b/rs/moq-studio/src/interact.rs new file mode 100644 index 0000000..5d38c6f --- /dev/null +++ b/rs/moq-studio/src/interact.rs @@ -0,0 +1,414 @@ +//! Interaction with browser sources through the preview: the pointer goes to +//! the page under it, and the keyboard to the page last clicked. + +use std::time::Duration; + +use kio::clock::Instant; +use moq_scene::browser::{Button, Input, Modifiers}; +use moq_scene::{Collection, Source}; +use moq_scene::source::Sources; + +/// How close in time and space a press must follow the last to count toward a +/// double or triple click. +const MULTI_CLICK: Duration = Duration::from_millis(500); +const MULTI_CLICK_SLOP: i32 = 4; + +/// Which page has the pointer and the keyboard. +#[derive(Default)] +pub struct Interact { + /// The page under the pointer, so it hears when the pointer leaves. + hovered: Option, + /// The page last clicked, which receives the keyboard. + focused: Option, + /// The last press, to count multi-clicks. + last: Option, +} + +struct Press { + page: String, + button: Button, + x: i32, + y: i32, + at: Instant, + clicks: u32, +} + +/// A point on one page. +struct Hit { + page: String, + x: i32, + y: i32, +} + +impl Interact { + /// Forward this frame's input over `image`, which shows `scene` of + /// `collection` on `canvas`, to the pages it touches. + pub fn update( + &mut self, + ui: &egui::Ui, + image: egui::Rect, + collection: &Collection, + scene: Option<&str>, + sources: &Sources, + ) { + let canvas = collection.canvas; + let hit = |pos: egui::Pos2| -> Option { + if !image.contains(pos) { + return None; + } + let x = (pos.x - image.min.x) / image.width() * canvas.width as f32; + let y = (pos.y - image.min.y) / image.height() * canvas.height as f32; + page_at(collection, scene, x, y) + }; + let send = |page: &str, input: Input| { + if let Some(page) = sources.get(page).and_then(|feed| feed.page()) { + page.send(input); + } + }; + + let (events, buttons) = ui.input(|input| (input.events.clone(), held(&input.pointer))); + for event in events { + match event { + egui::Event::PointerMoved(pos) => { + let hit = hit(pos); + let page = hit.as_ref().map(|hit| hit.page.clone()); + if let Some(left) = self.hovered.take().filter(|left| Some(left) != page.as_ref()) { + send(&left, Input::Leave { modifiers: buttons }); + } + if let Some(Hit { page, x, y }) = hit { + send(&page, Input::Move { x, y, modifiers: buttons }); + self.hovered = Some(page); + } + } + egui::Event::PointerGone => { + if let Some(left) = self.hovered.take() { + send(&left, Input::Leave { modifiers: buttons }); + } + } + egui::Event::PointerButton { + pos, + button, + pressed, + modifiers, + } => { + let Some(button) = self::button(button) else { continue }; + let Some(Hit { page, x, y }) = hit(pos) else { + if pressed { + self.blur(&send); + } + continue; + }; + let modifiers = keys(modifiers, buttons); + let clicks = if pressed { + self.press(&page, button, x, y) + } else { + self.last.as_ref().map_or(1, |last| last.clicks) + }; + if pressed && self.focused.as_ref() != Some(&page) { + self.blur(&send); + send(&page, Input::Focus(true)); + self.focused = Some(page.clone()); + } + send( + &page, + Input::Button { + x, + y, + button, + down: pressed, + clicks, + modifiers, + }, + ); + } + egui::Event::MouseWheel { unit, delta, modifiers, .. } => { + let Some(Hit { page, x, y }) = ui.input(|input| input.pointer.hover_pos()).and_then(hit) else { + continue; + }; + // Pixels, as a page scrolls; a line is what a browser + // scrolls per wheel notch. + let scale = match unit { + egui::MouseWheelUnit::Point => 1.0, + egui::MouseWheelUnit::Line => 40.0, + egui::MouseWheelUnit::Page => canvas.height as f32, + }; + send( + &page, + Input::Wheel { + x, + y, + dx: (delta.x * scale) as i32, + dy: (delta.y * scale) as i32, + modifiers: keys(modifiers, buttons), + }, + ); + } + egui::Event::Key { + key, pressed, modifiers, .. + } => { + let (Some(page), Some(code)) = (&self.focused, code(key)) else { + continue; + }; + let modifiers = keys(modifiers, buttons); + send(page, Input::Key { + code, + down: pressed, + modifiers, + }); + // A page sees Enter as typed text too, which is what + // submits a form or breaks a line. + if pressed && key == egui::Key::Enter { + send(page, Input::Text("\r".into())); + } + } + egui::Event::Text(text) => { + if let Some(page) = &self.focused { + send(page, Input::Text(text)); + } + } + _ => {} + } + } + } + + /// Give the keyboard back, as when interaction ends. + pub fn release(&mut self, sources: &Sources) { + self.hovered = None; + self.last = None; + if let Some(page) = self.focused.take().and_then(|name| sources.get(&name)?.page().cloned()) { + page.send(Input::Focus(false)); + } + } + + fn blur(&mut self, send: &impl Fn(&str, Input)) { + if let Some(page) = self.focused.take() { + send(&page, Input::Focus(false)); + } + } + + /// Count this press toward a multi-click, returning the count. + fn press(&mut self, page: &str, button: Button, x: i32, y: i32) -> u32 { + let at = Instant::now(); + let clicks = match &self.last { + Some(last) + if last.page == page + && last.button == button + && at.saturating_duration_since(last.at) <= MULTI_CLICK + && (last.x - x).abs() <= MULTI_CLICK_SLOP + && (last.y - y).abs() <= MULTI_CLICK_SLOP => + { + last.clicks % 3 + 1 + } + _ => 1, + }; + self.last = Some(Press { + page: page.to_string(), + button, + x, + y, + at, + clicks, + }); + clicks + } +} + +/// The topmost visible browser source in `scene` at canvas point (`x`, `y`), +/// and the point on its page. +fn page_at(collection: &Collection, scene: Option<&str>, x: f32, y: f32) -> Option { + let scene = collection.scene(scene?)?; + scene.items.iter().rev().filter(|item| item.visible).find_map(|item| { + let Some(Source::Browser { width, height, .. }) = collection.sources.get(&item.source) else { + return None; + }; + let rect = item.rect; + let inside = x >= rect.x && y >= rect.y && x < rect.x + rect.width && y < rect.y + rect.height; + if !inside || rect.width <= 0.0 || rect.height <= 0.0 { + return None; + } + Some(Hit { + page: item.source.clone(), + x: ((x - rect.x) / rect.width * *width as f32) as i32, + y: ((y - rect.y) / rect.height * *height as f32) as i32, + }) + }) +} + +fn button(button: egui::PointerButton) -> Option