diff --git a/Cargo.lock b/Cargo.lock index 7313e330..44e96c0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,6 +182,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "anyrender" version = "0.14.0" @@ -592,7 +598,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d" dependencies = [ - "bit-vec", + "bit-vec 0.9.1", +] + +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec 0.10.1", ] [[package]] @@ -604,6 +619,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -756,10 +781,24 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" dependencies = [ + "borsh-derive", "bytes", "cfg_aliases", ] +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "brotli-decompressor" version = "5.0.3" @@ -1663,6 +1702,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-new" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1763,6 +1813,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "download-cef" version = "3.0.0" @@ -1827,6 +1883,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dyn-hash" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fdab65db9274e0168143841eb8f864a0a21f8b1b8d2ba6812bbe6024346e99e" + [[package]] name = "ebml-iterable" version = "0.7.1" @@ -2118,6 +2180,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e59b668157f32d3459d4bd634861a06ffa2503a461ca5f0916b9f0252605e66" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -3321,6 +3394,15 @@ dependencies = [ "libc", ] +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -3828,6 +3910,12 @@ dependencies = [ "synstructure 0.13.2", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "markup5ever" version = "0.40.0" @@ -3854,6 +3942,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.8.3" @@ -3878,12 +3976,31 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minijinja" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f5a04892e86ddb4fc254e9160835370b63a0d44042d3ab4725bc2dbcbab136" +dependencies = [ + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -4230,6 +4347,7 @@ dependencies = [ "tokio", "toml", "tracing", + "tract-onnx", "url", "vello", "wuff", @@ -4479,7 +4597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130" dependencies = [ "arrayvec", - "bit-set", + "bit-set 0.10.0", "bitflags 2.13.2", "cfg-if", "cfg_aliases", @@ -4519,6 +4637,21 @@ dependencies = [ "log", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.9.0" @@ -4589,6 +4722,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "nom-language" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" +dependencies = [ + "nom 8.0.0", +] + [[package]] name = "notify" version = "8.2.0" @@ -5750,6 +5892,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -5831,6 +5982,29 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pxfm" version = "0.1.30" @@ -5981,6 +6155,16 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_distr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "rand_pcg" version = "0.10.2" @@ -6002,6 +6186,15 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -6020,6 +6213,12 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -6332,6 +6531,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -7130,6 +7343,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "string-interner" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3df9b59e2eded8d825c7c4363ad339a20fb6bc0b9a4778560f518f59910b15" +dependencies = [ + "hashbrown 0.16.1", + "serde", +] + [[package]] name = "string_cache" version = "0.11.0" @@ -7880,6 +8109,173 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tract-core" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690191b7344a995507bed7523aeea052f907aa1316c0500540eec801e824a1c" +dependencies = [ + "anyhow", + "anymap3", + "bit-set 0.11.1", + "derive-new", + "downcast-rs 2.0.2", + "dyn-clone", + "erased-serde", + "inventory", + "lazy_static", + "log", + "maplit", + "metrics", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pastey", + "rustfft", + "serde", + "smallvec", + "tract-data", + "tract-linalg", +] + +[[package]] +name = "tract-data" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5feff0e09b8cf92be4932457bda1b2156e348bbaf8311753da7f9b102c4af0b3" +dependencies = [ + "anyhow", + "downcast-rs 2.0.2", + "dyn-clone", + "dyn-hash", + "half", + "inventory", + "itertools 0.15.0", + "lazy_static", + "libm", + "maplit", + "ndarray", + "nom 8.0.0", + "nom-language", + "num-integer", + "num-traits", + "parking_lot", + "smallvec", + "string-interner", +] + +[[package]] +name = "tract-extra" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9b3105cbc51d3a24e6b640e18c7d6cdedebeef5b9435a74518e7658ee93578" +dependencies = [ + "tract-nnef", +] + +[[package]] +name = "tract-hir" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a84e8187c517f1a641dcd1b764f1dca2b6b8d39d4c78b2743e1b69874e65b735" +dependencies = [ + "derive-new", + "log", + "tract-core", +] + +[[package]] +name = "tract-linalg" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d389398bb62713b4dd4fc92c3a36ae1552faf522e17b1ae3f636537893a55c" +dependencies = [ + "byteorder", + "cc", + "derive-new", + "downcast-rs 2.0.2", + "dyn-clone", + "dyn-hash", + "half", + "inventory", + "lazy_static", + "libc", + "log", + "minijinja", + "num-traits", + "pastey", + "tract-data", + "walkdir", +] + +[[package]] +name = "tract-nnef" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def0bced596beb4318148cac90de95dc9953089bb1df86ce2ef4c733ee450773" +dependencies = [ + "byteorder", + "erased-serde", + "log", + "nom 8.0.0", + "nom-language", + "serde", + "serde_json", + "simd-adler32", + "tar", + "tract-core", + "walkdir", +] + +[[package]] +name = "tract-onnx" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "994cd1cb5ea6747ac82707e39ae9588784f6794f96a4a86d137e71ebc49e2689" +dependencies = [ + "bytes", + "derive-new", + "log", + "memmap2", + "nom 8.0.0", + "nom-language", + "num-integer", + "num-rational", + "num-traits", + "prost", + "smallvec", + "tract-extra", + "tract-hir", + "tract-nnef", + "tract-onnx-opl", +] + +[[package]] +name = "tract-onnx-opl" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1b86a41af0b03fa2bb6d50c86d79ec363ba12e8c76fb0bff1d899d584bf310" +dependencies = [ + "getrandom 0.4.3", + "log", + "rand", + "rand_distr", + "rustfft", + "tract-extra", + "tract-nnef", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -8340,7 +8736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078" dependencies = [ "cc", - "downcast-rs", + "downcast-rs 1.2.1", "rustix 1.1.5", "scoped-tls", "smallvec", @@ -8584,8 +8980,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14" dependencies = [ "arrayvec", - "bit-set", - "bit-vec", + "bit-set 0.10.0", + "bit-vec 0.9.1", "bitflags 2.13.2", "bytemuck", "cfg_aliases", @@ -8657,7 +9053,7 @@ dependencies = [ "android_system_properties", "arrayvec", "ash", - "bit-set", + "bit-set 0.10.0", "bitflags 2.13.2", "block2 0.6.2", "bytemuck", @@ -9263,7 +9659,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" dependencies = [ - "bit-vec", + "bit-vec 0.9.1", "time", ] diff --git a/Cargo.toml b/Cargo.toml index 1d709807..b4418c67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,18 @@ opt-level = 3 [profile.dev.package.rubato] opt-level = 3 +# Background removal's model, which runs on every picture of a source and is +# ten times slower unoptimized: a camera would cut out a few pictures a +# second rather than every one. +[profile.dev.package.tract-core] +opt-level = 3 + +[profile.dev.package.tract-data] +opt-level = 3 + +[profile.dev.package.tract-linalg] +opt-level = 3 + [profile.release] lto = "fat" codegen-units = 1 diff --git a/README.md b/README.md index b482fca4..59368c9e 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ Every panel is a tab you can drag, split, or stack. | Sources | Add a source by hand, see whether each and its own track are live and where it is shown, clone one, and remove one. Select a source to filter it. A camera or broadcast without a new picture, or a microphone or system audio without new sound, for a second reads as stalled. A browser source shows whether its page is copied on the GPU or, as a fallback, uploaded from memory (CPU), in place of a track, and **Reload** restarts its page. A media file reads as ended once it plays to its end without looping, **Play** plays it from its start, **Stop** stops it at once, blank and silent, until played again, and **Reload** reads the file again. | | Devices | List this machine's cameras, displays, windows, and audio inputs, and add one as a source. | | Guests | Invite a guest, show their join link as a QR code, see who joined, and take a guest out of every scene. | -| Filters | Crop the selected source or item, add, order, and set its filters, set an item's blend mode, and copy or paste filters. | +| Filters | Remove the selected source's background, crop the selected source or item, add, order, and set its filters, set an item's blend mode, and copy or paste filters. | | Keyframes | Animate the selected item: add, time, ease, and remove its keyframes, and set how far each moves, resizes, crops, and filters it (see [Keyframes](#keyframes)). **Play** plays the preview scene's keyframes again. | | Audio | Add, rename (double-click), and remove mixes, pick the one on program and the one to listen to and on which output, and set each audio source's gain, mute, delay, and the mixes it feeds, with a meter on every mix and source. | | Output | Start and stop recording, pick the mix it records, set the canvas size and frame rate, add, resize, and remove further canvases, and see the broadcast path, whether the program is encoding, the encoder sessions open, and each canvas's renditions' encoder work. | @@ -543,6 +543,7 @@ kind = "lut" path = "teal-orange.cube" [video.camera] +background = "remove" # cut the person out, leaving the rest transparent crop = { left = 0.1, right = 0.1 } # trim 10% of the picture off each side filters = [ { kind = "chroma_key", color = "#00ff00", similarity = 0.4, smoothness = 0.08, spill = 0.1 }, @@ -598,8 +599,16 @@ without `[mixes]`, the collection has one mix, `Program`. `[audio.]` sets a source's linear `gain`, `muted`, the mixes it is left out of (`exclude`), and its `delay`. -`[video.]` sets a source's `crop` and `filters`, wherever it is shown, -and an item's `crop`, `filters`, and `blend` its own. A `crop` trims a fraction +`[video.]` sets a source's `background`, `crop`, and `filters`, +wherever it is shown, and an item's `crop`, `filters`, and `blend` its own. +`background = "remove"` cuts the person out of a camera, display, window, +image, media, broadcast, browser, or replay source's picture before its crop +and filters, leaving the rest transparent. A segmentation model finds them in +each picture on the CPU, so each picture shows only once its cut-out is +ready, a frame or more late, and a model that falls behind skips pictures +rather than queue them. It is MediaPipe's selfie segmentation model, made for +one or two people near the camera. A clone shows its source's cut-out, so it +cannot remove a background of its own. A `crop` trims a fraction (0 to 1) of the picture off its `left`, `top`, `right`, and `bottom`, opposite sides together trimming less than the whole, and stretches the rest over the item; an item's crop trims what its source's @@ -717,7 +726,9 @@ changes size or color space moves to a new track, which its catalog entry follows. Images and colors keep their picture once, as the image file and as a one-pixel PNG. Removing or editing a source ends its track. A track carries the plain capture, before any filter, and a clone has none, since it shows its -source's capture. A browser source or an overlay has no track of its own, and a +source's capture. A removed background stays in the track, since H.264 carries +no transparency, so a replay of the source shows it unless the replay source +removes it too. A browser source or an overlay has no track of its own, and a broadcast source is already its publisher's. A microphone, system audio, or media file's sound encodes its track as Opus, diff --git a/quest/m0/README.md b/quest/m0/README.md index f5caa7e8..eda72ca4 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -91,7 +91,6 @@ repository. - [Studio on macOS with capture granted](/quest/m0/app-macos.md) - camera, screen, and audio capture run on a Mac, not only Linux - [Studio on Windows](/quest/m0/app-windows.md) - the studio runs on Windows, not only Linux - [Wayland display and window sources](/quest/m0/wayland-sources.md) - portal-picked sources that reopen without the picker -- [ML video effects](/quest/m0/effects.md) - background removal and other ML effects, baked into their source's track - [Destinations](/quest/m0/destinations.md) - RTMP, RTMPS, and WHIP push with per-destination codecs and shared encoders, on a UI-free egress library a hosted service can reuse - [Composable scenes](/quest/m0/scenes/README.md) - scenes nest scenes and the program, any scene publishes at its own size, and mixes become audio scenes - [The join page is live](/quest/m0/join-page-live.md) - `join.studio.moq.dev` serves the join page `--join-page` defaults to diff --git a/quest/m0/effects.md b/quest/m0/effects.md deleted file mode 100644 index 04f5f82d..00000000 --- a/quest/m0/effects.md +++ /dev/null @@ -1,27 +0,0 @@ -# [M] ML video effects - -## Goal - -Sources in `moq-studio` offer ML video effects, starting with background -removal (a person cut out of a camera), applied to the source's picture before -it is drawn or published, so every viewer sees the same cut-out from the -source's own track. - -## Plan - -The rest of the effect set landed as moq-scene `Filter`s that every -composition host renders with the same WGSL and the layout carries by name and -settings. A model cannot meet that parity rule: viewers would each need the -model and a GPU to run it, and inference differs across backends. So an ML -effect is baked into its source's track per the -[composition epic](/quest/m1/compose/README.md): it belongs to a source, never -to one scene item, and sits before the source's crop and filters, as the -encoder in `rs/moq-scene/src/publish.rs` sees the capture. - -This quest picks the runtime crate (`ort`, `tract`, or `burn`) and the model -(a small, permissively licensed person segmentation model such as MediaPipe's -selfie segmentation, Apache-2.0) after measuring each candidate's per-frame -cost at 1080p and 30 fps (decided 2026-11-03, maintainer, on -[moq-dev/studio#17](https://github.com/moq-dev/studio/pull/17)). Running on the -studio's own wgpu device keeps the frame on the GPU and works toward the -[web studio](/quest/m0/web.md); a CPU runtime costs a readback per frame. diff --git a/quest/m2/publish-independently.md b/quest/m2/publish-independently.md index 17450735..1dff0be9 100644 --- a/quest/m2/publish-independently.md +++ b/quest/m2/publish-independently.md @@ -25,6 +25,11 @@ source among those on program (`Publish::set_on_program` in Moved to m2 on 2026-11-05 (maintainer, /quest-plan): no service places sources itself yet, so it waits behind m0 and m1. +A source with its background removed (`background = "remove"`) keeps the +background in its track, since H.264 carries no transparency. Publishing one +independently needs its cut-out in the track, such as moq-video encoding +alpha, or the option refused for it. + Low priority: this is for services that move webcams and overlays themselves, and none exists yet. The option lives on the source, in the scene file, so it travels with the scene. The option and its effect on the program change what a diff --git a/rs/moq-scene/Cargo.toml b/rs/moq-scene/Cargo.toml index aeb1af6f..88f941dd 100644 --- a/rs/moq-scene/Cargo.toml +++ b/rs/moq-scene/Cargo.toml @@ -74,6 +74,9 @@ thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "sync", "time"] } toml = { workspace = true } tracing = { workspace = true } +# Runs background removal's person segmentation model on the CPU. Pure Rust, so +# it builds wherever the studio does with no runtime library to download. +tract-onnx = { version = "0.23.10", default-features = false } url = { workspace = true } vello = "0.11" # Decompresses an overlay's WOFF and WOFF2 fonts to check them. Already in the diff --git a/rs/moq-scene/src/browser/run.rs b/rs/moq-scene/src/browser/run.rs index cae9a335..c242580a 100644 --- a/rs/moq-scene/src/browser/run.rs +++ b/rs/moq-scene/src/browser/run.rs @@ -64,10 +64,7 @@ pub(crate) async fn run( drop(opened); // A stopped page leaves the canvas rather than freezing on its last // picture, which for an alert or a ticker would show stale news. - *picture.lock().unwrap() = Some(Picture::Rgba { - size: moq_video::Size::new(1, 1), - pixels: vec![0; 4], - }); + picture.lock().unwrap().clear(); Err(Error::Browser(reason)) } @@ -167,7 +164,7 @@ impl Routes { Ok(None) => return, Err(reason) => return self.end(id, reason), }; - *route.picture.lock().unwrap() = Some(Picture::Texture(texture)); + route.picture.lock().unwrap().show(Picture::Texture(texture)); if !std::mem::replace(&mut route.live, true) && let Ok(mut status) = route.status.write() { diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs index 55da088f..a37da9a6 100644 --- a/rs/moq-scene/src/collection.rs +++ b/rs/moq-scene/src/collection.rs @@ -315,6 +315,12 @@ impl Collection { "video settings for `{name}`, which is not a source with a picture" ))); } + if video.background == Background::Remove && !self.sources[name].removes_background() { + return Err(Error::Invalid(format!( + "`{name}` cannot have its background removed: only a camera, display, window, image, media, \ + broadcast, browser, or replay source can, and a clone shows its source's" + ))); + } let on = format!("`{name}`"); video .crop @@ -1335,6 +1341,10 @@ pub(crate) fn validate_filters(filters: &[Filter], on: &str) -> Result<(), Error #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Video { + /// What happens behind the person in the picture, before the crop and + /// every filter. + #[serde(default, skip_serializing_if = "Background::is_kept")] + pub background: Background, /// Trimmed before every filter and each item's own crop. #[serde(default, skip_serializing_if = "Crop::is_none")] pub crop: Crop, @@ -1343,6 +1353,27 @@ pub struct Video { pub filters: Vec, } +/// What happens to the background behind a person in a source's picture. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Background { + /// Shown as captured. + #[default] + Keep, + /// Made transparent, leaving the person a segmentation model finds in each + /// picture. Each picture draws once its own cut-out is ready, a frame or + /// more late, skipping pictures while the model is behind. The + /// source's own track keeps the background, since its encoding carries no + /// transparency. + Remove, +} + +impl Background { + fn is_kept(&self) -> bool { + *self == Self::Keep + } +} + /// A rectangle in canvas pixels, from the top-left corner. #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -1513,6 +1544,22 @@ impl Source { !matches!(self, Self::Clone { .. }) } + /// Whether the source's picture can have its background removed: a + /// picture of its own that may show a person. + pub fn removes_background(&self) -> bool { + matches!( + self, + Self::Camera { .. } + | Self::Display { .. } + | Self::Window { .. } + | Self::Image { .. } + | Self::Media { .. } + | Self::Broadcast { .. } + | Self::Browser { .. } + | Self::Replay + ) + } + /// Whether the source feeds the program audio. pub fn has_audio(&self) -> bool { matches!( @@ -2103,6 +2150,42 @@ path = "sting.webm" } } + /// A background is removed only from a picture of the source's own that + /// may show a person. + #[test] + fn removes_a_background_only_from_a_picture_of_its_own() { + let text = r#" +version = 1 + +[canvas] +width = 1280 +height = 720 +framerate = 30 + +[sources.cam] +kind = "camera" + +[video.cam] +background = "remove" +"#; + let collection = Collection::from_toml(text).unwrap(); + assert_eq!(collection.video["cam"].background, Background::Remove); + assert_eq!( + Collection::from_toml(&collection.to_toml().unwrap()).unwrap(), + collection + ); + + let color = Source::Color { + color: Rgba([0, 0, 0, 255]), + }; + for source in [Source::Clone { source: "cam".into() }, color] { + let mut collection = collection.clone(); + collection.sources.insert("other".into(), source.clone()); + collection.video.insert("other".into(), collection.video["cam"].clone()); + assert!(matches!(collection.validate(), Err(Error::Invalid(_))), "{source:?}"); + } + } + #[test] fn refuses_a_filter_out_of_range() { let red = Rgba([255, 0, 0, 255]); diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index 61f0d56b..931842d1 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -10,9 +10,10 @@ use moq_video::render::{Renderer, wgpu}; use crate::Error; use crate::collection::{ - Canvas, Collection, Crop, Direction, Filter, Item, Layer as Place, Rect, Settled, Source, Transition, + Background, Canvas, Collection, Crop, Direction, Filter, Item, Layer as Place, Rect, Settled, Source, Transition, }; use crate::effect::{self, Effects, Frame, Lut, Picture, Prepared}; +use crate::matte::Matte; use crate::overlay::{Overlay, Overlays}; use crate::source::{self, Feed, Sources, Status}; @@ -149,7 +150,21 @@ struct Layer { source: Source, /// Converts video frames, created on the first one. renderer: Option, - /// The newest picture; `None` until the first arrives. + /// The newest picture as the source delivered it, and whether it is newer + /// than what `picture` shows or `matte` started on. + raw: Option, + fresh: bool, + /// Whether the source cleared its picture since `matte` last looked, such + /// as a media source stopped or played again, so nothing from before may + /// show. + cleared: bool, + /// Cuts the person out of each picture, while the source's background is + /// removed, or why it cannot. A failure stays until the background is kept + /// or the source edited, since retrying the model would fail the same way, + /// and is reported over `error`. + matte: Option>, + /// The newest picture drawn; `None` until the first arrives, and while the + /// first cut-out is under way. picture: Option, /// A LUT source's table; `None` until it is read. lut: Option, @@ -159,6 +174,71 @@ struct Layer { error: Option, } +impl Layer { + /// Bring the picture drawn up to date with the newest one delivered, + /// through its matte while `background` is removed. + fn develop( + &mut self, + name: &str, + background: Background, + device: &wgpu::Device, + queue: &wgpu::Queue, + effects: &Effects, + ) { + let cleared = std::mem::take(&mut self.cleared); + if background == Background::Keep { + // One switched back shows the picture it held back. + if self.matte.take().is_none() && !self.fresh { + return; + } + self.fresh = false; + self.picture = self.raw.as_ref().map(|raw| effects.picture(raw)); + return; + } + + if self.matte.is_none() { + // What was drawn shows the background, so nothing is until the + // first cut-out. + self.picture = None; + self.fresh = self.raw.is_some(); + self.matte = Some(Matte::new(device, queue).map_err(|err| err.to_string())); + } else if let Some(Err(_)) = &self.matte { + return; + } + // The cut-out in hand is from before the clear, such as a media + // source's last play, so it never shows. + if cleared && let Some(Ok(matte)) = &mut self.matte { + matte.discard(); + self.picture = None; + } + let failed = match self.matte.as_mut().expect("set above") { + Err(err) => Some(err.clone()), + Ok(matte) => match matte.finish() { + Some(Ok(cut)) => { + self.picture = Some(effects.picture(cut)); + None + } + Some(Err(err)) => Some(err.to_string()), + None => None, + }, + }; + if let Some(err) = failed { + tracing::warn!(source = %name, %err, "cannot remove the background"); + self.picture = None; + self.matte = Some(Err(err)); + return; + } + if let Some(Ok(matte)) = &mut self.matte + && self.fresh + && !matte.busy() + && let Some(raw) = &self.raw + { + matte.start(raw); + self.fresh = false; + } + } +} + /// The pictures and tables of the compositor's layers, for filters to read. struct Library<'a>(&'a HashMap); @@ -285,9 +365,14 @@ impl Compositor { self.program_scene.as_deref() } - /// Why `source`'s newest picture could not be drawn, until one is. + /// Why `source`'s newest picture could not be drawn, until one is, or why + /// its background cannot be removed. pub fn error(&self, source: &str) -> Option<&str> { - self.layers.get(source)?.error.as_deref() + let layer = self.layers.get(source)?; + match &layer.matte { + Some(Err(err)) => Some(err), + _ => layer.error.as_deref(), + } } /// Whether a transition is under way. @@ -484,72 +569,101 @@ impl Compositor { let Some(feed) = sources.get(name).filter(|feed| feed.source() == source) else { continue; }; - let Some(picture) = feed.take() else { - continue; + let taken = feed.take(); + let mut layer = match self.layers.remove(name) { + Some(layer) => layer, + None if taken.is_some() => Layer { + source: source.clone(), + renderer: None, + raw: None, + fresh: false, + cleared: false, + matte: None, + picture: None, + lut: None, + overlay: None, + error: None, + }, + None => continue, }; - let layer = self.layers.entry(name.clone()).or_insert_with(|| Layer { - source: source.clone(), - renderer: None, - picture: None, - lut: None, - overlay: None, - error: None, - }); - // Every picture lands premultiplied, so scaling it mixes no - // transparent texel's color: a page arrives so, moq-video renders - // a frame opaque, a still is premultiplied here, and an overlay's - // paint as it lands. - let texture = match picture { - source::Picture::Frame(frame) => { - let renderer = match layer.renderer.as_mut() { - Some(renderer) => Ok(renderer), - None => Renderer::new(&self.device, &self.queue, Default::default()) - .map(|renderer| layer.renderer.insert(renderer)), - }; - renderer - .and_then(|renderer| renderer.render(&frame)) - .map_err(Error::from) - } - source::Picture::Rgba { size, mut pixels } => { - premultiply(&mut pixels); - upload(&self.device, &self.queue, size, &pixels) - } - source::Picture::Texture(texture) => Ok(texture), - source::Picture::Lut(cube) => { - layer.lut = Some(self.effects.lut(&cube)); - continue; + if let Some(taken) = taken { + if taken.cleared { + // Nothing from before the clear may show, even if the picture + // after it fails to draw. + layer.cleared = true; + layer.raw = None; + layer.fresh = true; } - source::Picture::Overlay(content) => { - let Source::Overlay { width, height, .. } = source else { - unreachable!("only an overlay source shows an overlay") - }; - let overlays = match &mut self.overlays { - Some(overlays) => Ok(overlays), - None => Overlays::new(&self.device, &self.queue).map(|overlays| self.overlays.insert(overlays)), - }; - match overlays { - Ok(overlays) => { - let overlay = overlays.open(content, *width, *height, now); - layer.picture = Some(self.effects.picture(overlay.texture())); - layer.overlay = Some(overlay); - } - Err(err) => layer.error = Some(err.to_string()), + self.take(name, source, &mut layer, taken.picture, now); + } + let background = collection + .video + .get(name) + .map(|video| video.background) + .unwrap_or_default(); + layer.develop(name, background, &self.device, &self.queue, &self.effects); + self.layers.insert(name.clone(), layer); + } + } + + /// Put `picture` from source `name` on the GPU as `layer`'s newest. An + /// overlay that arrives now starts its CSS timeline at `now`. + fn take(&mut self, name: &str, source: &Source, layer: &mut Layer, picture: source::Picture, now: Instant) { + // Every picture lands premultiplied, so scaling it mixes no + // transparent texel's color: a page arrives so, moq-video renders + // a frame opaque, a still is premultiplied here, and an overlay's + // paint as it lands. + let texture = match picture { + source::Picture::Frame(frame) => { + let renderer = match layer.renderer.as_mut() { + Some(renderer) => Ok(renderer), + None => Renderer::new(&self.device, &self.queue, Default::default()) + .map(|renderer| layer.renderer.insert(renderer)), + }; + renderer + .and_then(|renderer| renderer.render(&frame)) + .map_err(Error::from) + } + source::Picture::Rgba { size, mut pixels } => { + premultiply(&mut pixels); + upload(&self.device, &self.queue, size, &pixels) + } + source::Picture::Texture(texture) => Ok(texture), + source::Picture::Lut(cube) => { + layer.lut = Some(self.effects.lut(&cube)); + return; + } + source::Picture::Overlay(content) => { + let Source::Overlay { width, height, .. } = source else { + unreachable!("only an overlay source shows an overlay") + }; + let overlays = match &mut self.overlays { + Some(overlays) => Ok(overlays), + None => Overlays::new(&self.device, &self.queue).map(|overlays| self.overlays.insert(overlays)), + }; + match overlays { + Ok(overlays) => { + let overlay = overlays.open(content, *width, *height, now); + layer.picture = Some(self.effects.picture(overlay.texture())); + layer.overlay = Some(overlay); } - continue; + Err(err) => layer.error = Some(err.to_string()), } - }; + return; + } + }; - match texture { - Ok(texture) => { - layer.error = None; - layer.picture = Some(self.effects.picture(&texture)); - } - Err(err) => { - if layer.error.is_none() { - tracing::warn!(source = %name, %err, "cannot draw source"); - } - layer.error = Some(err.to_string()); + match texture { + Ok(texture) => { + layer.error = None; + layer.raw = Some(texture); + layer.fresh = true; + } + Err(err) => { + if layer.error.is_none() { + tracing::warn!(source = %name, %err, "cannot draw source"); } + layer.error = Some(err.to_string()); } } } @@ -1173,7 +1287,7 @@ mod tests { use std::time::Duration; use super::*; - use crate::collection::{Blend, Extra, MaskMode, Rgba, Scene}; + use crate::collection::{Blend, Extra, MaskMode, Rgba, Scene, Video}; use crate::source::test::{context, listen, local, start}; /// A fade over a second. @@ -2243,6 +2357,124 @@ mod tests { .await; } + /// A source with its background removed draws only the person, each + /// picture once the model has it, and its whole picture again once kept. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn removes_the_background() { + local(async { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("person.jpg"), include_bytes!("test_data/person.jpg")).unwrap(); + let mut collection = Collection::new(); + collection.canvas.width = 256; + collection.canvas.height = 144; + collection.sources.insert( + "person".into(), + Source::Image { + path: "person.jpg".into(), + }, + ); + let removed = Video { + background: Background::Remove, + ..Video::default() + }; + collection.video.insert("person".into(), removed.clone()); + collection.scenes[0].items = vec![Item::new("person", Rect::fill(&collection.canvas))]; + collection.validate().unwrap(); + let scene = collection.scenes[0].name.clone(); + + let mut studio = Studio::with_files(collection, dir).await; + studio.compositor.set_preview(Some(scene)); + studio.compositor.cut(&studio.sources); + let at = |rgba: &[u8], x: usize, y: usize| -> [u8; 4] { + let at = (y * 256 + x) * 4; + rgba[at..at + 4].try_into().unwrap() + }; + let face = (128, 50); + let flag = (30, 100); + let cut = |studio: &mut Studio| { + studio.render(Instant::now()); + let program = studio.program(); + // Nothing shows until the cut-out is ready, not even the background. + assert_eq!(at(&program, face.0, face.1), BLACK); + let layer = studio.compositor.layers.get_mut("person").unwrap(); + let Some(Ok(matte)) = &mut layer.matte else { + panic!("not removing the background") + }; + matte.wait(); + studio.render(Instant::now()); + let program = studio.program(); + assert_ne!(at(&program, face.0, face.1), BLACK); + assert_eq!(at(&program, flag.0, flag.1), BLACK); + }; + cut(&mut studio); + assert!(studio.compositor.error("person").is_none()); + + studio.collection.video.remove("person"); + studio.render(Instant::now()); + let program = studio.program(); + assert_ne!(at(&program, flag.0, flag.1), BLACK); + + studio.collection.video.insert("person".into(), removed); + cut(&mut studio); + }) + .await; + } + + /// A source that clears its picture, such as a media source played + /// again, never shows the cut-out of a picture from before, even when its + /// next picture overtakes the clear. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_clear_drops_the_cut_out_in_hand() { + let (device, queue) = gpu().await; + let effects = Effects::new(&device, &queue, 256, 144); + let person = image::load_from_memory(include_bytes!("test_data/person.jpg")).unwrap(); + let picture = |width: u32, height: u32| { + let rgba = person.resize_exact(width, height, image::imageops::FilterType::Triangle); + upload( + &device, + &queue, + moq_video::Size::new(width, height), + rgba.to_rgba8().as_raw(), + ) + .unwrap() + }; + let mut layer = Layer { + source: Source::Replay, + renderer: None, + raw: Some(picture(256, 144)), + fresh: true, + cleared: false, + matte: None, + picture: None, + lut: None, + overlay: None, + error: None, + }; + let develop = |layer: &mut Layer| layer.develop("replay", Background::Remove, &device, &queue, &effects); + let wait = |layer: &mut Layer| { + let Some(Ok(matte)) = &mut layer.matte else { + panic!("not removing the background") + }; + matte.wait(); + }; + develop(&mut layer); + assert!(layer.picture.is_none(), "until the cut-out"); + wait(&mut layer); + + // The next play's first picture, smaller, replaced the clear unseen. + layer.raw = Some(picture(128, 72)); + layer.fresh = true; + layer.cleared = true; + develop(&mut layer); + assert!(layer.picture.is_none(), "the old play's cut-out is dropped"); + wait(&mut layer); + develop(&mut layer); + assert_eq!(layer.picture.as_ref().map(Picture::size), Some((128, 72))); + assert!(layer.error.is_none()); + } + /// The program of `collection` with scene "red" cut to it, its files in `dir`. async fn drawn(collection: Collection, dir: tempfile::TempDir) -> Vec { let mut studio = Studio::with_files(collection, dir).await; diff --git a/rs/moq-scene/src/lib.rs b/rs/moq-scene/src/lib.rs index f71e24a5..c9c9b458 100644 --- a/rs/moq-scene/src/lib.rs +++ b/rs/moq-scene/src/lib.rs @@ -20,7 +20,9 @@ //! textures on the caller's `wgpu` device, one pair per canvas from one //! upload of each source's picture, cutting or running a [`Transition`] //! between them on every canvas together, and plays each item's -//! [`Keyframe`]s from when its scene went on. `moq_video::render::export` hands the +//! [`Keyframe`]s from when its scene went on. A source whose [`Background`] +//! is removed is cut out first, by a segmentation model on the CPU. +//! `moq_video::render::export` hands the //! program to an encoder, on the GPU where the platform allows, and a //! [`Scaler`] draws it smaller for a smaller rendition. //! - [`publish`] gives each local source its own track, in a broadcast the @@ -38,6 +40,7 @@ mod compositor; mod edit; mod effect; mod lut; +mod matte; mod media; mod overlay; mod playout; @@ -48,8 +51,8 @@ pub mod source; pub use clip::Clip; pub use collection::{ - Audio, Blend, Canvas, Collection, Crop, Direction, Ease, Extra, Filter, Item, Keyframe, Layer, MaskMode, Mixes, - Rect, Rgba, Scene, Source, Transition, Video, + Audio, Background, Blend, Canvas, Collection, Crop, Direction, Ease, Extra, Filter, Item, Keyframe, Layer, + MaskMode, Mixes, Rect, Rgba, Scene, Source, Transition, Video, }; pub use compositor::{Compositor, FORMAT}; pub use edit::{Change, Draft, Edit, History}; @@ -131,4 +134,9 @@ pub enum Error { /// failed to draw. #[error("overlay: {0}")] Overlay(String), + + /// Removing a source's background failed: running the model, or reading + /// a picture back for it. + #[error("background removal: {0}")] + Matte(String), } diff --git a/rs/moq-scene/src/matte.rs b/rs/moq-scene/src/matte.rs new file mode 100644 index 00000000..9b8f9803 --- /dev/null +++ b/rs/moq-scene/src/matte.rs @@ -0,0 +1,492 @@ +//! Background removal: a person segmentation model finds who is in each of a +//! source's pictures, and the rest of the picture turns transparent. +//! +//! The compositor shrinks each picture on the GPU to the model's size and +//! reads that back. A thread of the matte's own runs the model on the CPU, +//! with tract, and the compositor cuts the held full-size picture out with +//! the result. Each picture is drawn with its own matte, a frame or more +//! late, never with an older one; pictures that arrive while the model works +//! are skipped for the newest. + +use std::collections::HashMap; +use std::sync::{Arc, OnceLock, mpsc}; + +use moq_video::render::wgpu; +use tract_onnx::prelude::*; + +use crate::Error; +use crate::compositor::FORMAT; + +/// MediaPipe's selfie segmentation, landscape variant: see `model/README.md`. +const MODEL: &[u8] = include_bytes!("model/selfie_segmentation_landscape.onnx"); + +/// The model's input and output size: RGB in, a person's alpha out. +const WIDTH: u32 = 256; +const HEIGHT: u32 = 144; + +type Model = Arc; + +/// The model, parsed and optimized once per process, on the first matte's +/// thread. +fn model() -> Result<&'static Model, String> { + static MODEL_PLAN: OnceLock> = OnceLock::new(); + let load = || -> TractResult { + let model = tract_onnx::onnx().model_for_read(&mut &MODEL[..])?.into_typed()?; + let batch = model.symbols.sym("batch_size"); + model + .set_symbols(&HashMap::from([(batch, TDim::from(1))]))? + .into_optimized()? + .into_runnable() + }; + MODEL_PLAN + .get_or_init(|| load().map_err(|err| format!("the segmentation model: {err}"))) + .as_ref() + .map_err(Clone::clone) +} + +/// The alpha of each pixel of a `WIDTH` by `HEIGHT` RGBA picture: 255 where +/// it shows a person, 0 where it shows what is behind them. +fn segment(model: &Model, rgba: &[u8]) -> Result, String> { + let plane = (WIDTH * HEIGHT) as usize; + let mut input = vec![0.0f32; 3 * plane]; + for (at, pixel) in rgba.as_chunks::<4>().0.iter().enumerate() { + for (channel, value) in pixel[..3].iter().enumerate() { + input[channel * plane + at] = f32::from(*value) / 255.0; + } + } + let run = || -> TractResult> { + let input = Tensor::from_shape(&[1, 3, HEIGHT as usize, WIDTH as usize], &input)?; + let output = model.run(tvec!(input.into()))?; + let alpha = output[0].to_plain_array_view::()?; + Ok(alpha + .iter() + .map(|alpha| (alpha.clamp(0.0, 1.0) * 255.0).round() as u8) + .collect()) + }; + run().map_err(|err| format!("the segmentation model: {err}")) +} + +/// Cuts a person out of one source's pictures. Dropping it stops its thread +/// once the picture in hand is done. +pub(crate) struct Matte { + device: wgpu::Device, + queue: wgpu::Queue, + copy: wgpu::RenderPipeline, + shrink: wgpu::RenderPipeline, + cut: wgpu::RenderPipeline, + picture_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + /// The picture at the model's size, read back into `readback`. + small: wgpu::Texture, + readback: wgpu::Buffer, + mask: wgpu::Texture, + mask_bind: wgpu::BindGroup, + /// The newest picture started, and its cut-out, at its size. + held: Option, + jobs: mpsc::Sender, + results: mpsc::Receiver, String>>, + /// Whether the thread has a picture, which also has `readback` mapped. + busy: bool, + /// Whether the picture the thread has was overtaken, so its cut-out is + /// thrown away. + stale: bool, + /// The thread's result, once a test has waited for it. + #[cfg(test)] + ready: Option, String>>, +} + +struct Held { + picture: wgpu::Texture, + bind: wgpu::BindGroup, + cut: wgpu::Texture, +} + +impl Matte { + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Result { + let shader = device.create_shader_module(wgpu::include_wgsl!("matte.wgsl")); + let texture_entry = |binding| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }; + let picture_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene matte picture"), + entries: &[ + texture_entry(0), + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let mask_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene matte mask"), + entries: &[texture_entry(0)], + }); + let pipeline = |entry: &str, layouts: &[Option<&wgpu::BindGroupLayout>]| { + let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("moq-scene matte"), + bind_group_layouts: layouts, + immediate_size: 0, + }); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("moq-scene matte"), + layout: Some(&layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some(entry), + targets: &[Some(FORMAT.into())], + compilation_options: Default::default(), + }), + primitive: Default::default(), + depth_stencil: None, + multisample: Default::default(), + multiview_mask: None, + cache: None, + }) + }; + let copy = pipeline("fs_copy", &[Some(&picture_layout)]); + let shrink = pipeline("fs_shrink", &[Some(&picture_layout)]); + let cut = pipeline("fs_cut", &[Some(&picture_layout), Some(&mask_layout)]); + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("moq-scene matte"), + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + let size = wgpu::Extent3d { + width: WIDTH, + height: HEIGHT, + depth_or_array_layers: 1, + }; + let small = device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene matte input"), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + // A row of the model's input is 1024 bytes, already aligned for a copy. + let readback = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("moq-scene matte input"), + size: u64::from(WIDTH * HEIGHT * 4), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let mask = device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene matte"), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let mask_bind = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene matte mask"), + layout: &mask_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&mask.create_view(&Default::default())), + }], + }); + + let (jobs, inbox) = mpsc::channel(); + let (outbox, results) = mpsc::channel(); + let thread = Thread { + device: device.clone(), + readback: readback.clone(), + }; + std::thread::Builder::new() + .name("moq-scene matte".into()) + .spawn(move || thread.run(inbox, outbox)) + .map_err(|err| Error::Unsupported(format!("cannot start background removal: {err}")))?; + + Ok(Self { + device: device.clone(), + queue: queue.clone(), + copy, + shrink, + cut, + picture_layout, + sampler, + small, + readback, + mask, + mask_bind, + held: None, + jobs, + results, + busy: false, + stale: false, + #[cfg(test)] + ready: None, + }) + } + + /// Whether a picture is still being cut out, so another cannot start. + pub fn busy(&self) -> bool { + self.busy + } + + /// Start cutting `picture` out: hold a copy, since its texture may be + /// drawn over, and hand the thread a shrunk one. + pub fn start(&mut self, picture: &wgpu::Texture) { + assert!(!self.busy, "a matte cuts one picture at a time"); + let size = picture.size(); + let held = match self.held.take() { + Some(held) if held.picture.size() == size => held, + _ => self.hold(size), + }; + + let bind = self.bind(picture); + let mut encoder = self.device.create_command_encoder(&Default::default()); + self.pass(&mut encoder, &self.copy, &held.picture, &[&bind]); + self.pass(&mut encoder, &self.shrink, &self.small, &[&held.bind]); + encoder.copy_texture_to_buffer( + self.small.as_image_copy(), + wgpu::TexelCopyBufferInfo { + buffer: &self.readback, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(WIDTH * 4), + rows_per_image: None, + }, + }, + self.small.size(), + ); + let submission = self.queue.submit([encoder.finish()]); + self.held = Some(held); + // A thread that is gone reports so through `finish`. + let _ = self.jobs.send(submission); + self.busy = true; + } + + /// Throw away the cut-out of the picture in hand, such as when the source + /// clears its picture, so it never shows after what overtook it. + pub fn discard(&mut self) { + self.stale = self.busy; + } + + /// The cut-out of the picture last started, once the thread has its + /// matte; `None` while it works on it, when nothing is started, or when it + /// was discarded. + pub fn finish(&mut self) -> Option> { + if !self.busy { + return None; + } + #[cfg(test)] + if let Some(result) = self.ready.take() { + return self.cut(result); + } + let result = match self.results.try_recv() { + Ok(result) => result, + Err(mpsc::TryRecvError::Empty) => return None, + Err(mpsc::TryRecvError::Disconnected) => Err("the background removal thread stopped".into()), + }; + self.cut(result) + } + + /// Block until the thread is done with the picture last started, for + /// the next [`finish`](Self::finish) to cut out. + #[cfg(test)] + pub fn wait(&mut self) { + assert!(self.busy, "nothing is started"); + let result = self + .results + .recv() + .unwrap_or_else(|_| Err("the background removal thread stopped".into())); + self.ready = Some(result); + } + + fn cut(&mut self, alpha: Result, String>) -> Option> { + self.busy = false; + if std::mem::take(&mut self.stale) { + return None; + } + Some(self.paint(alpha)) + } + + fn paint(&mut self, alpha: Result, String>) -> Result<&wgpu::Texture, Error> { + let alpha = alpha.map_err(Error::Matte)?; + self.queue.write_texture( + self.mask.as_image_copy(), + &alpha, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(WIDTH), + rows_per_image: None, + }, + self.mask.size(), + ); + let held = self.held.as_ref().expect("a started picture is held"); + let mut encoder = self.device.create_command_encoder(&Default::default()); + self.pass(&mut encoder, &self.cut, &held.cut, &[&held.bind, &self.mask_bind]); + self.queue.submit([encoder.finish()]); + Ok(&held.cut) + } + + /// The textures a picture of `size` is held and cut out in. + fn hold(&self, size: wgpu::Extent3d) -> Held { + let texture = |label| { + self.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) + }; + let picture = texture("moq-scene matte held"); + Held { + bind: self.bind(&picture), + picture, + cut: texture("moq-scene matte cut"), + } + } + + fn bind(&self, texture: &wgpu::Texture) -> wgpu::BindGroup { + self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene matte picture"), + layout: &self.picture_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&texture.create_view(&Default::default())), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }) + } + + /// Draw `pipeline` over the whole of `target`. + fn pass( + &self, + encoder: &mut wgpu::CommandEncoder, + pipeline: &wgpu::RenderPipeline, + target: &wgpu::Texture, + binds: &[&wgpu::BindGroup], + ) { + let view = target.create_view(&Default::default()); + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("moq-scene matte"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + ..Default::default() + }); + pass.set_pipeline(pipeline); + for (index, bind) in binds.iter().enumerate() { + pass.set_bind_group(index as u32, *bind, &[]); + } + pass.draw(0..3, 0..1); + } +} + +/// What a matte's thread holds: the device to wait on, and the buffer the +/// shrunk picture lands in. +struct Thread { + device: wgpu::Device, + readback: wgpu::Buffer, +} + +impl Thread { + /// Run the model on each shrunk picture, once the GPU has it, until the + /// matte is dropped. + fn run(self, jobs: mpsc::Receiver, results: mpsc::Sender, String>>) { + for submission in jobs { + let alpha = self.read(submission).and_then(|rgba| segment(model()?, &rgba)); + if results.send(alpha).is_err() { + return; + } + } + } + + /// The shrunk picture, once `submission` has copied it into the buffer. + fn read(&self, submission: wgpu::SubmissionIndex) -> Result, String> { + let (mapped, map) = mpsc::channel(); + self.readback.map_async(wgpu::MapMode::Read, .., move |result| { + let _ = mapped.send(result); + }); + self.device + .poll(wgpu::PollType::Wait { + submission_index: Some(submission), + timeout: None, + }) + .map_err(|err| format!("waiting for the GPU: {err}"))?; + map.recv() + .map_err(|_| "the GPU dropped the background removal buffer".to_string())? + .map_err(|err| format!("reading the picture back: {err}"))?; + let pixels = self + .readback + .slice(..) + .get_mapped_range() + .map_err(|err| format!("reading the picture back: {err}"))? + .to_vec(); + self.readback.unmap(); + Ok(pixels) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The model finds the person in a portrait and nothing beside them. + /// `person.jpg` is Ilan Ramon's NASA portrait (public domain), cropped to + /// head and shoulders and shrunk to the model's size. + #[test] + fn finds_the_person() { + let image = image::load_from_memory(include_bytes!("test_data/person.jpg")) + .unwrap() + .to_rgba8(); + assert_eq!(image.dimensions(), (WIDTH, HEIGHT)); + let alpha = segment(model().unwrap(), image.as_raw()).unwrap(); + let at = |x: u32, y: u32| alpha[(y * WIDTH + x) as usize]; + // The face, and the suit below it. + assert!(at(128, 50) > 240, "{}", at(128, 50)); + assert!(at(128, 130) > 240, "{}", at(128, 130)); + // The flag on the left and the shuttle model on the right. + for (x, y) in [(10, 10), (30, 100), (245, 10), (225, 70)] { + assert!(at(x, y) < 15, "({x}, {y}): {}", at(x, y)); + } + } + + /// A plain picture shows nobody. + #[test] + fn finds_nobody_in_a_wall() { + let wall = [180u8, 170, 160, 255].repeat((WIDTH * HEIGHT) as usize); + let alpha = segment(model().unwrap(), &wall).unwrap(); + assert!(alpha.iter().all(|alpha| *alpha < 15)); + } +} diff --git a/rs/moq-scene/src/matte.wgsl b/rs/moq-scene/src/matte.wgsl new file mode 100644 index 00000000..9ae74533 --- /dev/null +++ b/rs/moq-scene/src/matte.wgsl @@ -0,0 +1,43 @@ +// Background removal's GPU half: hold a picture, shrink it for the model, and +// cut it out with the model's matte. + +@group(0) @binding(0) var picture: texture_2d; +@group(0) @binding(1) var linear: sampler; +@group(1) @binding(0) var matte: texture_2d; + +struct Varyings { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +// One triangle covering the whole target. +@vertex fn vs_main(@builtin(vertex_index) index: u32) -> Varyings { + let corner = vec2(f32((index << 1u) & 2u), f32(index & 2u)); + var out: Varyings; + out.position = vec4(corner * vec2(2.0, -2.0) + vec2(-1.0, 1.0), 0.0, 1.0); + out.uv = corner; + return out; +} + +// Same size, so each pixel samples its own texel's center. +@fragment fn fs_copy(in: Varyings) -> @location(0) vec4 { + return textureSample(picture, linear, in.uv); +} + +// Sixteen bilinear taps spread over the target pixel's footprint: a box over +// 8x8 texels, enough for 1080p shrunk to the model's 256x144. +@fragment fn fs_shrink(in: Varyings) -> @location(0) vec4 { + let step = fwidth(in.uv) * 0.25; + var sum = vec4(0.0); + for (var y = 0; y < 4; y++) { + for (var x = 0; x < 4; x++) { + sum += textureSample(picture, linear, in.uv + (vec2(f32(x), f32(y)) - 1.5) * step); + } + } + return sum / 16.0; +} + +// The picture is premultiplied, so scaling every channel by the matte keeps it so. +@fragment fn fs_cut(in: Varyings) -> @location(0) vec4 { + return textureSample(picture, linear, in.uv) * textureSample(matte, linear, in.uv).r; +} diff --git a/rs/moq-scene/src/model/README.md b/rs/moq-scene/src/model/README.md new file mode 100644 index 00000000..60ef68a1 --- /dev/null +++ b/rs/moq-scene/src/model/README.md @@ -0,0 +1,9 @@ +# Models + +`selfie_segmentation_landscape.onnx` is MediaPipe's selfie segmentation, +landscape variant: 256x144 RGB in `[0, 1]` in, a 256x144 person alpha out. +It is Google's model under the Apache License 2.0, as converted to ONNX by +[onnx-community/mediapipe_selfie_segmentation_landscape](https://huggingface.co/onnx-community/mediapipe_selfie_segmentation_landscape) +at commit `2497d5bec26c626c7b3c4edc6e1fefc21b64f6c3` (`onnx/model.onnx`, +SHA-256 `7a0adcfdb1715d3b0ff0f61486d8a181c33f4a208343a5abfccbc6540e872d24`), +unmodified. diff --git a/rs/moq-scene/src/model/selfie_segmentation_landscape.onnx b/rs/moq-scene/src/model/selfie_segmentation_landscape.onnx new file mode 100644 index 00000000..2671dd4c Binary files /dev/null and b/rs/moq-scene/src/model/selfie_segmentation_landscape.onnx differ diff --git a/rs/moq-scene/src/publish.rs b/rs/moq-scene/src/publish.rs index 75443851..9ff46f1f 100644 --- a/rs/moq-scene/src/publish.rs +++ b/rs/moq-scene/src/publish.rs @@ -795,7 +795,7 @@ mod tests { /// threads the runtime cannot see, such as a decoder's or an encoder's, /// finishing within any span of the clock. async fn pictures(feed: &crate::source::Feed, first: kio::clock::Instant, count: u32) { - let shown = || matches!(feed.take(), Some(crate::source::Picture::Frame(_))).then_some(()); + let shown = || matches!(feed.next_picture(), Some(crate::source::Picture::Frame(_))).then_some(()); for index in 0..count { until_time(first + std::time::Duration::from_millis(100) * index).await; until(shown).await; diff --git a/rs/moq-scene/src/source.rs b/rs/moq-scene/src/source.rs index f9acbf3d..eae2d792 100644 --- a/rs/moq-scene/src/source.rs +++ b/rs/moq-scene/src/source.rs @@ -55,9 +55,47 @@ pub(crate) enum Picture { Overlay(crate::overlay::Content), } -/// The one picture a source holds for the compositor. A newer picture replaces -/// an undrawn one, so a slow canvas skips pictures instead of queuing them. -pub(crate) type Mailbox = Arc>>; +/// The one picture a source holds for the compositor, and whether it cleared +/// its picture since the compositor last took one. A newer picture replaces an +/// undrawn one, so a slow canvas skips pictures instead of queuing them, but +/// not the clear before it. +#[derive(Default)] +pub(crate) struct Slot { + picture: Option, + cleared: bool, +} + +impl Slot { + /// Hold `picture` for the compositor, replacing one not yet drawn. + pub fn show(&mut self, picture: Picture) { + self.picture = Some(picture); + } + + /// Show nothing, ending what the source showed before, such as a play + /// stopped or a page closed. + pub fn clear(&mut self) { + self.picture = Some(clear()); + self.cleared = true; + } + + fn take(&mut self) -> Option { + let picture = self.picture.take()?; + Some(Taken { + picture, + cleared: std::mem::take(&mut self.cleared), + }) + } +} + +pub(crate) type Mailbox = Arc>; + +/// A picture the compositor takes from a source. +pub(crate) struct Taken { + pub picture: Picture, + /// Whether the source cleared its picture since the last one taken, even + /// if `picture` came after the clear: nothing from before it may show. + pub cleared: bool, +} /// When a source last delivered a picture or, for a microphone or system /// audio, a buffer of sound. `None` before its first. @@ -457,7 +495,7 @@ impl Feed { // Cleared before the driver hears of it, so the next frame drawn, such // as the first of a scene going on program, shows nothing rather than // where the last play was, and the new first picture replaces it. - *self.picture.lock().unwrap() = Some(clear()); + self.picture.lock().unwrap().clear(); let cue = Cue { play: latest.run.play, stopped: false, @@ -477,7 +515,7 @@ impl Feed { let mut latest = self.runs.lock().unwrap(); latest.run.ended.get_or_insert_with(kio::clock::now); latest.stopped = true; - *self.picture.lock().unwrap() = Some(clear()); + self.picture.lock().unwrap().clear(); // Kept even with the driver gone, which plays nothing anyway. cues.send_replace(Cue { play: latest.run.play, @@ -561,9 +599,15 @@ impl Feed { } /// The newest picture not yet drawn. - pub(crate) fn take(&self) -> Option { + pub(crate) fn take(&self) -> Option { self.picture.lock().unwrap().take() } + + /// The newest picture not yet drawn, without whether it follows a clear. + #[cfg(test)] + pub(crate) fn next_picture(&self) -> Option { + self.take().map(|taken| taken.picture) + } } /// The work behind one [`Feed`]: opening the source and delivering what it @@ -732,7 +776,7 @@ impl Driver { } fn show(&self, picture: Picture) { - *self.picture.lock().unwrap() = Some(picture); + self.picture.lock().unwrap().show(picture); *self.delivered.lock().unwrap() = Some(kio::clock::now()); set(&self.status, Status::Live); } @@ -1194,7 +1238,7 @@ impl Driver { /// Play each replay [`Feed::replay`] hands over, showing nothing between /// them. A new one cuts off the one playing. async fn replays(&self, mut plays: watch::Receiver, listing: Option<&Listing<'_>>) -> Result<(), Error> { - *self.picture.lock().unwrap() = Some(clear()); + self.picture.lock().unwrap().clear(); set(&self.status, Status::Live); let mut playing: Option> = None; @@ -1211,13 +1255,13 @@ impl Driver { // already, and clearing it again would show before the new // one's first picture even when that is due at once. if playing.is_some() { - *self.picture.lock().unwrap() = Some(clear()); + self.picture.lock().unwrap().clear(); } playing = play.map(|play| Box::pin(self.replay(play, listing)) as Playing<'_>); } result = async { playing.as_mut().expect("guarded").await }, if playing.is_some() => { playing = None; - *self.picture.lock().unwrap() = Some(clear()); + self.picture.lock().unwrap().clear(); set(&self.status, match result { Ok(()) => Status::Live, Err(err) => Status::Failed(err.to_string()), @@ -1267,7 +1311,7 @@ impl Driver { moq_video::decode::Read::Gap { timestamp, .. } | moq_video::decode::Read::End { end: timestamp } => { kio::clock::sleep_until(at(timestamp.clamp(take.start, take.end))).await; - *self.picture.lock().unwrap() = Some(clear()); + self.picture.lock().unwrap().clear(); if let Some(video) = &mut own.track && let Err(err) = video.discontinuity() { @@ -2044,6 +2088,26 @@ mod tests { collection } + /// A picture that replaces a clear before the compositor takes it still + /// says the source cleared, once. + #[test] + fn a_clear_outlives_the_picture_after_it() { + let mut slot = Slot::default(); + slot.show(clear()); + assert!(!slot.take().unwrap().cleared); + slot.clear(); + slot.show(Picture::Rgba { + size: moq_video::Size::new(2, 1), + pixels: vec![255; 8], + }); + let taken = slot.take().unwrap(); + assert!(taken.cleared); + assert!(matches!(taken.picture, Picture::Rgba { size, .. } if size == moq_video::Size::new(2, 1))); + assert!(slot.take().is_none()); + slot.show(clear()); + assert!(!slot.take().unwrap().cleared); + } + #[tokio::test] async fn a_color_shows_one_pixel() { local(async { @@ -2057,12 +2121,12 @@ mod tests { let red = sources.get("red").unwrap(); assert_eq!(red.status(), Status::Live); assert!(red.strip().is_none(), "a color is silent"); - let Some(Picture::Rgba { size, pixels }) = red.take() else { + let Some(Picture::Rgba { size, pixels }) = red.next_picture() else { panic!("a color shows a picture"); }; assert_eq!(size, moq_video::Size::new(1, 1)); assert_eq!(pixels, [255, 0, 0, 255]); - assert!(red.take().is_none(), "taken once"); + assert!(red.next_picture().is_none(), "taken once"); let guest = sources.get("guest").unwrap(); assert_eq!(guest.status(), Status::Offline, "nothing is announced"); @@ -2212,7 +2276,7 @@ mod tests { let drivers = sources.sync(&collection); assert_eq!(drivers.iter().map(Driver::name).collect::>(), ["red"]); start(&sources, drivers).await; - let Some(Picture::Rgba { pixels, .. }) = sources.get("red").unwrap().take() else { + let Some(Picture::Rgba { pixels, .. }) = sources.get("red").unwrap().next_picture() else { panic!("the edited color shows"); }; assert_eq!(pixels, [0, 0, 255, 255]); @@ -2405,7 +2469,7 @@ mod tests { /// The picture a feed shows next, once it shows one. async fn shown(feed: &Feed) -> Duration { - until(|| match feed.take() { + until(|| match feed.next_picture() { Some(Picture::Frame(frame)) => Some(frame.timestamp.into()), Some(_) => panic!("a broadcast shows only frames"), None => None, @@ -2433,7 +2497,7 @@ mod tests { let stamp = moq_net::Timestamp::from_millis(1_000).unwrap(); let due = until(|| clock.due(stamp)).await; assert_eq!(due, arrived + VIDEO_DELAY); - assert!(feed.take().is_none(), "shown on arrival"); + assert!(feed.next_picture().is_none(), "shown on arrival"); until_time(due).await; assert_eq!(shown(feed).await, Duration::from(stamp)); assert_eq!(feed.delay(), Some(VIDEO_DELAY)); @@ -2478,14 +2542,14 @@ mod tests { until_time(kio::clock::now()).await; let arrived = kio::clock::now(); picture(&mut live.video, &mut encoder, 0); - let moved = until(|| match feed.take() { + let moved = until(|| match feed.next_picture() { Some(Picture::Frame(frame)) => panic!("{:?} shown on arrival", frame.timestamp), _ => clock.due(first).filter(|at| *at != due), }) .await; assert!(moved > due, "the replacement re-anchored behind the first track"); until_time(arrived + VIDEO_DELAY - Duration::from_millis(1)).await; - assert!(feed.take().is_none(), "shown before its delay"); + assert!(feed.next_picture().is_none(), "shown before its delay"); until_time(arrived + VIDEO_DELAY).await; assert!( shown(feed).await < Duration::from(first), @@ -2521,7 +2585,7 @@ mod tests { let due = arrived + target + crate::playout::BUS_LATENCY; until(|| (clock.due(stamp) == Some(due)).then_some(())).await; until_time(due - Duration::from_millis(1)).await; - assert!(feed.take().is_none(), "shown before its sound"); + assert!(feed.next_picture().is_none(), "shown before its sound"); until_time(due).await; assert_eq!(shown(feed).await, Duration::from(stamp)); assert_eq!(feed.delay(), Some(target)); @@ -2590,7 +2654,7 @@ mod tests { let drivers = sources.sync(&collection); start(&sources, drivers).await; - let Some(Picture::Rgba { size, pixels }) = sources.get("logo").unwrap().take() else { + let Some(Picture::Rgba { size, pixels }) = sources.get("logo").unwrap().next_picture() else { panic!("an image shows a picture"); }; assert_eq!(size, moq_video::Size::new(3, 2)); @@ -2663,7 +2727,7 @@ mod tests { let title = sources.get("title").unwrap(); assert_eq!(title.status(), Status::Live); - assert!(matches!(title.take(), Some(Picture::Overlay(_)))); + assert!(matches!(title.next_picture(), Some(Picture::Overlay(_)))); let Status::Failed(reason) = sources.get("shadow").unwrap().status() else { panic!("an overlay outside the subset fails"); }; @@ -2707,7 +2771,7 @@ mod tests { let drivers = sources.sync(&collection); start(&sources, drivers).await; let feed = sources.get("replay").unwrap(); - assert!(matches!(feed.take(), Some(Picture::Rgba { pixels, .. }) if pixels == [0; 4])); + assert!(matches!(feed.next_picture(), Some(Picture::Rgba { pixels, .. }) if pixels == [0; 4])); let (broadcast, video, _live) = video(None).await; let target = Target { @@ -2729,9 +2793,9 @@ mod tests { for index in 45..75u64 { let timestamp = moq_net::Timestamp::from_micros(index * 1_000_000 / 30).unwrap(); let due = begin + (Duration::from(timestamp) - Duration::from(start)).div_f32(speed); - assert!(feed.take().is_none(), "a picture before {timestamp:?} is due"); + assert!(feed.next_picture().is_none(), "a picture before {timestamp:?} is due"); until_time(due).await; - let shown = until(|| match feed.take() { + let shown = until(|| match feed.next_picture() { Some(Picture::Frame(frame)) => Some(frame.timestamp), Some(_) => panic!("cleared before {timestamp:?}"), None => None, @@ -2742,12 +2806,15 @@ mod tests { // Nothing more, and the replay ends once the span's second has // played over two. - assert!(feed.take().is_none(), "the picture at the end shows for no time"); + assert!( + feed.next_picture().is_none(), + "the picture at the end shows for no time" + ); let end = begin + Duration::from_secs(2); until_time(end - Duration::from_millis(1)).await; - assert!(feed.take().is_none(), "cleared before the end"); + assert!(feed.next_picture().is_none(), "cleared before the end"); until_time(end).await; - until(|| matches!(feed.take(), Some(Picture::Rgba { .. })).then_some(())).await; + until(|| matches!(feed.next_picture(), Some(Picture::Rgba { .. })).then_some(())).await; assert_eq!(feed.status(), Status::Live, "ready for the next replay"); }) .await; @@ -2779,10 +2846,10 @@ mod tests { }; // Its first picture, at 1.5 s, is due at once. feed.replay(take(2_500_000), 1.0).unwrap(); - until(|| matches!(feed.take(), Some(Picture::Frame(_))).then_some(())).await; + until(|| matches!(feed.next_picture(), Some(Picture::Frame(_))).then_some(())).await; // This one's first picture, at the same 1.5 s, is due 20 ms in. feed.replay(take(2_480_000), 1.0).unwrap(); - let shown = until(|| feed.take()).await; + let shown = until(|| feed.next_picture()).await; assert!(matches!(shown, Picture::Rgba { pixels, .. } if pixels == [0; 4])); }) .await; @@ -2814,7 +2881,7 @@ mod media_tests { /// The next picture `feed` shows, passing over the clear a restart puts up, /// and when it showed. async fn shown(feed: &Feed) -> (moq_net::Timestamp, Instant) { - let timestamp = until(|| match feed.take() { + let timestamp = until(|| match feed.next_picture() { Some(Picture::Frame(frame)) => Some(frame.timestamp), Some(Picture::Rgba { pixels, .. }) if pixels == [0; 4] => None, Some(_) => panic!("not a decoded picture"), @@ -2849,7 +2916,7 @@ mod media_tests { let into = Duration::from_millis(index * 100); // Within the timer wheel's millisecond of when the test saw the first. until_time(begin + into - Duration::from_millis(2)).await; - assert!(feed.take().is_none(), "picture {index} before its moment"); + assert!(feed.next_picture().is_none(), "picture {index} before its moment"); until_time(begin + into).await; let (timestamp, _) = shown(feed).await; assert_eq!(Duration::from(timestamp) - Duration::from(first), into); @@ -2861,10 +2928,10 @@ mod media_tests { assert_eq!(feed.status(), Status::Live); until_time(begin + Duration::from_millis(1_100)).await; until_status(&sources, "clip.mkv", Status::Ended).await; - assert!(feed.take().is_none(), "the last picture holds"); + assert!(feed.next_picture().is_none(), "the last picture holds"); feed.play().unwrap(); - let cleared = feed.take(); + let cleared = feed.next_picture(); assert!( matches!(cleared, Some(Picture::Rgba { pixels, .. }) if pixels == [0; 4]), "the held picture is cleared at once" @@ -2895,7 +2962,7 @@ mod media_tests { let feed = sources.get("clip.mkv").unwrap(); assert_eq!(feed.status(), Status::Ended); until_time(kio::clock::now() + Duration::from_millis(1_100)).await; - assert!(feed.take().is_none(), "nothing plays until asked"); + assert!(feed.next_picture().is_none(), "nothing plays until asked"); assert_eq!(feed.run(), Run::default()); feed.play().unwrap(); @@ -2982,7 +3049,7 @@ mod media_tests { feed.stop().unwrap(); let stopped = kio::clock::now(); - let blank = feed.take(); + let blank = feed.next_picture(); assert!( matches!(blank, Some(Picture::Rgba { pixels, .. }) if pixels == [0; 4]), "blank at once" @@ -2992,7 +3059,7 @@ mod media_tests { let heard = listen(&mut mixer, &program, &mut at, stopped + Duration::from_millis(500)).await; assert_eq!(heard, 0.0, "nothing it queued plays"); assert!( - !matches!(feed.take(), Some(Picture::Frame(_))), + !matches!(feed.next_picture(), Some(Picture::Frame(_))), "no picture after the stop" ); assert_eq!(feed.run().ended, Some(stopped), "the play stays ended there"); @@ -3024,7 +3091,7 @@ mod media_tests { let (last, _) = shown(feed).await; assert_eq!(Duration::from(last) - Duration::from(first), Duration::from_millis(900)); until_time(begin + Duration::from_millis(998)).await; - assert!(feed.take().is_none(), "the start again before the end"); + assert!(feed.next_picture().is_none(), "the start again before the end"); until_time(begin + Duration::from_millis(1_100)).await; let (again, _) = shown(feed).await; assert_eq!(again, first); diff --git a/rs/moq-scene/src/test_data/person.jpg b/rs/moq-scene/src/test_data/person.jpg new file mode 100644 index 00000000..57e7c744 Binary files /dev/null and b/rs/moq-scene/src/test_data/person.jpg differ diff --git a/rs/moq-studio/src/ui.rs b/rs/moq-studio/src/ui.rs index 188e79c5..d808f9fd 100644 --- a/rs/moq-studio/src/ui.rs +++ b/rs/moq-studio/src/ui.rs @@ -11,8 +11,8 @@ use egui_dock::{DockArea, DockState, NodeIndex, Style, TabViewer}; use moq_scene::browser::Paint; use moq_scene::source::{Feed, Status}; use moq_scene::{ - Blend, Canvas, Clip, Collection, Crop, Direction, Draft, Ease, Filter, Item, Keyframe, Layer, MaskMode, Rect, - Scene, Source, Transition, Video, wgpu, + Background, Blend, Canvas, Clip, Collection, Crop, Direction, Draft, Ease, Filter, Item, Keyframe, Layer, MaskMode, + Rect, Scene, Source, Transition, Video, wgpu, }; use crate::autosave::Conflict; @@ -393,6 +393,8 @@ enum Selection { /// What the Filters panel sets on a selection. #[derive(Clone, Debug, PartialEq)] struct Look { + /// A source's background, where it can be removed; an item has none. + background: Option, crop: Crop, filters: Vec, /// An item's blend mode; a source has none. @@ -404,9 +406,10 @@ impl Selection { fn look(&self, collection: &Collection) -> Option { match self { Self::Source(name) => { - collection.sources.get(name).filter(|source| source.has_video())?; + let source = collection.sources.get(name).filter(|source| source.has_video())?; let video = collection.video.get(name).cloned().unwrap_or_default(); Some(Look { + background: source.removes_background().then_some(video.background), crop: video.crop, filters: video.filters, blend: None, @@ -415,6 +418,7 @@ impl Selection { Self::Item(layer, index) => { let item = collection.items(layer)?.get(*index)?; Some(Look { + background: None, crop: item.crop, filters: item.filters.clone(), blend: Some(item.blend), @@ -429,9 +433,21 @@ impl Selection { } fn set_look(&self, draft: &mut Draft, look: Look) -> Result<(), moq_scene::Error> { - let Look { crop, filters, blend } = look; + let Look { + background, + crop, + filters, + blend, + } = look; match self { - Self::Source(name) => draft.set_video(name, Some(Video { crop, filters })), + Self::Source(name) => draft.set_video( + name, + Some(Video { + background: background.unwrap_or_default(), + crop, + filters, + }), + ), Self::Item(layer, index) => { let item = draft .collection() @@ -1621,13 +1637,8 @@ impl Panels { draft.remove_source(&name) }) } else if let Some(name) = clone { - // A clone of a clone shows the same capture, so it clones that, with - // a copy of the clone's filters. let collection = state.collection(); - let source = Source::Clone { - source: collection.capture(&name).to_string(), - }; - let video = collection.video.get(&name).cloned(); + let (source, video) = clone_of(collection, &name); let clone = collection.unused_name(&format!("{name} clone")); let layer = self.layer(&state).ok(); add(&mut state, layer, clone, source, video).map(drop) @@ -1846,6 +1857,19 @@ impl Panels { let mut next = look.clone(); let mut action: Option<(String, Option)> = None; egui::Grid::new("look").show(ui, |ui| { + if let Some(background) = &mut next.background { + ui.label("Background"); + let mut removed = *background == Background::Remove; + let response = ui + .checkbox(&mut removed, "Remove") + .on_hover_text("Cut the person out of the picture, leaving the rest transparent, before the crop and every filter. Each picture shows a frame or more late, and the source's own track keeps the background."); + if response.changed() { + *background = if removed { Background::Remove } else { Background::Keep }; + let verb = if removed { "Remove" } else { "Keep" }; + action = Some((format!("{verb} the background of {what}"), None)); + } + ui.end_row(); + } ui.label("Crop"); ui.horizontal(|ui| { let crop = &mut next.crop; @@ -3253,6 +3277,20 @@ fn color_ui(ui: &mut Ui, color: &mut moq_scene::Rgba) -> egui::Response { response } +/// The source and video settings of a clone of `name`. A clone of a clone +/// shows the same capture, so it clones that. It copies the crop and filters, +/// but not a removed background: the clone already shows the cut-out. +fn clone_of(collection: &Collection, name: &str) -> (Source, Option