diff --git a/justfile b/justfile index 2faec15..0a8f431 100644 --- a/justfile +++ b/justfile @@ -55,7 +55,7 @@ render-sw: icd=("$mesa"/share/vulkan/icd.d/lvp_icd.*.json) export VK_DRIVER_FILES="${icd[0]}" cargo nextest run --locked -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/) | test(/^browser::layers::tests::/)' - cargo nextest run --locked -p moq-studio --features openh264 --run-ignored only -E 'test(/^engine::tests::the_program_/) | test(=setup::tests::times_every_canvas)' + cargo nextest run --locked -p moq-studio --features openh264 --run-ignored only -E '(test(/^engine::tests::/) - test(=engine::tests::the_virtual_camera_reaches_a_pipewire_app)) | test(=setup::tests::times_every_canvas)' cargo nextest run --locked -p moq-studio --features cef --run-ignored only -E 'binary(browser)' sh/pipewire-session.sh cargo nextest run --locked -p moq-studio --features pipewire --run-ignored only -E 'test(=engine::tests::the_virtual_camera_reaches_a_pipewire_app)' diff --git a/quest/m0/README.md b/quest/m0/README.md index 224f775..fd6cd60 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -93,7 +93,6 @@ repository. - [Studio on Windows](/quest/m0/app-windows.md) - the studio runs on Windows, not only Linux - [Wayland display and window sources](/quest/m0/wayland-sources.md) - portal-picked sources that reopen without the picker - [Destinations](/quest/m0/destinations.md) - RTMP, RTMPS, and WHIP push with per-destination codecs and shared encoders, on a UI-free egress library a hosted service can reuse -- [Composable scenes](/quest/m0/scenes/README.md) - scenes nest scenes and the program, any scene publishes at its own size, and mixes become audio scenes - [Pan each sound](/quest/m0/audio-pan.md) - the Audio panel and `[audio.]` pan each sound wherever it plays - [Clap-to-measure sync delay](/quest/m0/sync-clap.md) - a clap sets a microphone's delay against its camera - [Audio filters](/quest/m0/audio-filters.md) - per-source noise suppression, gate, compressor, limiter, and plugins, configured from the studio diff --git a/quest/m0/scenes/README.md b/quest/m0/scenes/README.md deleted file mode 100644 index 3fba322..0000000 --- a/quest/m0/scenes/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# [M] Composable scenes - -## Goal - -Scenes form a graph. A scene can draw another scene, or the live program, as -a source. Any scene can be published as its own broadcast at its own size, and -the old audio mixes are audio-only scenes that nest the same way. This -replaces separate canvases: a portrait broadcast is a published 1080x1920 -scene that draws the program, cropped, with its own overlays. - -This epic's own work is an end-to-end test: one studio publishes the program -at `<--broadcast>` and a portrait scene drawing it at -`<--broadcast>/`, each carrying its picked audio scene, and a viewer of -each sees and hears the same cut at the same moment. - -## Plan - -Decided 2026-11-05 with the maintainer (/quest-plan), replacing the canvases -of [moq-dev/studio#42](https://github.com/moq-dev/studio/pull/42) and closing -[moq-dev/studio#54](https://github.com/moq-dev/studio/pull/54): - -- ✅ **Published scenes + scene source** rather than canvases with a layout - per scene and linked or unlinked switching. One concept (a scene) instead of - two, and no unlinking. Rejected: published scenes that never follow cuts; - keeping canvases. -- ✅ **A scene source draws any scene, or `program`** (whatever is on air, - transitions included), scaled and cropped like any picture. Rejected: a - program-only source. -- ✅ **The program stays one switcher at `<--broadcast>`**: Preview, Program - and Cut work as today. Other published scenes are always live at - `<--broadcast>/` and follow cuts only through a scene source. - Rejected: every published scene switchable on its own. -- ✅ **Every scene has a size**, defaulting to the program's. The switcher - offers only scenes of the program's size. -- ✅ **Mixes become audio scenes** that hold only sounds and nest, so a - "Mics" scene can feed "Stream" and "Headphones". Audio stays independent of - cuts, as today. Each published output pairs a picture scene with an audio - scene, defaulting to the one on program. Rejected: scenes carrying their own - audio, so a cut switches audio; keeping `[mixes]` separate. -- ✅ **UI:** the Scenes panel lists picture and audio scenes, tagged by kind. - The Audio panel stays a mixer (fader, pan, meter, and sends to audio scenes) - for live work, as pro switchers do. A graph view of the same data is - [its own quest](/quest/m1/scene-graph.md). -- Nothing is released, so the scene file breaks freely: `[canvases]` and - `[mixes]` go away, with no migration. - -Carried over from #54: Health grades each published broadcast; a recording -picks a picture scene and an audio scene; a scene no panel shows and no output -reads skips drawing. Per-scene cost grows only with what that scene draws; -`compositor::bench` measures it. - -Changed 2026-11-07 with the maintainer, on -[moq-dev/studio#77](https://github.com/moq-dev/studio/pull/77): pan left -this epic for [its own quest](/quest/m0/audio-pan.md), since it waits on -moq-audio, so it no longer holds up this end-to-end test. diff --git a/quest/m1/compose/layout.md b/quest/m1/compose/layout.md index f293428..e427624 100644 --- a/quest/m1/compose/layout.md +++ b/quest/m1/compose/layout.md @@ -52,11 +52,6 @@ Only sources the creator publishes independently ([Published independently](/quest/m2/publish-independently.md)) have tracks a viewer can fetch; the rest reach viewers only in the program. -Decided 2026-11-05 (maintainer, /quest-plan): the layout waits for -[Composable scenes](/quest/m0/scenes/README.md), which removes canvases and -mixes and adds scene sources, published scenes, and audio scenes, so it -serializes that model once instead of chasing it. - -## Required - -- [Composable scenes](/quest/m0/scenes/README.md) - the scene model the layout serializes +Decided 2026-11-05 (maintainer, /quest-plan): the layout serializes the +scene model with scene sources, published scenes, and audio scenes, not the +canvases and mixes they replaced. diff --git a/quest/m1/scene-graph.md b/quest/m1/scene-graph.md index a6c3735..a2244f9 100644 --- a/quest/m1/scene-graph.md +++ b/quest/m1/scene-graph.md @@ -13,7 +13,3 @@ Decided 2026-11-05 with the maintainer (/quest-plan): the live controls stay a mixer, as pro switchers keep them, and this view is for building and inspecting routing, as node tools (TouchDesigner, Fusion) do. It reads and edits the same data, so it adds no second model. - -## Required - -- [Composable scenes](/quest/m0/scenes/README.md) - the graph this view shows diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs index 039dde6..f69dbac 100644 --- a/rs/moq-studio/src/engine.rs +++ b/rs/moq-studio/src/engine.rs @@ -3386,13 +3386,7 @@ mod tests { None => panic!("the video ended"), }; assert_eq!(frame.size(), size); - let rgba = frame.surface.to_rgba(&Default::default()).expect("rgba"); - let center = ((size.height / 2 * size.width + size.width / 2) * 4) as usize; - let [r, g, b, _] = rgba.data()[center..center + 4] else { - unreachable!() - }; - // H.264 and its 4:2:0 round trip shift a primary a little. - if r > 200 && g < 60 && b < 60 { + if shows(center(&frame), RED) { return frame.timestamp; } } @@ -3400,18 +3394,43 @@ mod tests { .await } - /// The broadcast of the published scene `tall` as `relay` routes it, its - /// catalogs, and the first that lists its video. - async fn routed_tall( + const RED: [u8; 3] = [255, 0, 0]; + const GREEN: [u8; 3] = [0, 255, 0]; + + /// The color at the center of `frame`. + fn center(frame: &moq_video::Frame) -> [u8; 3] { + let size = frame.size(); + let rgba = frame.surface.to_rgba(&Default::default()).expect("rgba"); + let center = ((size.height / 2 * size.width + size.width / 2) * 4) as usize; + let [r, g, b, _] = rgba.data()[center..center + 4] else { + unreachable!() + }; + [r, g, b] + } + + /// Whether `color` is `primary`, whose channels are each full or empty, + /// as H.264 and its 4:2:0 round trip leave it: a little shifted. + fn shows(color: [u8; 3], primary: [u8; 3]) -> bool { + color + .iter() + .zip(primary) + .all(|(&channel, full)| channel.abs_diff(full) < 60) + } + + /// The broadcast at `path` as `relay` routes it, its catalogs, and the + /// first that lists its video. + async fn routed( relay: &moq_net::origin::Consumer, + path: &str, ) -> ( moq_net::broadcast::Consumer, moq_mux::catalog::Consumer<()>, hang::Catalog, ) { - // Until the scene's broadcast is announced, the program's route covers - // its path and cannot serve it; the follow moves on once it is. - let mut following = relay.follow("studio/tall", None).unwrap(); + // Until a published scene's broadcast is announced, the program's + // route covers its path and cannot serve it; the follow moves on once + // it is. + let mut following = relay.follow(path, None).unwrap(); let (broadcast, mut catalogs) = kio::clock::timeout(TIMEOUT, async { loop { let broadcast = following.available().await.expect("broadcast").expect("an instance"); @@ -3422,7 +3441,7 @@ mod tests { } }) .await - .expect("the relay routes the canvas"); + .expect("the relay routes the broadcast"); // Audio is listed from the start, and the video once its encoder opens. let catalog = kio::clock::timeout(TIMEOUT, async { loop { @@ -3433,7 +3452,7 @@ mod tests { } }) .await - .expect("the canvas lists its video"); + .expect("the broadcast lists its video"); (broadcast, catalogs, catalog) } @@ -3510,7 +3529,7 @@ mod tests { .await .expect("the studio starts"); - let (broadcast, catalogs, catalog) = routed_tall(&relay).await; + let (broadcast, catalogs, catalog) = routed(&relay, "studio/tall").await; let (audio, config) = catalog.audio.enabled().next().expect("its audio"); assert!(config.broadcast.is_none(), "its own, not the program's"); let mut options = moq_audio::decode::Options::new(); @@ -3603,7 +3622,7 @@ mod tests { // Back again, quitting ends it as cleanly. engine.state.lock().unwrap().undo().unwrap(); - let (broadcast, catalogs, catalog) = routed_tall(&relay).await; + let (broadcast, catalogs, catalog) = routed(&relay, "studio/tall").await; let (video, _) = canvas_size(&catalog); let track = broadcast.track(&video).unwrap().subscribe(None).await.unwrap(); let stopping = engine.clone(); @@ -3618,6 +3637,340 @@ mod tests { assert_eq!(engine.stats.sessions.open(), 0); } + /// What a viewer of one broadcast takes in, frame by frame as it arrives, + /// each stamped with its timestamp: the color at the center of each + /// picture, and the loudest channel of each sample. + struct Viewer { + pictures: tokio::sync::mpsc::UnboundedReceiver<(Duration, [u8; 3])>, + sound: tokio::sync::mpsc::UnboundedReceiver<(Duration, Vec)>, + /// Samples per second of the sound. + rate: u32, + /// When the last picture read was shown, and the color last looked + /// for, which it showed. + seen: Option<(Duration, [u8; 3])>, + } + + impl Viewer { + /// Play `broadcast`'s video at `size` and its audio, as `catalog` lists + /// them, from the newest group on. Each decodes on a task of its own, + /// and neither skips a group it falls behind on, as a player would, so + /// a loaded host slows the test rather than losing what it measures. + async fn watch( + broadcast: &moq_net::broadcast::Consumer, + catalog: &hang::Catalog, + size: moq_video::Size, + ) -> Self { + let (video, config) = catalog + .video + .renditions + .iter() + .find(|(_, config)| config.coded_width == Some(size.width) && config.coded_height == Some(size.height)) + .expect("the video at its size"); + let mut options = moq_video::decode::Options::new(); + options.start = moq_video::decode::Start::Latest; + options.delay.max = None; + let mut pictures = moq_video::decode::Consumer::new(broadcast, config, video.clone(), options) + .await + .expect("video decoder"); + let (seen, seeing) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + while let Some(read) = pictures.read().await.expect("decode") { + if let moq_video::decode::Read::Frame(frame) = read + && seen.send((Duration::from(frame.timestamp), center(&frame))).is_err() + { + return; + } + } + }); + + let (audio, config) = catalog.audio.enabled().next().expect("its audio"); + let rate = config.sample_rate; + let channels = config.channel_count as usize; + let mut options = moq_audio::decode::Options::new(); + options.start = moq_audio::decode::Start::Latest; + options.delay.max = None; + let mut sound = moq_audio::decode::Consumer::new(broadcast, config, audio.clone(), options) + .await + .expect("audio decoder"); + let (heard, hearing) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + while let Some(read) = sound.read().await.expect("decode") { + let moq_audio::decode::Read::Frame(frame) = read else { + continue; + }; + let (samples, _) = frame.data.as_chunks::<4>(); + let samples: Vec = samples.iter().map(|bytes| f32::from_le_bytes(*bytes).abs()).collect(); + let loudest = samples + .chunks_exact(channels) + .map(|sample| sample.iter().copied().fold(0.0, f32::max)); + if heard + .send((Duration::from(frame.timestamp), loudest.collect())) + .is_err() + { + return; + } + } + }); + + Self { + pictures: seeing, + sound: hearing, + rate, + seen: None, + } + } + + /// When the viewer next sees `color` at the center, and the picture it + /// saw before that, if any. Every picture between shows the color it + /// saw last. + async fn sees(&mut self, color: [u8; 3]) -> (Option, Duration) { + kio::clock::timeout(TIMEOUT, async { + loop { + let (at, center) = self.pictures.recv().await.expect("the video ended"); + if shows(center, color) { + return (self.seen.replace((at, color)).map(|(before, _)| before), at); + } + if let Some((_, held)) = self.seen { + assert!(shows(center, held), "{center:?} at {at:?} before {color:?}"); + self.seen = Some((at, held)); + } + } + }) + .await + .expect("the viewer sees the color") + } + + /// The first moment, from `from` on, the viewer hears a sample louder + /// than `level`, and the loudest it heard from `from` until `until`. + async fn hears(&mut self, from: Duration, until: Duration, level: f32) -> (Duration, f32) { + let rate = self.rate; + let mut before = 0.0f32; + kio::clock::timeout(TIMEOUT, async { + loop { + let (start, samples) = self.sound.recv().await.expect("the audio ended"); + for (index, sample) in samples.into_iter().enumerate() { + let at = start + Duration::from_secs(index as u64) / rate; + if at < from { + continue; + } + if sample > level { + return (at, before); + } + if at < until { + before = before.max(sample); + } + } + } + }) + .await + .expect("the viewer hears it") + } + } + + /// Where `at` falls on `engine`'s program clock, which stamps every + /// picture and sample it publishes. + fn stamp(engine: &Engine, at: Instant) -> Duration { + let now = kio::clock::now(); + let reading = Duration::from(engine.program.clock.capture(now).unwrap()); + reading + at.saturating_duration_since(now) - now.saturating_duration_since(at) + } + + /// One studio goes live with the program at its broadcast and, beneath + /// it, a portrait scene drawing the program, each carrying the audio + /// scene it picks. A cut reaches a viewer of each on the same frame, and + /// the tone that restarts with it at the same moment. The portrait's + /// audio scene nests the program's and adds a quiet bed, so only its + /// viewer hears anything before the cut. + #[tokio::test(flavor = "multi_thread")] + #[ignore = "needs a Vulkan device and an H.264 encoder; run by just render-sw"] + async fn the_program_and_a_portrait_cut_together() { + // The tone peaks near 0.1 (-20 dBFS): it is heard above half that, + // and the bed, at a fifth of it, above a tenth. + const LOUD: f32 = 0.05; + const QUIET: f32 = 0.01; + const BED: f32 = 0.2; + + let _ = moq_tokio::crypto::install_default(); + let fixture = moq_relay::test_relay().await.expect("test relay"); + let ready = fixture.relay.ready(); + let relay = fixture.relay.cluster().origin.consume(); + tokio::spawn(fixture.relay.run()); + ready.wait().await.expect("relay ready"); + + let mut client = moq_tokio::connect::Config::default(); + client.tls.fingerprint = vec![fixture.fingerprint.clone()]; + let client = client.init(Default::default()).expect("client"); + + let dir = tempfile::tempdir().unwrap(); + let tone = include_bytes!("../../moq-scene/src/test_data/tone.webm"); + std::fs::write(dir.path().join("tone.webm"), tone).unwrap(); + let gpu = Gpu::new().await.expect("a GPU"); + let (autosave, _) = Autosave::open(dir.path().join("show.toml")).unwrap(); + let relays = vec![Relay { + client, + url: fixture.url.clone(), + }]; + + // Program starts on the red scene; the cut takes it to green, whose + // tone plays from its start as it goes on program. + let mut loaded = red_canvas(); + let collection = &mut loaded.collection; + let tone = |looping, restart| moq_scene::Source::Media { + path: "tone.webm".into(), + looping, + restart, + }; + let green = moq_scene::Source::Color { + color: moq_scene::Rgba([0, 255, 0, 255]), + }; + collection.sources.insert("green".into(), green); + collection.sources.insert("tone".into(), tone(false, true)); + collection.sources.insert("bed".into(), tone(true, false)); + collection.sources.insert("program".into(), moq_scene::Source::Program); + let bed = moq_scene::Audio { + gain: BED, + ..Default::default() + }; + collection.audio.insert("bed".into(), bed); + let fill = moq_scene::Rect::fill(&collection.canvas); + collection.scenes.push(moq_scene::Scene { + items: vec![moq_scene::Item::new("green", fill), moq_scene::Item::new("tone", fill)], + ..moq_scene::Scene::new("Cut") + }); + let portrait = moq_scene::Size { + width: 360, + height: 640, + }; + let tall = moq_scene::Rect { + x: 0.0, + y: 0.0, + width: 360.0, + height: 640.0, + }; + collection.scenes.push(moq_scene::Scene { + size: Some(portrait), + published: true, + audio: Some("Portrait".into()), + items: vec![moq_scene::Item::new("program", tall)], + ..moq_scene::Scene::new("tall") + }); + collection.audio_scenes[0].sounds.insert("tone".into()); + collection.audio_scenes.push(moq_scene::AudioScene { + scenes: [moq_scene::AudioScene::DEFAULT.to_string()].into(), + sounds: ["bed".to_string()].into(), + ..moq_scene::AudioScene::new("Portrait") + }); + let engine = Engine::start(launch(gpu, relays, dir.path()), autosave, loaded) + .await + .expect("the studio starts"); + + let (broadcast, _, catalog) = routed(&relay, "studio").await; + let mut program = Viewer::watch(&broadcast, &catalog, moq_video::Size::new(640, 360)).await; + let (broadcast, _, catalog) = routed(&relay, "studio/tall").await; + let mut tall = Viewer::watch(&broadcast, &catalog, moq_video::Size::new(360, 640)).await; + program.sees(RED).await; + tall.sees(RED).await; + + // The tone plays once as it opens. From the first silence the + // program's viewer hears after that, what either viewer hears of it is + // the cut's. + let ended = kio::clock::timeout(TIMEOUT, async { + loop { + let run = engine.state.lock().unwrap().sources.get("tone").unwrap().run(); + if let Some(ended) = run.ended { + return stamp(&engine, ended); + } + program.sound.recv().await.expect("the audio ended"); + } + }) + .await + .expect("the tone's first play ends"); + let quiet = kio::clock::timeout(TIMEOUT, async { + loop { + let (start, samples) = program.sound.recv().await.expect("the audio ended"); + if start >= ended && samples.iter().all(|sample| *sample < QUIET) { + return start; + } + } + }) + .await + .expect("the program falls quiet"); + // The portrait's viewer hears its bed meanwhile, once it plays. + let (bed, _) = tall.hears(quiet, quiet, QUIET).await; + + { + let mut state = engine.state.lock().unwrap(); + state.compositor.set_preview(Some("Cut".into())); + state.cut().unwrap(); + } + // Both show the cut from the same frame: neither shows red where the + // other shows green. An encoder still busy with the frame before + // drops the cut's own, so the first green each shows may differ. + let (Some(program_red), program_green) = program.sees(GREEN).await else { + unreachable!("the program showed red first") + }; + let (Some(tall_red), tall_green) = tall.sees(GREEN).await else { + unreachable!("the portrait showed red first") + }; + assert!( + program_red < tall_green && tall_red < program_green, + "the portrait cuts on the program's frame: program red to {program_red:?}, green from \ + {program_green:?}; portrait red to {tall_red:?}, green from {tall_green:?}" + ); + + // The cut played the tone again before drawing its first green + // frame, which begins once its file decodes, after the last red + // frame. Decoding takes as long as it takes, so the tone's run is read + // again at each picture the program's viewer sees until then. + let run = kio::clock::timeout(TIMEOUT, async { + loop { + let run = engine.state.lock().unwrap().sources.get("tone").unwrap().run(); + if run.begun.is_some() { + return run; + } + program.pictures.recv().await.expect("the video ended"); + } + }) + .await + .expect("the tone's second play begins"); + assert_eq!(run.play, 1, "the cut played the tone again, once"); + let played = stamp(&engine, run.begun.unwrap()); + assert!( + played > program_red.max(tall_red), + "the tone plays with the cut, not before: {played:?}" + ); + // Until then, the program's viewer heard nothing while the portrait's + // heard the bed. + assert!(bed < played, "the bed is heard before the cut"); + let (heard, before) = program.hears(quiet, played, LOUD).await; + assert!( + before < QUIET, + "the program's audio scene holds the tone alone: {before}" + ); + // The viewer hears the tone once the bus mixes it: a media source + // writes each sound when it is due, and the bus holds every source + // 50 ms (moq-audio's `bus::LATENCY`) before mixing it, give or take a + // block. + let mixed = Duration::from_millis(50) + AUDIO_PERIOD; + assert!( + heard >= played && heard - played < mixed, + "the tone is heard as the bus mixes it: played {played:?}, heard {heard:?}" + ); + let (also, _) = tall.hears(quiet, played, LOUD).await; + assert!( + heard.abs_diff(also) < AUDIO_PERIOD, + "both hear the tone within a block of the bus: {heard:?} and {also:?}" + ); + + let stopping = engine.clone(); + tokio::task::spawn_blocking(move || stopping.stop()).await.unwrap(); + kio::clock::timeout(TIMEOUT, engine.close()) + .await + .expect("the studio closes"); + let failed = engine.stats.error(); + assert!(failed.is_none(), "{failed:?}"); + } + /// A guest's return feed lists every size of `engine`'s program that /// `published` lists, read from the program as served at `.program`, which /// holds the program's tracks and none of the sources'.