diff --git a/README.md b/README.md index e05ffbd..d5cb3f4 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Every panel is a tab you can drag, split, or stack. | Scenes | Pick the scene in preview, and add, rename (double-click), reorder, or remove one. A scene on program, or still fading out, cannot be removed, even by an undo, until program cuts away from it. | | History | See every edit, and undo or redo back to any of them. | | Items | Place, show, hide, and order what the preview scene or the top layer draws, back to front. Select an item to filter or copy it. | -| Sources | Add a source by hand, see whether each and its own track are live and where it is shown, clone one, and remove one. Select a source to filter it. A browser source shows whether its page is copied on the GPU or, as a fallback, uploaded from memory (CPU), in place of a track. | +| Sources | Add a source by hand, see whether each and its own track are live and where it is shown, clone one, and remove one. Select a source to filter it. A camera or broadcast without a new picture, or a microphone or system audio without new sound, for a second reads as stalled. A browser source shows whether its page is copied on the GPU or, as a fallback, uploaded from memory (CPU), in place of a track. | | Devices | List this machine's cameras, displays, windows, and audio inputs, and add one as a source. | | Guests | Invite a guest, show their join link as a QR code, see who joined, and take a guest out of every scene. | | Filters | Crop the selected source or item, add, order, and set its filters, set an item's blend mode, and copy or paste filters. | @@ -518,8 +518,9 @@ The Health panel grades the stream every second on the thresholds moq's `hang::health` shares with the rest of the stack, and lists each problem with its cause in plain words: a relay is unreachable, the uplink is congested or losing packets, the relay is far away, the encoder is behind or -stuck, a camera or broadcast source on program stalled, or viewers are late, -stalling, or cannot decode a track. A relay lost while the other serves is +stuck, a source stalled (a camera or broadcast the program draws, or an +unmuted microphone or system audio), or viewers are late, stalling, or cannot +decode a track. A relay lost while the other serves is degraded, since viewers have nothing left to fall back to; with none serving it is unhealthy. Below that it shows: diff --git a/quest/m0/README.md b/quest/m0/README.md index 4c0c3a9..8f64e32 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -94,7 +94,6 @@ repository. - [Broadcast sources play in sync](/quest/m0/broadcast-sync.md) - subscribed broadcasts and guests play audio and video on the publisher's timestamps, with `moq play`'s delay - [Wayland display and window sources](/quest/m0/wayland-sources.md) - portal-picked sources that reopen without the picker - [Live encode control](/quest/m0/encode-control.md) - simulcast rungs that follow the bandwidth grant, live bitrate and resolution, keyframes on request -- [Audio sources report stalls](/quest/m0/audio-stall.md) - a dead microphone or system audio reads stalled like a camera - [HTML/CSS overlays](/quest/m0/overlays.md) - script-free HTML and CSS graphics through Blitz natively and the DOM in the browser - [Animation and transitions](/quest/m0/animation.md) - move, slide, wipe, and stinger transitions, and keyframed item properties - [ML video effects](/quest/m0/effects.md) - background removal and other ML effects, baked into their source's track diff --git a/quest/m0/audio-stall.md b/quest/m0/audio-stall.md deleted file mode 100644 index 6dfff22..0000000 --- a/quest/m0/audio-stall.md +++ /dev/null @@ -1,15 +0,0 @@ -# [S] Audio sources report stalls - -## Goal - -A live audio source (a microphone or system audio) that stops delivering -samples reads "stalled" in the Sources panel and Health, as a live camera does -after a second without a picture. - -## Plan - -The audio bus reports no last-delivery time per source, so the stall check -video sources have cannot see it. The studio opens each capture as a -`capture::Producer`, so read delivery from a consumer of its own rather than -adding a side channel in the bus. Raised on -[#5](https://github.com/moq-dev/studio/pull/5). diff --git a/rs/moq-scene/src/source.rs b/rs/moq-scene/src/source.rs index 787bb03..f5ce216 100644 --- a/rs/moq-scene/src/source.rs +++ b/rs/moq-scene/src/source.rs @@ -49,8 +49,9 @@ pub(crate) enum Picture { /// an undrawn one, so a slow canvas skips pictures instead of queuing them. pub(crate) type Mailbox = Arc>>; -/// When a source last delivered a picture, `None` before its first. -type Shown = Arc>>; +/// When a source last delivered a picture or, for a microphone or system +/// audio, a buffer of sound. `None` before its first. +type Delivered = Arc>>; /// What a broadcast source plays, for a replay of it to copy. #[derive(Default)] @@ -83,11 +84,30 @@ fn clear() -> Picture { } } -/// How long a camera or broadcast may go without a new picture before it reads -/// as stalled. Either delivers several pictures a second while it works, so a -/// second without one is a freeze a viewer sees. +/// How long a camera, broadcast, microphone, or system audio may go without +/// delivering before it reads as stalled. Each delivers several pictures or +/// buffers a second while it works, so a second without one is a freeze a +/// viewer sees or a silence they hear. pub const STALL: Duration = Duration::from_secs(1); +/// What a live source stopped delivering, and for how long. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Stall { + /// A camera or broadcast without a new picture: the program repeats its last one. + Picture(Duration), + /// A microphone or system audio without new samples: the program plays silence for it. + Sound(Duration), +} + +impl Stall { + /// How long the source has gone without delivering. + pub fn since(&self) -> Duration { + match self { + Self::Picture(since) | Self::Sound(since) => *since, + } + } +} + /// One instance of a broadcast being decoded. type Playing<'a> = Pin> + 'a>>; @@ -194,7 +214,7 @@ impl Sources { pub struct Feed { source: Source, picture: Mailbox, - shown: Shown, + delivered: Delivered, strip: Option, status: kio::Consumer, page: Option, @@ -263,19 +283,25 @@ impl Feed { self.track.as_ref().map(|track| track.read().clone()) } - /// How long a live camera or broadcast has gone without a new picture, once - /// that is at least [`STALL`] at `now`. + /// What a live camera, broadcast, microphone, or system audio has gone + /// without delivering, once that is at least [`STALL`] at `now`. /// /// Displays and windows are never stalled: a capture of a still screen /// delivers nothing until it changes. Neither are sources that have not - /// delivered their first picture, such as an audio-only broadcast. - pub fn stalled(&self, now: Instant) -> Option { - if !matches!(self.source, Source::Camera { .. } | Source::Broadcast { .. }) || self.status() != Status::Live { + /// delivered yet, such as an audio-only broadcast: a broadcast stalls by its + /// picture alone. + pub fn stalled(&self, now: Instant) -> Option { + let stall = match self.source { + Source::Camera { .. } | Source::Broadcast { .. } => Stall::Picture, + Source::Microphone { .. } | Source::SystemAudio => Stall::Sound, + _ => return None, + }; + if self.status() != Status::Live { return None; } - let shown = (*self.shown.lock().unwrap())?; - let since = now.saturating_duration_since(shown); - (since >= STALL).then_some(since) + let delivered = (*self.delivered.lock().unwrap())?; + let since = now.saturating_duration_since(delivered); + (since >= STALL).then(|| stall(since)) } /// The source's gain, mute, and meter in the program audio, if it has sound. @@ -301,7 +327,7 @@ pub struct Driver { source: Source, context: Context, picture: Mailbox, - shown: Shown, + delivered: Delivered, strip: Option, status: kio::Producer, #[cfg_attr( @@ -458,7 +484,7 @@ impl Driver { fn show(&self, picture: Picture) { *self.picture.lock().unwrap() = Some(picture); - *self.shown.lock().unwrap() = Some(kio::clock::now()); + *self.delivered.lock().unwrap() = Some(kio::clock::now()); set(&self.status, Status::Live); } @@ -524,11 +550,22 @@ impl Driver { let strip = self.strip.clone().expect("audio sources have a strip"); let capture = capture(source).await?; let channel = self.context.bus.capture(capture.consume(), strip)?; + let heard = self.hear(capture.consume()); set(&self.status, Status::Live); - tokio::try_join!(capture.run(), channel.run())?; + tokio::try_join!(capture.run(), channel.run(), heard)?; Ok(()) } + /// Note each buffer `capture` delivers, so one that goes quiet reads as + /// stalled. Ends with the capture's error. + #[cfg(feature = "capture")] + async fn hear(&self, mut capture: moq_audio::capture::Consumer) -> Result<(), moq_audio::Error> { + loop { + capture.read().await?; + *self.delivered.lock().unwrap() = Some(kio::clock::now()); + } + } + #[cfg(not(feature = "capture"))] async fn device(&self, _: &Source, _: Option<&Listing<'_>>) -> Result<(), Error> { Err(Error::Unsupported( @@ -912,7 +949,7 @@ fn pick<'a, C: Reference + Clone + 'a>( /// Start the feed for `source`, and the driver that fills it. fn open(name: &str, source: &Source, context: &Context) -> (Feed, Driver) { let picture = Mailbox::default(); - let shown = Shown::default(); + let delivered = Delivered::default(); let strip = source.has_audio().then(Strip::default); if let Some(strip) = &strip { context.program.route(strip); @@ -943,7 +980,7 @@ fn open(name: &str, source: &Source, context: &Context) -> (Feed, Driver) { let feed = Feed { source: source.clone(), picture: picture.clone(), - shown: shown.clone(), + delivered: delivered.clone(), strip: strip.clone(), status: status.consume(), page: page.clone(), @@ -956,7 +993,7 @@ fn open(name: &str, source: &Source, context: &Context) -> (Feed, Driver) { source: source.clone(), context: context.clone(), picture, - shown, + delivered, strip, status, page, @@ -1196,11 +1233,11 @@ mod tests { .await; } - /// A camera or broadcast that stops delivering pictures reads as stalled once - /// [`STALL`] passes; a display, a still, and a source that never delivered - /// one do not. + /// A camera or broadcast that stops delivering pictures, or a microphone or + /// system audio that stops delivering sound, reads as stalled once [`STALL`] + /// passes; a display, a still, and a source that never delivered do not. #[tokio::test] - async fn a_live_source_without_new_pictures_stalls() { + async fn a_live_source_without_new_deliveries_stalls() { local(async { let dir = tempfile::tempdir().unwrap(); let (context, _origin) = context(dir.path()); @@ -1213,14 +1250,22 @@ mod tests { let (camera, driver) = opened(Source::Camera { id: None }); assert_eq!(camera.stalled(start + STALL), None, "no picture yet"); - *driver.shown.lock().unwrap() = Some(start); + *driver.delivered.lock().unwrap() = Some(start); assert_eq!(camera.stalled(start + STALL / 2), None); - assert_eq!(camera.stalled(start + STALL * 3), Some(STALL * 3)); + assert_eq!(camera.stalled(start + STALL * 3), Some(Stall::Picture(STALL * 3))); set(&driver.status, Status::Failed("unplugged".into())); assert_eq!(camera.stalled(start + STALL * 3), None, "a failure says why on its own"); + for source in [Source::Microphone { id: None }, Source::SystemAudio] { + let (audio, driver) = opened(source); + assert_eq!(audio.stalled(start + STALL), None, "no sound yet"); + *driver.delivered.lock().unwrap() = Some(start); + assert_eq!(audio.stalled(start + STALL / 2), None); + assert_eq!(audio.stalled(start + STALL * 3), Some(Stall::Sound(STALL * 3))); + } + let (display, driver) = opened(Source::Display { id: None }); - *driver.shown.lock().unwrap() = Some(start); + *driver.delivered.lock().unwrap() = Some(start); assert_eq!(display.stalled(start + STALL * 3), None, "a still screen sends nothing"); }) .await; diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs index 786a400..6f1a0db 100644 --- a/rs/moq-studio/src/engine.rs +++ b/rs/moq-studio/src/engine.rs @@ -649,25 +649,30 @@ impl Engine { /// What only the studio knows about its stream at `now`. fn local(&self, now: Instant) -> health::Local { let stats = &self.stats; + // A stalled picture matters where the program draws it: a scene on air, + // including one fading out, or the top layer over them. Stalled sound + // matters wherever the program plays it. let stalled = { let state = self.state.lock().unwrap(); - let program = state + let shown: Vec<&str> = state .compositor - .program_scene() - .and_then(|name| state.collection.scene(name)); - let mut shown: Vec<&str> = program - .into_iter() + .on_air() + .filter_map(|name| state.collection.scene(name)) .flat_map(|scene| &scene.items) + .chain(&state.collection.top) .filter(|item| item.visible) - .map(|item| item.source.as_str()) + .map(|item| state.collection.capture(&item.source)) .collect(); - shown.sort_unstable(); - shown.dedup(); - shown - .into_iter() - .filter_map(|name| { - let since = state.sources.get(name)?.stalled(now)?; - Some((name.to_string(), since)) + state + .sources + .iter() + .filter_map(|(name, feed)| { + let stall = feed.stalled(now)?; + let matters = match stall { + source::Stall::Picture(_) => shown.contains(&name), + source::Stall::Sound(_) => feed.strip().is_some_and(|strip| !strip.muted()), + }; + matters.then(|| (name.to_string(), stall)) }) .collect() }; diff --git a/rs/moq-studio/src/health.rs b/rs/moq-studio/src/health.rs index 92e4d5b..0dfd477 100644 --- a/rs/moq-studio/src/health.rs +++ b/rs/moq-studio/src/health.rs @@ -21,10 +21,20 @@ pub use hang::health::Verdict; use hang::health::{self, Health, Input, Observer, Thresholds}; use kio::clock::Instant; use moq_net::bandwidth::Rate; +use moq_scene::source::Stall; /// How often the health is graded: as often as reporters write. pub const INTERVAL: Duration = Duration::from_secs(1); +/// What a stalled source has gone without, such as "no new picture for 3 s". +pub fn without(stall: Stall) -> String { + let what = match stall { + Stall::Picture(_) => "picture", + Stall::Sound(_) => "sound", + }; + format!("no new {what} for {:.0} s", stall.since().as_secs_f64()) +} + /// The prefix, under the broadcast's own path, where viewers publish feedback. const ECHO: &str = "viewers"; @@ -195,8 +205,8 @@ pub struct Local { pub encoder: Option, /// Why the output stopped, if it did. pub failed: Option, - /// Sources the program shows that stopped delivering pictures, and for how long. - pub stalled: Vec<(String, Duration)>, + /// Sources the program shows or plays that stopped delivering, and what. + pub stalled: Vec<(String, Stall)>, } /// What the studio publishes, for grading its delivery. @@ -578,12 +588,16 @@ fn problems(facts: &Facts<'_>) -> Vec { add(Verdict::Unhealthy, format!("The output stopped: {failed}.")); } - for (name, since) in &local.stalled { + for (name, stall) in &local.stalled { + let instead = match stall { + Stall::Picture(_) => "repeats its last one", + Stall::Sound(_) => "plays silence for it", + }; add( Verdict::Unhealthy, format!( - "Source `{name}` stalled: no new picture for {:.0} s, so the program repeats its last one.", - since.as_secs_f64() + "Source `{name}` stalled: {}, so the program {instead}.", + without(*stall) ), ); } @@ -1222,12 +1236,15 @@ mod tests { ); } - /// Problems list worst first: an unreachable relay and a frozen source - /// above a slow round trip. + /// Problems list worst first: an unreachable relay and a frozen or silent + /// source above a slow round trip. #[test] fn problems_list_worst_first() { let local = Local { - stalled: vec![("camera".into(), Duration::from_secs(3))], + stalled: vec![ + ("camera".into(), Stall::Picture(Duration::from_secs(3))), + ("mic".into(), Stall::Sound(Duration::from_secs(2))), + ], ..Default::default() }; let uplink = Uplink { @@ -1243,6 +1260,7 @@ mod tests { [ "The relay https://a is unreachable (timed out), so nothing reaches viewers until a relay reconnects.", "Source `camera` stalled: no new picture for 3 s, so the program repeats its last one.", + "Source `mic` stalled: no new sound for 2 s, so the program plays silence for it.", "The relay is far away: a round trip to it takes 400 ms.", ] ); diff --git a/rs/moq-studio/src/ui.rs b/rs/moq-studio/src/ui.rs index 74292d4..cb4c091 100644 --- a/rs/moq-studio/src/ui.rs +++ b/rs/moq-studio/src/ui.rs @@ -1235,9 +1235,9 @@ impl Panels { feed.and_then(|feed| feed.stalled(now)), ) { (Some(err), _, _) => status(ui, &Status::Failed(err.to_string())), - (None, _, Some(since)) => { + (None, _, Some(stall)) => { ui.colored_label(DEGRADED, "stalled") - .on_hover_text(format!("no new picture for {:.0} s", since.as_secs_f64())); + .on_hover_text(health::without(stall)); } (None, Some(feed), None) => status(ui, &feed.status()), (None, None, None) => status(ui, &Status::Starting),