diff --git a/quest/m0/README.md b/quest/m0/README.md index 64c5905..40c732b 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -82,7 +82,6 @@ repository. - [Studio on macOS with capture granted](/quest/m0/app-macos.md) - camera, screen, and audio capture run on a Mac, not only Linux - [Studio on Windows](/quest/m0/app-windows.md) - the studio runs on Windows, not only Linux - [Viewer-side composition](/quest/m0/compose/README.md) - sources as their own tracks plus a layout track, composed by viewers, with a server-composed program as the fallback -- [The program encoder drains](/quest/m0/program-drain.md) - a hardware encoder never loses the program's last frames at exit or on pause - [Broadcast sources play in sync](/quest/m0/broadcast-sync.md) - subscribed broadcasts and guests play audio and video on the publisher's timestamps, with `moq play`'s delay - [Wayland display and window sources](/quest/m0/wayland-sources.md) - portal-picked sources that reopen without the picker - [Live encode control](/quest/m0/encode-control.md) - simulcast rungs that follow the bandwidth grant, live bitrate and resolution, keyframes on request diff --git a/quest/m0/program-drain.md b/quest/m0/program-drain.md deleted file mode 100644 index e4df0f7..0000000 --- a/quest/m0/program-drain.md +++ /dev/null @@ -1,17 +0,0 @@ -# [XS] The program encoder drains - -## Goal - -The program's encoder drains its buffered pictures before it stops at exit -or when a new canvas ends its rendition, and before the studio publishes the -gap marker of an on-demand pause, so a -pipelined hardware encoder never loses the program's last frames. - -## Plan - -Source encoders already drain before they are replaced or their track ends, -and a failed drain aborts the track -([#3](https://github.com/moq-dev/studio/pull/3)). Give the program the same -behaviour through the same path rather than a second one. openh264 buffers -nothing, so prove it on an encoder that does, or with a test double that -holds frames. diff --git a/rs/moq-scene/src/publish.rs b/rs/moq-scene/src/publish.rs index 35636e8..c368f85 100644 --- a/rs/moq-scene/src/publish.rs +++ b/rs/moq-scene/src/publish.rs @@ -218,14 +218,153 @@ impl Drop for Listing<'_> { } } -/// A captured or replayed source's video, encoded frame by frame as it is shown. -pub(crate) struct Video { +/// What an [`Encoding`] needs of its encoder: a +/// [`moq_video::encode::Encoder`], or a test's stand-in that holds pictures the +/// way a pipelined hardware backend does. +pub trait Encode { + /// [`Encoder::encode`](moq_video::encode::Encoder::encode). + fn encode(&mut self, frame: &moq_video::Frame) -> Result, moq_video::Error>; + /// [`Encoder::flush`](moq_video::encode::Encoder::flush). + fn flush(&mut self) -> Result, moq_video::Error>; + /// [`Encoder::finish`](moq_video::encode::Encoder::finish). + fn finish(self) -> Result, moq_video::Error>; + /// [`Encoder::cut`](moq_video::encode::Encoder::cut). + fn cut(&mut self) -> Result<(), moq_video::Error>; +} + +impl Encode for moq_video::encode::Encoder { + fn encode(&mut self, frame: &moq_video::Frame) -> Result, moq_video::Error> { + self.encode(frame) + } + + fn flush(&mut self) -> Result, moq_video::Error> { + self.flush() + } + + fn finish(self) -> Result, moq_video::Error> { + self.finish() + } + + fn cut(&mut self) -> Result<(), moq_video::Error> { + self.cut() + } +} + +/// An encoder and the video track it publishes into. +/// +/// A pipelined backend still holds its last pictures when its encoder is +/// replaced, the track pauses, or the track ends, so each of those publishes +/// them first. Dropping this ends the track cleanly rather than leaving +/// subscribers with a dropped one, and a drain that fails aborts the track with +/// the reason. +pub struct Encoding { /// Taken only to abort or finish it, as the track ends. producer: Option, + encoder: Option, +} + +impl Encoding { + /// Publish into `producer`, with no encoder open yet. + pub fn new(producer: moq_video::encode::Producer) -> Self { + Self { + producer: Some(producer), + encoder: None, + } + } + + /// Whether an encoder is open. + pub fn is_open(&self) -> bool { + self.encoder.is_some() + } + + /// Encode with `encoder` from here on, after draining the one it replaces. + pub fn open(&mut self, encoder: E) -> Result<(), Error> { + self.drain()?; + self.encoder = Some(encoder); + Ok(()) + } + + /// Encode `frame` and publish what the encoder hands back. + /// + /// # Panics + /// + /// When no encoder is [open](Self::open). + pub fn encode(&mut self, frame: &moq_video::Frame) -> Result<(), Error> { + let encoder = self.encoder.as_mut().expect("an encoder is open"); + let encoded = encoder.encode(frame)?; + self.producer.as_mut().expect("open until it ends").publish(&encoded)?; + Ok(()) + } + + /// Close the encoder, publishing what it still holds. + pub fn drain(&mut self) -> Result<(), Error> { + if let (Some(encoder), Some(producer)) = (self.encoder.take(), &mut self.producer) { + producer.publish(&encoder.finish()?)?; + } + Ok(()) + } + + /// Mark a gap in the track after [draining](Self::drain) the encoder, so + /// the next one [opened](Self::open) starts with a keyframe, even on a + /// backend that cannot [cut](Encode::cut). + pub(crate) fn discontinuity(&mut self) -> Result<(), Error> { + self.drain()?; + self.producer.as_mut().expect("open until it ends").discontinuity()?; + Ok(()) + } + + /// Mark a gap in the track, such as the last viewer leaving an on-demand + /// one, keeping the encoder and its session open. What the encoder still + /// holds lands before the gap rather than after it, and the next picture + /// is a keyframe, so a viewer arriving after the gap starts there. + pub fn pause(&mut self) -> Result<(), Error> { + let producer = self.producer.as_mut().expect("open until it ends"); + if let Some(encoder) = &mut self.encoder { + producer.publish(&encoder.flush()?)?; + encoder.cut()?; + } + producer.discontinuity()?; + Ok(()) + } + + /// End the track cleanly, after publishing what the encoder still holds. + /// A failure aborts the track with the reason. + pub fn finish(mut self) -> Result<(), Error> { + self.end() + } + + /// End the track, failed with `err` so viewers see why, dropping what the + /// encoder still holds. + pub fn abort(mut self, err: moq_net::Error) { + self.encoder = None; + if let Some(producer) = self.producer.take() { + producer.abort(err); + } + } + + fn end(&mut self) -> Result<(), Error> { + let drained = self.drain(); + let Some(mut producer) = self.producer.take() else { + return drained; + }; + let result = drained.and_then(|()| Ok(producer.finish()?)); + if let Err(err) = &result { + producer.abort(moq_net::Error::Transport(err.to_string())); + } + result + } +} + +impl Drop for Encoding { + fn drop(&mut self) { + let _ = self.end(); + } +} + +/// A captured or replayed source's video, encoded frame by frame as it is shown. +pub(crate) struct Video { + track: Encoding, config: moq_video::encode::Config, - /// Opened on the first frame, and reopened when the picture's size or color - /// space changes, which moves the rendition to a new track. - encoder: Option, clock: moq_mux::Clock, /// Where the capture's private timeline starts on the studio clock, in /// microseconds, taken at the first frame. @@ -254,15 +393,16 @@ impl Video { .with_hls_hint(config.gop.duration(config.framerate)); let producer = moq_video::encode::Producer::new(listing.broadcast.clone(), reserved, rendition)?; Ok(Self { - producer: Some(producer), + track: Encoding::new(producer), config, - encoder: None, clock: listing.studio.clock(), epoch: None, }) } - /// Encode and publish `frame`, restamping it onto the studio clock. + /// Encode and publish `frame`, restamping it onto the studio clock. The + /// encoder opens on the first frame, and reopens when the picture's size or + /// color space changes, which moves the rendition to a new track. pub(crate) fn encode(&mut self, frame: &mut moq_video::Frame) -> Result<(), Error> { let captured = frame.timestamp.as_micros(); let epoch = *self @@ -276,60 +416,29 @@ impl Video { self.config.width = size.width; self.config.height = size.height; self.config.color = color; - self.drain()?; + // Before the next one opens, so a GPU at its session cap has room. + self.track.drain()?; } - let encoder = match &mut self.encoder { - Some(encoder) => encoder, - None => self.encoder.insert(moq_video::encode::Encoder::new(&self.config)?), - }; - let encoded = if frame.size() == size { - encoder.encode(frame)? + if !self.track.is_open() { + self.track.open(moq_video::encode::Encoder::new(&self.config)?)?; + } + if frame.size() == size { + self.track.encode(frame) } else { - encoder.encode(&frame.resize(size, &Default::default())?)? - }; - self.producer.as_mut().expect("open until it ends").publish(&encoded)?; - Ok(()) + self.track.encode(&frame.resize(size, &Default::default())?) + } } /// Mark a break, such as a gap in a replay: what follows does not continue /// what came before, and a viewer arriving during it waits for what follows /// rather than being served the last picture as if it were live. pub(crate) fn discontinuity(&mut self) -> Result<(), Error> { - self.drain()?; - self.producer.as_mut().expect("open until it ends").discontinuity()?; - Ok(()) - } - - /// Publish what the encoder still holds and close it, so a pipelined - /// backend's last pictures are not lost when the encoder is replaced or the - /// track ends. - fn drain(&mut self) -> Result<(), Error> { - if let (Some(encoder), Some(producer)) = (self.encoder.take(), &mut self.producer) { - producer.publish(&encoder.finish()?)?; - } - Ok(()) + self.track.discontinuity() } /// End the track, failed with `err` so viewers see why. - pub(crate) fn abort(mut self, err: &Error) { - if let Some(producer) = self.producer.take() { - producer.abort(moq_net::Error::Transport(err.to_string())); - } - } -} - -impl Drop for Video { - fn drop(&mut self) { - // A capture that ends, or a source removed or edited, ends its track - // cleanly rather than leaving subscribers with a dropped one. - let drained = self.drain(); - match (self.producer.take(), drained) { - (Some(mut producer), Ok(())) => { - let _ = producer.finish(); - } - (Some(producer), Err(err)) => producer.abort(moq_net::Error::Transport(err.to_string())), - (None, _) => {} - } + pub(crate) fn abort(self, err: &Error) { + self.track.abort(moq_net::Error::Transport(err.to_string())); } } @@ -569,6 +678,125 @@ mod tests { .await; } + /// Holds its newest pictures back the way a pipelined hardware encoder + /// does, handing them over only on a flush or finish. openh264, the encoder + /// on every machine, holds nothing. + struct Pipelined { + encoder: moq_video::encode::Encoder, + held: Vec, + } + + impl Pipelined { + const DEPTH: usize = 2; + } + + impl Encode for Pipelined { + fn encode(&mut self, frame: &moq_video::Frame) -> Result, moq_video::Error> { + self.held.extend(self.encoder.encode(frame)?); + let ready = self.held.len().saturating_sub(Self::DEPTH); + Ok(self.held.drain(..ready).collect()) + } + + fn flush(&mut self) -> Result, moq_video::Error> { + let mut encoded = std::mem::take(&mut self.held); + encoded.extend(self.encoder.flush()?); + Ok(encoded) + } + + fn finish(mut self) -> Result, moq_video::Error> { + let mut encoded = std::mem::take(&mut self.held); + encoded.extend(self.encoder.finish()?); + Ok(encoded) + } + + fn cut(&mut self) -> Result<(), moq_video::Error> { + self.encoder.cut() + } + } + + /// A pipelined encoder's last pictures land before a pause's gap rather + /// than after it, the picture after the gap opens a group of its own, and + /// the pictures it still holds as the track ends are published before the + /// track finishes cleanly. + #[tokio::test] + async fn a_pipelined_encoder_loses_no_pictures() { + let (publish, _catalog, consumer) = studio(); + let claim = publish.claim("program"); + let listing = Listing::new(&claim, &publish).unwrap(); + let size = moq_video::Size::new(64, 48); + let framerate = moq_video::FrameRate::new(30, 1).unwrap(); + let mut config = moq_video::encode::Config::new(size.width, size.height, framerate); + config.kind = moq_video::encode::Kind::Named("openh264".into()); + let rendition = config.probe().await.unwrap(); + let producer = + moq_video::encode::Producer::new(listing.broadcast.clone(), listing.catalog.reserve(), rendition).unwrap(); + let (name, rendition) = listing.catalog.snapshot().video.renditions.pop_first().unwrap(); + // Every group, however far behind the newest, since this reads only once + // the track has ended. + let mut subscription = moq_net::track::Subscription::default(); + subscription.max_delay = None; + let subscriber = consumer.track(&name).unwrap().subscribe(subscription).await.unwrap(); + let container = moq_mux::catalog::hang::Container::try_from(&rendition).unwrap(); + let mut reader = moq_mux::container::Consumer::new(subscriber, container); + + let mut track = Encoding::new(producer); + let encoder = moq_video::encode::Encoder::new(&config).unwrap(); + track + .open(Pipelined { + encoder, + held: Vec::new(), + }) + .unwrap(); + for millis in [0, 33, 66] { + track.encode(&gray(size, millis)).unwrap(); + } + track.pause().unwrap(); + for millis in [500, 533, 566] { + track.encode(&gray(size, millis)).unwrap(); + } + track.finish().unwrap(); + + /// What a viewer reads, in order. + #[derive(Debug, PartialEq)] + enum Seen { + /// A group opens. + Group, + /// A picture, at its timestamp in milliseconds. + Picture(u128), + /// The media stops for a gap. + End, + } + let mut seen = Vec::new(); + let mut current = None; + while let Some(read) = reader.read().await.unwrap() { + match read { + moq_mux::container::Read::Frame { group, frame } => { + if current.replace(group) != Some(group) { + seen.push(Seen::Group); + } + seen.push(Seen::Picture(frame.timestamp.as_micros() / 1000)); + } + moq_mux::container::Read::End { .. } => seen.push(Seen::End), + moq_mux::container::Read::Gap { .. } => panic!("pictures went missing"), + } + } + use Seen::*; + assert_eq!( + seen, + [ + Group, + Picture(0), + Picture(33), + Picture(66), + End, + Group, + Picture(500), + Picture(533), + Picture(566), + ] + ); + } + /// An edited source's replacement owns the listing from the moment it /// opens, even when the old source's driver only gets to list itself later, /// and the old one's end leaves the replacement listed. diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs index fd4d57e..b501842 100644 --- a/rs/moq-studio/src/engine.rs +++ b/rs/moq-studio/src/engine.rs @@ -23,6 +23,7 @@ use std::time::Duration; use anyhow::Context; use bytes::Bytes; use kio::clock::Instant; +use moq_scene::publish::Encoding; use moq_scene::source::{self, Driver, Sources}; use moq_scene::{Canvas, Collection, Compositor, Draft, Edit, History, wgpu}; use moq_video::render::export::{Exporter, Pending, Route}; @@ -411,6 +412,10 @@ pub struct Engine { pub room: Option>, /// Keeps broadcast sources' origin open when no connection holds it. _origin: Option, + /// Set by [`stop`](Self::stop), which ends the program clock. + stopped: AtomicBool, + /// The program clock's thread, which [`stop`](Self::stop) waits on. + program: Mutex>>, } /// The audio the program publishes. @@ -514,6 +519,8 @@ impl Engine { output, room, _origin: origin, + stopped: AtomicBool::new(false), + program: Mutex::default(), }); let (video, audio, clock, publisher) = match publish { Some(Publish { @@ -535,9 +542,10 @@ impl Engine { None => Duration::ZERO, }; let program = engine.clone(); - clocked("program".into(), async move || { + let thread = clocked("program".into(), async move || { program.run_program(start, epoch, video).await })?; + *engine.program.lock().unwrap() = Some(thread); let mix = engine.clone(); clocked("audio".into(), async move || mix.run_audio(start, epoch, audio).await)?; let save = engine.clone(); @@ -553,6 +561,20 @@ impl Engine { Ok(engine) } + /// Stop the program clock, then wait for each rendition's encoder to + /// publish the copies on their way to it and what it still holds, and to + /// end its video track, so quitting loses none of the program's last + /// pictures. + pub fn stop(&self) { + self.stopped.store(true, Ordering::Relaxed); + let thread = self.program.lock().unwrap().take(); + if let Some(thread) = thread + && thread.join().is_err() + { + tracing::error!("the program thread panicked"); + } + } + /// Write the collection and its history to their files unless they /// already hold them, or the scene file changed outside the studio. pub fn save(&self) -> anyhow::Result<()> { @@ -626,6 +648,9 @@ impl Engine { /// drawn at and opens another, since a track never changes what it /// carries. Drawing outlives the encoder: a failed output stops copying the /// program, and the preview and program keep moving. + /// + /// Runs until the studio [stops](Self::stop), and returns once every + /// rendition's encode thread has finished its track. async fn run_program(&self, start: Instant, epoch: Duration, video: Option<(Renditions, Rendition)>) { let (renditions, mut rendition) = video.unzip(); let mut canvas = self.state.lock().unwrap().compositor.canvas(); @@ -646,6 +671,21 @@ impl Engine { loop { let tick = clock.tick().await; + if self.stopped.load(Ordering::Relaxed) { + // Without a sender, each encoder publishes what it was sent and + // still holds, then finishes its track. + ending.extend(rendition.take().map(Rendition::end)); + for thread in ending { + if thread.join().is_err() { + tracing::error!("an encode thread panicked"); + } + } + // It holds the threads of the renditions it waits on. + if let Some((_, probe)) = probing { + let _ = probe.join(); + } + return; + } let (program, drawn) = { let mut guard = self.state.lock().unwrap(); let state = &mut *guard; @@ -1128,14 +1168,15 @@ fn encode( rendition: u64, opened: tokio::sync::oneshot::Sender>, video: mpsc::Receiver