feat(studio): record the program, and optionally each source, to MP4

Cargo.toml

@@ -41,6 +41,7 @@ moq-video = { path = "moq/rs/moq-video", default-features = false }
 
 anyhow = "1"
 bytes = "1"
+humantime = "2.4"
 humantime-serde = "1.1.1"
 pollster = "1.0"
 serde = { version = "1", features = ["derive"] }

README.md

@@ -16,8 +16,10 @@ cargo run --release -p moq-studio --features capture,openh264 -- \
 - `show.toml` is the scene collection. If it does not exist yet,
   [guided setup](#first-launch) proposes one and creates it. Changes are saved
   to it a second after you stop editing, with no save step.
-- Without `--connect`, the studio composes and previews but publishes nothing,
-  and broadcast sources stay offline.
+- `--recordings <dir>` is where recordings are saved, by default beside
+  `show.toml`.
+- Without `--connect`, the studio composes, previews, and records but
+  publishes nothing, and broadcast sources stay offline.
 - `--failover <url>` publishes the same broadcast to a second relay under the
   same broadcast instance, so a viewer linked to both resumes mid-group from
   it when the first fails. Both relays must speak the one `--connect-version`,
@@ -97,7 +99,7 @@ Every panel is a tab you can drag, split, or stack.
 | Guests | Invite a guest, show their join link as a QR code, see who joined, and take a guest out of every scene. |
 | Filters | Crop the selected source or item, add, order, and set its filters, set an item's blend mode, and copy or paste filters. |
 | Audio | Set each audio source's gain and mute, and watch its meter and the program's. |
-| Output | Set the canvas size and frame rate, and see the broadcast path, whether the program is encoding, and the encoder's work. |
+| Output | Start and stop recording, set the canvas size and frame rate, and see the broadcast path, whether the program is encoding, and the encoder's work. |
 | Health | See how the stream reaches viewers, relay by relay and track by track, and the cause of each problem. |
 | Replay | Replay the last moments of the program or a source in a replay source, slowed down if you like, or save them as a clip. |
 
@@ -382,12 +384,13 @@ rendition's track ends and a new one is listed in its place in a catalog
 update, on the same timeline, and viewers switch to it; the video pauses while
 the new encoder opens. Items keep their place in canvas pixels. A canvas the
 encoder cannot take stops the video, saying why, until the canvas changes
-again.
+again. At startup, it stops the studio when live; without `--connect`, the
+studio starts without video.
 
 The program encodes whether or not anyone watches, so a viewer who arrives
 late can fetch the recent past. `--on-demand` copies and encodes the video only
-while someone watches it, which saves that work but leaves nothing from before
-the first viewer; audio encodes either way. A hardware encoder that reads
+while someone watches or records it, which saves that work but leaves nothing
+from before the first viewer; audio encodes either way. A hardware encoder that reads
 the GPU (NVENC on Linux, VideoToolbox on macOS, Media Foundation on Windows)
 takes each frame there when it can read the GPU the studio draws on. When it
 cannot, such as NVENC while the studio draws on a laptop's integrated GPU, the
@@ -420,7 +423,8 @@ the hang draft specifies source listings:
 
 A captured source encodes as H.264 for as long as it is in the collection,
 whether or not anyone watches, so a late viewer can fetch its recent past; the
-relay uploads only what someone subscribes to. A capture that changes size or
+relay uploads only what someone subscribes to. Without `--connect`, it encodes
+only while a recording reads it. A capture that changes size or
 color space moves to a new track, which its catalog entry follows. Images and
 colors publish once, as the image file and as a one-pixel PNG. Removing or
 editing a source ends its track. A track carries the plain capture, before any
@@ -439,6 +443,33 @@ past the cap fail rather than the program. When the
 program encodes in software, a source takes hardware where it can and software
 otherwise.
 
+### Recording
+
+**Start recording** in the Output panel saves the program to
+`<scene file name> <UTC start time>.mp4` in the `--recordings` directory, live
+or not. With **each source too**, every camera, display, and window that
+publishes a track of its own records into a file of its own beside it,
+`<scene file name> <UTC start time> <source>.mp4`, holding the capture before
+any filter. Broadcast sources, guests included, publish no track of the
+studio's, so they get no file of their own. A source added while recording
+starts its file then, and one removed finishes its file then. A new canvas
+size or frame rate, or a capture that changes size or color space, moves the
+video to a new track, so the program or that source goes on in a new file,
+numbered after the first, since players show only one video track. A file
+starts with up to 2 s from before it opened, so a recording starts a little
+early, and a track that goes on into the next file, such as the program
+audio, repeats up to 2 s of the last. A
+recording reads the broadcast the way a viewer does, so it shares the live
+encoders and starts and stops without touching what the relay serves. Without
+`--connect`, or with `--on-demand`, the program encodes only while something
+records or watches it.
+
+Each file plays while it grows, so a crash leaves a playable fragmented MP4,
+and becomes a regular MP4 when the recording stops or the studio quits.
+Quitting first lets the program's encoders publish what they still hold, and
+the program's file keeps it. A file nothing reached before the recording
+stopped is removed.
+
 ### Source frames
 
 Sources make the opposite trip. A frame already on the GPU is drawn where it

justfile

@@ -55,7 +55,7 @@ render-sw:
     icd=("$mesa"/share/vulkan/icd.d/lvp_icd.*.json)
     export LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}"
     cargo nextest run --locked -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/)'
-    cargo nextest run --locked -p moq-studio --features openh264 --run-ignored only -E 'test(=engine::tests::the_program_reaches_a_relay) | test(=setup::tests::times_every_canvas)'
+    cargo nextest run --locked -p moq-studio --features openh264 --run-ignored only -E 'test(/^engine::tests::the_program_/) | test(=setup::tests::times_every_canvas)'
     # CEF's browser process opens an X display even to paint offscreen.
     xvfb=$(nix build --inputs-from . --no-link --print-out-paths nixpkgs#xvfb-run)
     "$xvfb/bin/xvfb-run" -a cargo nextest run --locked -p moq-studio --features cef --run-ignored only -E 'binary(browser)'

quest/m0/README.md

@@ -104,7 +104,6 @@ repository.
 - [Guest return feed](/quest/m0/guest-return.md) - guests see the program and hear a mix-minus
 - [Audio mixes](/quest/m0/mixes.md) - per-output mixes, meters, monitoring, and sync offsets
 - [Audio filters](/quest/m0/audio-filters.md) - per-source noise suppression, gate, compressor, limiter, and plugins, configured from the studio
-- [Recording](/quest/m0/recording.md) - each canvas's program, and optionally each source, recorded to disk alongside live outputs
 - [Browser sources on macOS and Windows](/quest/m0/browser-source.md) - the out-of-process CEF pages Linux has, with shared GPU textures on the other desktops
 - [Browser sources keep a profile](/quest/m0/browser-profile.md) - one persistent profile, so logins survive restarts, and a reload action
 - [Browser source popups](/quest/m0/browser-popups.md) - dropdowns and other in-page popups draw and take input

rs/moq-scene/src/publish.rs

@@ -8,7 +8,8 @@
 //! be played, or packaged for HLS, as a variant of the program.
 //!
 //! A source encodes while it runs, whatever the demand, so a late viewer finds
-//! its recent past; MoQ uploads only what a subscriber pulls.
+//! its recent past; MoQ uploads only what a subscriber pulls. On
+//! [demand](Publish::on_demand), it encodes only while subscribed.
 
 use std::collections::BTreeMap;
 use std::sync::{Arc, Mutex};
@@ -57,6 +58,7 @@ pub struct Publish {
 	catalog: Catalog,
 	bandwidth: moq_net::bandwidth::Allocator,
 	kind: moq_video::encode::Kind,
+	on_demand: bool,
 	owners: Arc<Mutex<Owners>>,
 }
 
@@ -88,10 +90,19 @@ impl Publish {
 			catalog,
 			bandwidth,
 			kind,
+			on_demand: false,
 			owners: Default::default(),
 		}
 	}
 
+	/// Encode each source's video only while someone subscribes to it, for a
+	/// studio whose only readers are its own recordings, which never fetch a
+	/// past they were not reading.
+	pub fn on_demand(mut self) -> Self {
+		self.on_demand = true;
+		self
+	}
+
 	/// What each source track the studio catalog lists has written, keyed by
 	/// rendition ID, as [`Catalog::stats`](moq_mux::catalog::Producer::stats)
 	/// reports the program's. The broadcast names every track uniquely, so all
@@ -151,6 +162,7 @@ impl Drop for Claim {
 pub(crate) struct Listing<'a> {
 	claim: &'a Claim,
 	kind: moq_video::encode::Kind,
+	on_demand: bool,
 	broadcast: moq_net::broadcast::Producer,
 	studio: Catalog,
 	catalog: moq_mux::catalog::Producer,
@@ -172,6 +184,7 @@ impl<'a> Listing<'a> {
 		Ok(Self {
 			claim,
 			kind: publish.kind.clone(),
+			on_demand: publish.on_demand,
 			broadcast: publish.broadcast.clone(),
 			studio: publish.catalog.clone(),
 			catalog,
@@ -277,6 +290,12 @@ impl<E: Encode> Encoding<E> {
 		self.encoder.is_some()
 	}
 
+	/// Whether anyone subscribes to the track.
+	pub(crate) fn wanted(&self) -> bool {
+		let producer = self.producer.as_ref().expect("open until it ends");
+		producer.demand().wanted().is_some()
+	}
+
 	/// Encode with `encoder` from here on, after draining the one it replaces.
 	pub fn open(&mut self, encoder: E) -> Result<(), Error> {
 		self.drain()?;
@@ -365,6 +384,8 @@ impl<E: Encode> Drop for Encoding<E> {
 pub(crate) struct Video {
 	track: Encoding,
 	config: moq_video::encode::Config,
+	/// Encodes only while the track is subscribed.
+	on_demand: bool,
 	clock: moq_mux::Clock,
 	/// Where the capture's private timeline starts on the studio clock, in
 	/// microseconds, taken at the first frame.
@@ -395,21 +416,32 @@ impl Video {
 		Ok(Self {
 			track: Encoding::new(producer),
 			config,
+			on_demand: listing.on_demand,
 			clock: listing.studio.clock(),
 			epoch: None,
 		})
 	}
 
-	/// Encode and publish `frame`, restamping it onto the studio clock. The
-	/// encoder opens on the first frame, and reopens when the picture's size or
-	/// color space changes, which moves the rendition to a new track.
+	/// Encode and publish `frame`, restamping it onto the studio clock, unless
+	/// it is on demand and nobody subscribes. The encoder opens on the first
+	/// frame it encodes, and reopens when the picture's size or color space
+	/// changes, which moves the rendition to a new track.
 	pub(crate) fn encode(&mut self, frame: &mut moq_video::Frame) -> Result<(), Error> {
 		let captured = frame.timestamp.as_micros();
 		let epoch = *self
 			.epoch
 			.get_or_insert_with(|| self.clock.now().as_micros().saturating_sub(captured));
 		frame.timestamp = moq_net::Timestamp::from_micros((epoch + captured) as u64).map_err(moq_video::Error::from)?;
 
+		if self.on_demand && !self.track.wanted() {
+			// The next subscriber starts on a fresh encoder's keyframe, and the
+			// session is free meanwhile; the gap is marked so nobody reads across it.
+			if self.track.is_open() {
+				self.track.discontinuity()?;
+			}
+			return Ok(());
+		}
+
 		let size = encodable(frame.size());
 		let color = frame.surface.color();
 		if size != self.config.size() || color.is_some_and(|color| Some(color) != self.config.color) {
@@ -678,6 +710,60 @@ mod tests {
 		.await;
 	}
 
+	/// An on-demand source encodes only while its track is subscribed, holding
+	/// no encoder between subscribers.
+	#[tokio::test]
+	async fn an_on_demand_video_encodes_while_subscribed() {
+		local(async {
+			let (publish, catalog, consumer) = studio();
+			let publish = publish.on_demand();
+			let claim = publish.claim("camera");
+			let listing = Listing::new(&claim, &publish).unwrap();
+			let framerate = moq_video::FrameRate::new(30, 1).unwrap();
+			let size = moq_video::Size::new(64, 48);
+
+			tokio::select! {
+				result = listing.mirror() => panic!("the mirror ended: {result:?}"),
+				() = async {
+					let mut video = Video::new(&listing, size, framerate, None).await.unwrap();
+					video.encode(&mut gray(size, 0)).unwrap();
+					assert!(!video.track.is_open(), "nothing encodes unsubscribed");
+
+					// Advertised before anything encodes, so a recording can find it.
+					let rendition = |sources: &BTreeMap<String, hang::Catalog>| {
+						sources.get("camera").is_some_and(|camera| !camera.video.renditions.is_empty())
+					};
+					let snapshot = listed(&catalog, rendition).await;
+					let name = snapshot.ext.studio.sources["camera"].video.renditions.keys().next().unwrap().clone();
+					let mut track = consumer.track(&name).unwrap().subscribe(None).await.unwrap();
+					video.encode(&mut gray(size, 33)).unwrap();
+					assert!(video.track.is_open(), "a subscriber gets pictures");
+					let mut group = track.recv_group().await.unwrap().expect("a group");
+					assert!(group.read_frame().await.unwrap().is_some());
+					let first = group.sequence;
+
+					drop(group);
+					drop(track);
+					for _ in 0..100 {
+						tokio::task::yield_now().await;
+					}
+					video.encode(&mut gray(size, 66)).unwrap();
+					assert!(!video.track.is_open(), "the encoder closes once nobody subscribes");
+
+					// The next subscriber gets a fresh encoder's pictures, in a group
+					// after the gap rather than the one before it.
+					let mut track = consumer.track(&name).unwrap().subscribe(None).await.unwrap();
+					video.encode(&mut gray(size, 100)).unwrap();
+					assert!(video.track.is_open(), "a new subscriber reopens the encoder");
+					let mut group = track.recv_group().await.unwrap().expect("a group");
+					assert!(group.sequence > first, "{} after {first}", group.sequence);
+					assert!(group.read_frame().await.unwrap().is_some());
+				} => {}
+			}
+		})
+		.await;
+	}
+
 	/// Holds its newest pictures back the way a pipelined hardware encoder
 	/// does, handing them over only on a flush or finish. openh264, the encoder
 	/// on every machine, holds nothing.

rs/moq-studio/Cargo.toml

@@ -47,6 +47,9 @@ egui = "0.36"
 # Docked, rearrangeable panels. 0.21 is the newest release and pins egui 0.36.
 egui_dock = "0.21"
 hang = { workspace = true }
+# Stamps a recording's file name with when it started. Already in the graph
+# through moq-scene's humantime-serde.
+humantime = { workspace = true }
 kio = { workspace = true, features = ["async", "runtime"] }
 moq-audio = { workspace = true, features = ["bus"] }
 # Signs guest tokens with the room key.