feat(scene): video effects: color filters, blur, drop shadow, keys, LUTs, masks, crop, and blend modes

README.md

@@ -81,7 +81,7 @@ Every panel is a tab you can drag, split, or stack.
 | Items | Place, show, hide, and order what the preview scene or the top layer draws, back to front. Select an item to filter or copy it. |
 | Sources | Add a source by hand, see whether each and its own track are live and where it is shown, clone one, and remove one. Select a source to filter it. |
 | Devices | List this machine's cameras, displays, windows, and audio inputs, and add one as a source. |
-| Filters | Add, order, and set the filters of the selected source or item, and copy or paste them. |
+| Filters | Crop the selected source or item, add, order, and set its filters, set an item's blend mode, and copy or paste filters. |
 | Audio | Set each audio source's gain and mute, and watch its meter and the program's. |
 | Output | See the broadcast path, whether the program is encoding, the canvas, and the encoder's work. |
 | Health | See how the stream reaches viewers, track by track, and the cause of each problem. |
@@ -160,11 +160,23 @@ source = "camera"
 [video.camera-faded]
 filters = [{ kind = "opacity", amount = 0.5 }]
 
+[sources.grade]
+kind = "lut"
+path = "teal-orange.cube"
+
+[video.camera]
+crop = { left = 0.1, right = 0.1 } # trim 10% of the picture off each side
+filters = [
+    { kind = "chroma_key", color = "#00ff00", similarity = 0.4, smoothness = 0.08, spill = 0.1 },
+    { kind = "lut", source = "grade", amount = 0.8 },
+]
+
 [[scenes]]
 name = "Main"
 items = [
     { source = "camera", rect = { x = 0, y = 0, width = 1920, height = 1080 } },
-    { source = "guest", rect = { x = 1280, y = 720, width = 640, height = 360 }, filters = [{ kind = "opacity", amount = 0.8 }] },
+    { source = "guest", rect = { x = 1280, y = 720, width = 640, height = 360 }, filters = [{ kind = "drop_shadow", x = 8, y = 8, blur = 12, color = "#00000080" }] },
+    { source = "logo", rect = { x = 0, y = 0, width = 1920, height = 1080 }, blend = "multiply", filters = [{ kind = "opacity", amount = 0.3 }] },
 ]
 
 [[scenes]]
@@ -181,10 +193,36 @@ rect = { x = 1600, y = 40, width = 280, height = 120 }
 Source kinds are `camera`, `display`, and `microphone` (each with an optional
 `id`, the default device without one), `window` (with an `id`), `system_audio`
 (macOS), `image` (PNG or JPEG), `color` (`#rrggbb` or `#rrggbbaa`),
-`broadcast`, and `clone` (with the `source` it shows). The Devices panel fills
-in ids. `[video.<source>]` lists a source's filters, and an item's `filters`
-its own; the one filter so far is `opacity`, from 0 to 1, as in CSS. `[[top]]`
-items make up the top layer.
+`broadcast`, `clone` (with the `source` it shows), and `lut` (a 3D Adobe or
+Resolve `.cube` file, which no item places, but which grades what a `lut`
+filter names it in). The Devices panel fills in ids. `[[top]]` items make up the top
+layer.
+
+`[video.<source>]` sets a source's `crop` and `filters`, wherever it is shown,
+and an item's `crop`, `filters`, and `blend` its own. A `crop` trims a fraction
+(0 to 1) of the picture off its `left`, `top`, `right`, and `bottom`, opposite
+sides together trimming less than the whole, and stretches the rest over the
+item; an item's crop trims what its source's
+leaves. Filters apply in order, the source's then the item's, with lengths in
+canvas pixels:
+
+| Filter | Settings |
+| --- | --- |
+| `opacity`, `grayscale`, `sepia`, `invert` | `amount`, 0 to 1, as in CSS |
+| `brightness`, `contrast`, `saturate` | `amount`, 0 or more, 1 leaving the picture as it is, as in CSS |
+| `hue_rotate` | `degrees`, as in CSS |
+| `blur` | `radius`, the standard deviation; the edges fade out, as in CSS |
+| `drop_shadow` | `x`, `y`, `blur` (the standard deviation), and `color`, as in CSS |
+| `chroma_key` | `color` keyed out, and `similarity`, `smoothness`, and `spill` from 0 to 1 |
+| `luma_key` | `min` and `max` luma kept, `min` at most `max`, and `smoothness`, all from 0 to 1 |
+| `lut` | the `lut` `source` to grade through, and `amount` from 0 to 1 |
+| `mask` | a `source` whose picture, stretched over the item, keeps what it covers: by its alpha, or `mode = "luminance"` |
+
+An item's `blend` mixes it with what is under it by a CSS `mix-blend-mode`:
+`multiply`, `screen`, `overlay`, `darken`, `lighten`, `color_dodge`,
+`color_burn`, `hard_light`, `soft_light`, `difference`, `exclusion`, `hue`,
+`saturation`, `color`, or `luminosity`. Removing a source removes the filters
+that name it.
 
 The studio writes the file a second after the last change, so a burst such as a
 slider drag is one write, and once more on quit. Edits that never pause for a

quest/m0/README.md

@@ -94,7 +94,7 @@ repository.
 - [Instant replay and DVR](/quest/m0/replay.md) - replays and clips from held groups, and viewer scrubbing through the archive
 - [HTML/CSS overlays](/quest/m0/overlays.md) - script-free HTML and CSS graphics through Blitz natively and the DOM in the browser
 - [Animation and transitions](/quest/m0/animation.md) - move, slide, wipe, and stinger transitions, and keyframed item properties
-- [Video effects](/quest/m0/effects.md) - a curated effect set with CSS meanings, and ML effects baked into their source
+- [ML video effects](/quest/m0/effects.md) - background removal and other ML effects, baked into their source's track
 - [Destinations](/quest/m0/destinations.md) - RTMP, RTMPS, and WHIP push with per-destination codecs and shared encoders, on a UI-free egress library a hosted service can reuse
 - [Multiple canvases](/quest/m0/multi-canvas.md) - landscape and portrait canvases composite at once from shared sources
 - [Guests](/quest/m0/guests.md) - remote guests and phone cameras join through a moq-room link or QR code and appear as sources

quest/m0/compose/layout.md

@@ -25,10 +25,12 @@ catalog (decided 2026-11-02, maintainer; `rs/moq-scene/src/publish.rs`).
 Clones and video filters travel in the layout, not as tracks: a picture
 source's track is its capture before any filter, and a clone publishes none,
 so the layout names the source a clone shows and the video filters each
-source and item applies (decided 2026-11-02, maintainer). Only a video filter
-that cannot render the same way in every viewer is baked into a track of its
-own. Audio is the other way round: an audio source's track is published after
-its audio filters ([Audio source tracks](/quest/m0/compose/sources.md)), so
+source and item applies (decided 2026-11-02, maintainer), with crops and
+blend modes. Only a video filter that cannot render the same way in every
+viewer is baked into a track of its own. A mask names a picture source's
+track, and a LUT a `lut` source, whose `.cube` table publishes no track yet,
+so a viewer needs one carrying it, as an image's file is. Audio is the other
+way round: an audio source's track is published after its audio filters ([Audio source tracks](/quest/m0/compose/sources.md)), so
 the layout carries only what is applied after that point, such as gain and
 mute, and never names an audio filter for a viewer to apply again. The
 picture source tracks are enough to start; audio source tracks join the

quest/m0/effects.md

@@ -1,24 +1,27 @@
-# [M] Video effects
+# [M] ML video effects
 
 ## Goal
 
-Sources and scene items in `moq-studio` take a curated set of video effects:
-color correction, `.cube` LUTs, blur, crop and masks, chroma and luma key,
-and blend modes. They render the same in every composition host. ML effects
-such as background removal are offered too, applied before the source
-publishes.
+Sources in `moq-studio` offer ML video effects, starting with background
+removal (a person cut out of a camera), applied to the source's picture before
+it is drawn or published, so every viewer sees the same cut-out from the
+source's own track.
 
 ## Plan
 
-Write the effects in WGSL to the meanings the CSS Filter Effects and
-Compositing specs define (color matrices, blur, drop-shadow, blend modes), so
-an effect means the same thing as the CSS overlays use. The layout track
-carries effect names and parameters, never shader code, since a shader from
-the network costs every viewer's GPU and can lose the device. An effect the
-browser cannot reproduce, like an ML model, is baked into its source's track
-per the [composition epic](/quest/m0/compose/README.md), so ML effects
-apply to a source, never to one scene item. Prefer an
-existing model crate for ML effects over training one.
+The rest of the effect set landed as moq-scene `Filter`s that every
+composition host renders with the same WGSL and the layout carries by name and
+settings. A model cannot meet that parity rule: viewers would each need the
+model and a GPU to run it, and inference differs across backends. So an ML
+effect is baked into its source's track per the
+[composition epic](/quest/m0/compose/README.md): it belongs to a source, never
+to one scene item, and sits before the source's crop and filters, as the
+encoder in `rs/moq-scene/src/publish.rs` sees the capture.
 
-Effects extend moq-scene's `Filter`, which a source (`[video.<name>]`) or an
-item lists, so they undo, copy, and paste as opacity, the first filter, does.
+This quest picks the runtime crate (`ort`, `tract`, or `burn`) and the model
+(a small, permissively licensed person segmentation model such as MediaPipe's
+selfie segmentation, Apache-2.0) after measuring each candidate's per-frame
+cost at 1080p and 30 fps (decided 2026-11-03, maintainer, on
+[moq-dev/studio#17](https://github.com/moq-dev/studio/pull/17)). Running on the
+studio's own wgpu device keeps the frame on the GPU and works toward the
+[web studio](/quest/m0/web.md); a CPU runtime costs a readback per frame.

rs/moq-scene/src/clip.rs

@@ -48,33 +48,39 @@ pub struct Filters {
 }
 
 impl Clip {
-	/// A copy of `items` of `collection`, with what they show. Relative image
-	/// paths are resolved against `dir`, the absolute directory of the
-	/// collection's scene file, so they name the same file wherever the clip is
-	/// pasted.
+	/// A copy of `items` of `collection`, with what they show and what their
+	/// filters read. Relative image and LUT paths are resolved against `dir`,
+	/// the absolute directory of the collection's scene file, so they name the
+	/// same file wherever the clip is pasted.
 	pub fn items(collection: &Collection, items: Vec<Item>, dir: &Path) -> Self {
 		let mut names: Vec<String> = items.iter().map(|item| item.source.clone()).collect();
-		names.extend(items.iter().map(|item| collection.capture(&item.source).to_string()));
+		names.extend(items.iter().flat_map(|item| &item.filters).filter_map(read));
 
 		let mut clip = Items {
 			items,
 			sources: BTreeMap::new(),
 			audio: BTreeMap::new(),
 			video: BTreeMap::new(),
 		};
-		for name in names {
+		// Each source brings what it clones and what its own filters read.
+		while let Some(name) = names.pop() {
 			let Some(source) = collection.sources.get(&name) else {
 				continue;
 			};
-			let source = match source {
-				Source::Image { path } => Source::Image { path: dir.join(path) },
-				source => source.clone(),
-			};
+			if clip.sources.contains_key(&name) {
+				continue;
+			}
+			let mut source = source.clone();
+			if let Some(path) = source.path_mut() {
+				*path = dir.join(&path);
+			}
 			clip.sources.insert(name.clone(), source);
+			names.push(collection.capture(&name).to_string());
 			if let Some(audio) = collection.audio.get(&name) {
 				clip.audio.insert(name.clone(), *audio);
 			}
 			if let Some(video) = collection.video.get(&name) {
+				names.extend(video.filters.iter().filter_map(read));
 				clip.video.insert(name, video.clone());
 			}
 		}
@@ -109,6 +115,11 @@ impl Clip {
 	}
 }
 
+/// The source `filter` reads, if any.
+fn read(filter: &Filter) -> Option<String> {
+	filter.source().map(str::to_string)
+}
+
 impl Collection {
 	/// `base`, or `base 2`, `base 3`, and so on: the first no source is named.
 	pub fn unused_name(&self, base: &str) -> String {
@@ -126,58 +137,89 @@ impl Draft {
 	/// A source the collection has under the same name and definition is the
 	/// one shown, so items pasted within a collection are linked to the ones
 	/// copied. A source whose name is taken by a different one is added under
-	/// a new name, and an added source brings its audio and video settings. An
-	/// image under `dir` is named relative to it, as the scene file would.
+	/// a new name, which clones and filters naming it follow, and an added
+	/// source brings its audio and video settings. An image or LUT under `dir`
+	/// is named relative to it, as the scene file would.
 	pub fn paste(&mut self, layer: &Layer, items: &Items, dir: &Path) -> Result<(), Error> {
 		let mut names = BTreeMap::new();
+		let mut added = Vec::new();
 		// What clones clone first, so each clone names what it was renamed to.
 		let (clones, origins): (Vec<_>, Vec<_>) = items
 			.sources
 			.iter()
 			.partition(|(_, source)| matches!(source, Source::Clone { .. }));
 		for (name, source) in origins.into_iter().chain(clones) {
-			let source = match source {
+			let mut source = match source {
 				Source::Clone { source } => Source::Clone {
 					source: names.get(source).cloned().unwrap_or_else(|| source.clone()),
 				},
 				source => source.clone(),
 			};
-			// An image is the same source if it names the same file, however
+			// A file is the same source if it names the same file, however
 			// either spells the path.
-			let resolve = |source: &Source| match source {
-				Source::Image { path } => Source::Image { path: dir.join(path) },
-				source => source.clone(),
+			let resolve = |source: &Source| {
+				let mut source = source.clone();
+				if let Some(path) = source.path_mut() {
+					*path = dir.join(&path);
+				}
+				source
 			};
 			let existing = self.collection().sources.get(name).map(resolve);
 			if existing.as_ref() == Some(&resolve(&source)) {
 				names.insert(name.clone(), name.clone());
 				continue;
 			}
-			let source = match source {
-				Source::Image { path } => Source::Image {
-					path: path.strip_prefix(dir).unwrap_or(&path).to_path_buf(),
-				},
-				source => source,
-			};
+			if let Some(path) = source.path_mut() {
+				*path = path.strip_prefix(dir).unwrap_or(path).to_path_buf();
+			}
 			let renamed = self.collection().unused_name(name);
 			self.set_source(&renamed, Some(source))?;
+			names.insert(name.clone(), renamed.clone());
+			added.push((name, renamed));
+		}
+
+		// Filters name sources by their pasted names, once every one is known.
+		let rename = |filters: &[Filter]| -> Vec<Filter> {
+			let mut filters = filters.to_vec();
+			for name in filters.iter_mut().filter_map(Filter::source_mut) {
+				if let Some(renamed) = names.get(name) {
+					*name = renamed.clone();
+				}
+			}
+			filters
+		};
+		for (name, renamed) in added {
 			if let Some(audio) = items.audio.get(name) {
 				self.set_audio(&renamed, Some(*audio))?;
 			}
 			if let Some(video) = items.video.get(name) {
-				self.set_video(&renamed, Some(video.clone()))?;
+				let filters = rename(&video.filters);
+				self.set_video(
+					&renamed,
+					Some(Video {
+						filters,
+						..video.clone()
+					}),
+				)?;
 			}
-			names.insert(name.clone(), renamed);
 		}
-
 		for item in &items.items {
 			let source = names.get(&item.source).cloned().unwrap_or_else(|| item.source.clone());
 			let index = self
 				.collection()
 				.items(layer)
 				.ok_or_else(|| Error::Conflict(format!("there is no {layer}")))?
 				.len();
-			self.insert_item(layer, index, Item { source, ..item.clone() })?;
+			let filters = rename(&item.filters);
+			self.insert_item(
+				layer,
+				index,
+				Item {
+					source,
+					filters,
+					..item.clone()
+				},
+			)?;
 		}
 		Ok(())
 	}
@@ -186,7 +228,7 @@ impl Draft {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use crate::collection::{Rect, Rgba};
+	use crate::collection::{MaskMode, Rect, Rgba};
 
 	fn color(rgba: [u8; 4]) -> Source {
 		Source::Color { color: Rgba(rgba) }
@@ -203,6 +245,7 @@ mod tests {
 			"cam faded".into(),
 			Video {
 				filters: vec![Filter::Opacity { amount: 0.5 }],
+				..Default::default()
 			},
 		);
 		collection.scenes[0]
@@ -303,6 +346,55 @@ mod tests {
 		);
 	}
 
+	/// Copied items bring the sources their filters read, and a renamed one is
+	/// renamed in the filters too.
+	#[test]
+	fn filters_bring_what_they_read() {
+		let mut collection = collection();
+		collection.sources.insert(
+			"matte".into(),
+			Source::Image {
+				path: "matte.png".into(),
+			},
+		);
+		collection.sources.insert(
+			"grade".into(),
+			Source::Lut {
+				path: "grade.cube".into(),
+			},
+		);
+		let mask = |source: &str| Filter::Mask {
+			source: source.into(),
+			mode: MaskMode::Luminance,
+		};
+		collection.scenes[0].items[0].filters = vec![mask("matte")];
+		collection
+			.video
+			.get_mut("cam faded")
+			.unwrap()
+			.filters
+			.push(Filter::Lut {
+				source: "grade".into(),
+				amount: 0.5,
+			});
+		collection.validate().unwrap();
+		let text = Clip::items(&collection, collection.scenes[0].items.clone(), Path::new(DIR))
+			.to_toml()
+			.unwrap();
+
+		let mut other = Collection::new();
+		other.sources.insert("matte".into(), color([0, 255, 0, 255]));
+		let pasted = paste(&other, &Layer::Top, &text, "/shows/two");
+		assert_eq!(pasted.top[0].filters, [mask("matte 2")]);
+		assert_eq!(
+			pasted.sources["grade"],
+			Source::Lut {
+				path: "/shows/one/grade.cube".into()
+			}
+		);
+		assert_eq!(pasted.video["cam faded"], collection.video["cam faded"]);
+	}
+
 	#[test]
 	fn filters_round_trip() {
 		let clip = Clip::Filters(Filters {