From 52bd2e35276622716807f592491b0174cd5f4760 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 19:58:23 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m0/effects.md Quest-Claim: 16783c47a01f186a From e0153604fbb92afbb2a8e7b483e40d36eb25dc1d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:09:43 -0800 Subject: [PATCH 2/5] feat(scene): video effects: CSS color filters, blur, drop shadow, keys, LUTs, masks, crop, blend modes Co-Authored-By: Claude Opus 5.5 --- rs/moq-scene/src/clip.rs | 137 +++- rs/moq-scene/src/collection.rs | 521 +++++++++++++- rs/moq-scene/src/compositor.rs | 879 +++++++++++++--------- rs/moq-scene/src/edit.rs | 73 +- rs/moq-scene/src/effect.rs | 1241 ++++++++++++++++++++++++++++++++ rs/moq-scene/src/effect.wgsl | 343 +++++++++ rs/moq-scene/src/lib.rs | 10 +- rs/moq-scene/src/lut.rs | 154 ++++ rs/moq-scene/src/source.rs | 19 +- 9 files changed, 2955 insertions(+), 422 deletions(-) create mode 100644 rs/moq-scene/src/effect.rs create mode 100644 rs/moq-scene/src/effect.wgsl create mode 100644 rs/moq-scene/src/lut.rs diff --git a/rs/moq-scene/src/clip.rs b/rs/moq-scene/src/clip.rs index 9b16d90..6e5933c 100644 --- a/rs/moq-scene/src/clip.rs +++ b/rs/moq-scene/src/clip.rs @@ -48,13 +48,13 @@ pub struct Filters { } impl Clip { - /// A copy of `items` of `collection`, with what they show. Relative image - /// paths are resolved against `dir`, the absolute directory of the - /// collection's scene file, so they name the same file wherever the clip is - /// pasted. + /// A copy of `items` of `collection`, with what they show and what their + /// filters read. Relative image and LUT paths are resolved against `dir`, + /// the absolute directory of the collection's scene file, so they name the + /// same file wherever the clip is pasted. pub fn items(collection: &Collection, items: Vec, dir: &Path) -> Self { let mut names: Vec = items.iter().map(|item| item.source.clone()).collect(); - names.extend(items.iter().map(|item| collection.capture(&item.source).to_string())); + names.extend(items.iter().flat_map(|item| &item.filters).filter_map(read)); let mut clip = Items { items, @@ -62,19 +62,25 @@ impl Clip { audio: BTreeMap::new(), video: BTreeMap::new(), }; - for name in names { + // Each source brings what it clones and what its own filters read. + while let Some(name) = names.pop() { let Some(source) = collection.sources.get(&name) else { continue; }; - let source = match source { - Source::Image { path } => Source::Image { path: dir.join(path) }, - source => source.clone(), - }; + if clip.sources.contains_key(&name) { + continue; + } + let mut source = source.clone(); + if let Some(path) = source.path_mut() { + *path = dir.join(&path); + } clip.sources.insert(name.clone(), source); + names.push(collection.capture(&name).to_string()); if let Some(audio) = collection.audio.get(&name) { clip.audio.insert(name.clone(), *audio); } if let Some(video) = collection.video.get(&name) { + names.extend(video.filters.iter().filter_map(read)); clip.video.insert(name, video.clone()); } } @@ -109,6 +115,11 @@ impl Clip { } } +/// The source `filter` reads, if any. +fn read(filter: &Filter) -> Option { + filter.source().map(str::to_string) +} + impl Collection { /// `base`, or `base 2`, `base 3`, and so on: the first no source is named. pub fn unused_name(&self, base: &str) -> String { @@ -126,50 +137,66 @@ impl Draft { /// A source the collection has under the same name and definition is the /// one shown, so items pasted within a collection are linked to the ones /// copied. A source whose name is taken by a different one is added under - /// a new name, and an added source brings its audio and video settings. An - /// image under `dir` is named relative to it, as the scene file would. + /// a new name, which clones and filters naming it follow, and an added + /// source brings its audio and video settings. An image or LUT under `dir` + /// is named relative to it, as the scene file would. pub fn paste(&mut self, layer: &Layer, items: &Items, dir: &Path) -> Result<(), Error> { let mut names = BTreeMap::new(); + let mut added = Vec::new(); // What clones clone first, so each clone names what it was renamed to. let (clones, origins): (Vec<_>, Vec<_>) = items .sources .iter() .partition(|(_, source)| matches!(source, Source::Clone { .. })); for (name, source) in origins.into_iter().chain(clones) { - let source = match source { + let mut source = match source { Source::Clone { source } => Source::Clone { source: names.get(source).cloned().unwrap_or_else(|| source.clone()), }, source => source.clone(), }; - // An image is the same source if it names the same file, however + // A file is the same source if it names the same file, however // either spells the path. - let resolve = |source: &Source| match source { - Source::Image { path } => Source::Image { path: dir.join(path) }, - source => source.clone(), + let resolve = |source: &Source| { + let mut source = source.clone(); + if let Some(path) = source.path_mut() { + *path = dir.join(&path); + } + source }; let existing = self.collection().sources.get(name).map(resolve); if existing.as_ref() == Some(&resolve(&source)) { names.insert(name.clone(), name.clone()); continue; } - let source = match source { - Source::Image { path } => Source::Image { - path: path.strip_prefix(dir).unwrap_or(&path).to_path_buf(), - }, - source => source, - }; + if let Some(path) = source.path_mut() { + *path = path.strip_prefix(dir).unwrap_or(path).to_path_buf(); + } let renamed = self.collection().unused_name(name); self.set_source(&renamed, Some(source))?; + names.insert(name.clone(), renamed.clone()); + added.push((name, renamed)); + } + + // Filters name sources by their pasted names, once every one is known. + let rename = |filters: &[Filter]| -> Vec { + let mut filters = filters.to_vec(); + for name in filters.iter_mut().filter_map(Filter::source_mut) { + if let Some(renamed) = names.get(name) { + *name = renamed.clone(); + } + } + filters + }; + for (name, renamed) in added { if let Some(audio) = items.audio.get(name) { self.set_audio(&renamed, Some(*audio))?; } if let Some(video) = items.video.get(name) { - self.set_video(&renamed, Some(video.clone()))?; + let filters = rename(&video.filters); + self.set_video(&renamed, Some(Video { filters, ..video.clone() }))?; } - names.insert(name.clone(), renamed); } - for item in &items.items { let source = names.get(&item.source).cloned().unwrap_or_else(|| item.source.clone()); let index = self @@ -177,7 +204,16 @@ impl Draft { .items(layer) .ok_or_else(|| Error::Conflict(format!("there is no {layer}")))? .len(); - self.insert_item(layer, index, Item { source, ..item.clone() })?; + let filters = rename(&item.filters); + self.insert_item( + layer, + index, + Item { + source, + filters, + ..item.clone() + }, + )?; } Ok(()) } @@ -186,7 +222,7 @@ impl Draft { #[cfg(test)] mod tests { use super::*; - use crate::collection::{Rect, Rgba}; + use crate::collection::{MaskMode, Rect, Rgba}; fn color(rgba: [u8; 4]) -> Source { Source::Color { color: Rgba(rgba) } @@ -203,6 +239,7 @@ mod tests { "cam faded".into(), Video { filters: vec![Filter::Opacity { amount: 0.5 }], + ..Default::default() }, ); collection.scenes[0] @@ -303,6 +340,50 @@ mod tests { ); } + /// Copied items bring the sources their filters read, and a renamed one is + /// renamed in the filters too. + #[test] + fn filters_bring_what_they_read() { + let mut collection = collection(); + collection.sources.insert( + "matte".into(), + Source::Image { + path: "matte.png".into(), + }, + ); + collection.sources.insert( + "grade".into(), + Source::Lut { + path: "grade.cube".into(), + }, + ); + let mask = |source: &str| Filter::Mask { + source: source.into(), + mode: MaskMode::Luminance, + }; + collection.scenes[0].items[0].filters = vec![mask("matte")]; + collection.video.get_mut("cam faded").unwrap().filters.push(Filter::Lut { + source: "grade".into(), + amount: 0.5, + }); + collection.validate().unwrap(); + let text = Clip::items(&collection, collection.scenes[0].items.clone(), Path::new(DIR)) + .to_toml() + .unwrap(); + + let mut other = Collection::new(); + other.sources.insert("matte".into(), color([0, 255, 0, 255])); + let pasted = paste(&other, &Layer::Top, &text, "/shows/two"); + assert_eq!(pasted.top[0].filters, [mask("matte 2")]); + assert_eq!( + pasted.sources["grade"], + Source::Lut { + path: "/shows/one/grade.cube".into() + } + ); + assert_eq!(pasted.video["cam faded"], collection.video["cam faded"]); + } + #[test] fn filters_round_trip() { let clip = Clip::Filters(Filters { diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs index 4ab3e0a..9feb456 100644 --- a/rs/moq-scene/src/collection.rs +++ b/rs/moq-scene/src/collection.rs @@ -132,7 +132,8 @@ impl Collection { /// Check what the types cannot: the canvas can be encoded, names are unique, /// every item, clone, audio, and video entry names a source that exists, - /// and every filter's settings are in range. + /// and every crop and filter's settings are in range, each filter naming a + /// source of the kind it reads. pub fn validate(&self) -> Result<(), Error> { self.canvas.validate()?; @@ -154,7 +155,9 @@ impl Collection { ))); } item.rect.validate(&layer, &item.source)?; - validate_filters(&item.filters, &format!("`{}` in {layer}", item.source))?; + let on = format!("`{}` in {layer}", item.source); + item.crop.validate().map_err(|err| Error::Invalid(format!("{on}: {err}")))?; + self.validate_chain(&item.filters, &on)?; } } @@ -184,7 +187,9 @@ impl Collection { "video settings for `{name}`, which is not a source with a picture" ))); } - validate_filters(&video.filters, &format!("`{name}`"))?; + let on = format!("`{name}`"); + video.crop.validate().map_err(|err| Error::Invalid(format!("{on}: {err}")))?; + self.validate_chain(&video.filters, &on)?; } for (name, audio) in &self.audio { @@ -200,6 +205,34 @@ impl Collection { Ok(()) } + + /// Check `filters` are in range and name sources of the kind they read. + fn validate_chain(&self, filters: &[Filter], on: &str) -> Result<(), Error> { + validate_filters(filters, on)?; + for filter in filters { + let (name, ok, wanted) = match filter { + Filter::Lut { source, .. } => ( + source, + matches!(self.sources.get(source), Some(Source::Lut { .. })), + "a LUT", + ), + Filter::Mask { source, .. } => ( + source, + self.sources + .get(source) + .is_some_and(|source| source.has_video() && source.opens()), + "a source with a picture of its own", + ), + _ => continue, + }; + if !ok { + return Err(Error::Invalid(format!( + "{on}: a filter names `{name}`, which is not {wanted}" + ))); + } + } + Ok(()) + } } /// Parse a TOML file of this build's [`Collection::VERSION`], without its @@ -345,19 +378,27 @@ pub struct Item { /// Whether the item is drawn. A hidden item keeps its place. #[serde(default = "visible")] pub visible: bool, + /// Trimmed from what its source's crop leaves, before every filter. + #[serde(default, skip_serializing_if = "Crop::is_none")] + pub crop: Crop, /// This item's own filters, applied after its source's. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub filters: Vec, + /// How the filtered picture mixes with what is under it. + #[serde(default, skip_serializing_if = "Blend::is_normal")] + pub blend: Blend, } impl Item { - /// `source` shown, unfiltered, at `rect`. + /// `source` shown, uncropped and unfiltered, at `rect`. pub fn new(source: impl Into, rect: Rect) -> Self { Self { source: source.into(), rect, visible: true, + crop: Crop::default(), filters: Vec::new(), + blend: Blend::Normal, } } } @@ -368,9 +409,13 @@ fn visible() -> bool { /// What a source or item is drawn through, in order. /// -/// Each filter means what the CSS Filter Effects function of the same name -/// does, so every host that composes a scene draws it the same way. -#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +/// Every filter is a pure function of the picture, so every host that composes +/// a scene with moq-scene draws it the same way, and it travels in a layout as +/// its name and settings. The CSS ones mean what the CSS Filter Effects +/// function of the same name does, in the gamma-encoded sRGB the canvas holds, +/// with lengths in canvas pixels. A filter naming another source uses that +/// source's picture as it arrives, without its filters. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum Filter { /// CSS `opacity()`: 0 is transparent and 1 leaves the picture as it is. @@ -378,19 +423,291 @@ pub enum Filter { /// From 0 to 1. amount: f32, }, + /// CSS `brightness()`: each color channel times `amount`. + Brightness { + /// 0 or more: 0 is black and 1 leaves the picture as it is. + amount: f32, + }, + /// CSS `contrast()`: each color channel pushed away from middle gray. + Contrast { + /// 0 or more: 0 is gray and 1 leaves the picture as it is. + amount: f32, + }, + /// CSS `saturate()`. + Saturate { + /// 0 or more: 0 is gray, 1 leaves the picture as it is, and more + /// oversaturates. + amount: f32, + }, + /// CSS `grayscale()`. + Grayscale { + /// From 0 to 1, where 1 is fully gray. + amount: f32, + }, + /// CSS `sepia()`. + Sepia { + /// From 0 to 1, where 1 is fully sepia. + amount: f32, + }, + /// CSS `invert()`. + Invert { + /// From 0 to 1, where 1 is a negative. + amount: f32, + }, + /// CSS `hue-rotate()`. + HueRotate { + /// The angle around the color wheel. + degrees: f32, + }, + /// CSS `blur()`: a Gaussian blur whose edges fade into transparency. + Blur { + /// The standard deviation, in canvas pixels; 0 or more. + radius: f32, + }, + /// CSS `drop-shadow()`: the picture's shape, offset, blurred, and filled + /// with `color`, drawn under it. + DropShadow { + /// Offset right, in canvas pixels. + x: f32, + /// Offset down, in canvas pixels. + y: f32, + /// The blur's standard deviation, in canvas pixels; 0 or more. + blur: f32, + /// The shadow's color. + color: Rgba, + }, + /// Make what is close to `color` transparent, as a green screen is keyed. + /// + /// Closeness is the distance between the Rec. 709 chroma (Cb, Cr) of a + /// pixel and of `color`, so a shadow on the screen keys like its lit part. + /// Within `similarity` a pixel is transparent, and over the next + /// `smoothness` it fades in. Within `similarity + spill`, the key color + /// bleeding onto the subject is desaturated away. + ChromaKey { + /// The screen's color. + color: Rgba, + /// From 0 to 1. + similarity: f32, + /// From 0 to 1. + smoothness: f32, + /// From 0 to 1. + spill: f32, + }, + /// Make what is too dark or too bright transparent, by Rec. 709 luma. + /// + /// Luma from `min` to `max` stays opaque, and over `smoothness` past + /// either end fades out. + LumaKey { + /// From 0 to 1. + min: f32, + /// From `min` to 1. + max: f32, + /// From 0 to 1. + smoothness: f32, + }, + /// Look each color up in a [`Source::Lut`]'s table, as a color grade does. + Lut { + /// The [`Source::Lut`]. + source: String, + /// From 0 (the picture as it is) to 1 (fully graded). + amount: f32, + }, + /// Keep the picture only where another source's picture, stretched over + /// the item, is opaque, as CSS `mask-image` does. + Mask { + /// A source with a picture of its own: not a clone. + source: String, + /// What of the mask's picture counts. + #[serde(default)] + mode: MaskMode, + }, +} + +/// What of a mask's picture keeps the picture under it, as CSS `mask-mode`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MaskMode { + /// Its opacity. + #[default] + Alpha, + /// Its Rec. 709 luma times its opacity, so white keeps and black hides. + Luminance, } impl Filter { - fn validate(&self) -> Result<(), String> { + /// The source whose picture or table this filter reads, if any. + pub fn source(&self) -> Option<&str> { + match self { + Self::Lut { source, .. } | Self::Mask { source, .. } => Some(source), + _ => None, + } + } + + /// The source this filter reads, to rename it. + pub(crate) fn source_mut(&mut self) -> Option<&mut String> { match self { - Self::Opacity { amount } if !(0.0..=1.0).contains(amount) => { - Err(format!("opacity {amount} is not between 0 and 1")) + Self::Lut { source, .. } | Self::Mask { source, .. } => Some(source), + _ => None, + } + } + + fn validate(&self) -> Result<(), String> { + fn check(ok: bool, err: impl FnOnce() -> String) -> Result<(), String> { + if ok { Ok(()) } else { Err(err()) } + } + let unit = |name: &str, value: f32| { + check((0.0..=1.0).contains(&value), || { + format!("{name} {value} is not between 0 and 1") + }) + }; + let positive = |name: &str, value: f32| { + check(value.is_finite() && value >= 0.0, || { + format!("{name} {value} is not 0 or more") + }) + }; + let finite = |name: &str, value: f32| check(value.is_finite(), || format!("{name} {value} is not a number")); + + match *self { + Self::Opacity { amount } => unit("opacity", amount), + Self::Brightness { amount } => positive("brightness", amount), + Self::Contrast { amount } => positive("contrast", amount), + Self::Saturate { amount } => positive("saturation", amount), + Self::Grayscale { amount } => unit("grayscale", amount), + Self::Sepia { amount } => unit("sepia", amount), + Self::Invert { amount } => unit("invert", amount), + Self::HueRotate { degrees } => finite("hue rotation", degrees), + Self::Blur { radius } => positive("blur", radius), + Self::DropShadow { x, y, blur, .. } => { + finite("shadow x", x)?; + finite("shadow y", y)?; + positive("shadow blur", blur) + } + Self::ChromaKey { + similarity, + smoothness, + spill, + .. + } => { + unit("similarity", similarity)?; + unit("smoothness", smoothness)?; + unit("spill", spill) } - Self::Opacity { .. } => Ok(()), + Self::LumaKey { min, max, smoothness } => { + unit("luma min", min)?; + unit("luma max", max)?; + unit("smoothness", smoothness)?; + check(min <= max, || format!("luma min {min} is over max {max}")) + } + Self::Lut { amount, .. } => unit("LUT amount", amount), + Self::Mask { .. } => Ok(()), } } } +/// How an item's picture mixes with what is under it, as CSS +/// `mix-blend-mode`, by the formulas of the Compositing and Blending spec. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[expect(missing_docs, reason = "each is the CSS blend mode of the same name")] +pub enum Blend { + /// Drawn over, as without a blend mode. + #[default] + Normal, + Multiply, + Screen, + Overlay, + Darken, + Lighten, + ColorDodge, + ColorBurn, + HardLight, + SoftLight, + Difference, + Exclusion, + Hue, + Saturation, + Color, + Luminosity, +} + +impl Blend { + /// Every mode, in the CSS spec's order. + pub const ALL: [Self; 16] = [ + Self::Normal, + Self::Multiply, + Self::Screen, + Self::Overlay, + Self::Darken, + Self::Lighten, + Self::ColorDodge, + Self::ColorBurn, + Self::HardLight, + Self::SoftLight, + Self::Difference, + Self::Exclusion, + Self::Hue, + Self::Saturation, + Self::Color, + Self::Luminosity, + ]; + + fn is_normal(&self) -> bool { + *self == Self::Normal + } +} + +/// The fraction of a picture trimmed from each edge, the rest stretched to +/// fill the item, as CSS `object-view-box: inset()` does. +/// +/// Fractions rather than pixels, so a crop keeps its framing when the +/// picture's resolution changes, such as a viewer picking another rendition. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Crop { + /// From the left edge. + #[serde(default)] + pub left: f32, + /// From the top edge. + #[serde(default)] + pub top: f32, + /// From the right edge. + #[serde(default)] + pub right: f32, + /// From the bottom edge. + #[serde(default)] + pub bottom: f32, +} + +impl Crop { + /// Whether the crop trims nothing. + pub fn is_none(&self) -> bool { + *self == Self::default() + } + + /// `inner` applied to the picture this crop leaves, as one crop. + pub fn then(&self, inner: &Self) -> Self { + let width = 1.0 - self.left - self.right; + let height = 1.0 - self.top - self.bottom; + Self { + left: self.left + inner.left * width, + top: self.top + inner.top * height, + right: self.right + inner.right * width, + bottom: self.bottom + inner.bottom * height, + } + } + + fn validate(&self) -> Result<(), String> { + let edges = [self.left, self.top, self.right, self.bottom]; + if edges.iter().any(|edge| !(0.0..1.0).contains(edge)) + || self.left + self.right >= 1.0 + || self.top + self.bottom >= 1.0 + { + return Err(format!("{self:?} leaves nothing of the picture")); + } + Ok(()) + } +} + pub(crate) fn validate_filters(filters: &[Filter], on: &str) -> Result<(), Error> { for filter in filters { filter @@ -404,6 +721,9 @@ pub(crate) fn validate_filters(filters: &[Filter], on: &str) -> Result<(), Error #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Video { + /// Trimmed before every filter and each item's own crop. + #[serde(default, skip_serializing_if = "Crop::is_none")] + pub crop: Crop, /// Applied in order, before each item's own. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub filters: Vec, @@ -498,12 +818,26 @@ pub enum Source { /// The source cloned: one with a picture, and not itself a clone. source: String, }, + /// A 3D color lookup table for [`Filter::Lut`], from an Adobe or Resolve + /// `.cube` file. It has no picture or sound of its own. + Lut { + /// The file, relative to the scene file's directory unless absolute. + path: PathBuf, + }, } impl Source { /// Whether the source has a picture to place in a scene. pub fn has_video(&self) -> bool { - !matches!(self, Self::Microphone { .. } | Self::SystemAudio) + !matches!(self, Self::Microphone { .. } | Self::SystemAudio | Self::Lut { .. }) + } + + /// The file the source reads, to resolve or relativize it. + pub(crate) fn path_mut(&mut self) -> Option<&mut PathBuf> { + match self { + Self::Image { path } | Self::Lut { path } => Some(path), + _ => None, + } } /// Whether the source opens a capture, file, or subscription of its own, @@ -616,28 +950,28 @@ mod tests { "camera faded".into(), Video { filters: vec![Filter::Opacity { amount: 0.5 }], + ..Default::default() }, ); collection.top.push(Item::new("logo", Rect::fill(&collection.canvas))); let canvas = collection.canvas; let scene = &mut collection.scenes[0]; + scene.items.push(Item::new("background", Rect::fill(&canvas))); + let rect = Rect { + x: 100.0, + y: 50.0, + width: 640.0, + height: 360.0, + }; scene.items.push(Item { - source: "background".into(), - rect: Rect::fill(&canvas), - visible: true, - filters: Vec::new(), - }); - scene.items.push(Item { - source: "camera".into(), - rect: Rect { - x: 100.0, - y: 50.0, - width: 640.0, - height: 360.0, - }, visible: false, - filters: Vec::new(), + crop: Crop { + left: 0.25, + ..Crop::default() + }, + blend: Blend::Screen, + ..Item::new("camera", rect) }); collection } @@ -685,12 +1019,22 @@ source = "cam" [video.cam-faded] filters = [{ kind = "opacity", amount = 0.5 }] +[sources.grade] +kind = "lut" +path = "grade.cube" + [[scenes]] name = "Main" items = [ { source = "bg", rect = { x = 0, y = 0, width = 1280, height = 720 } }, { source = "cam", rect = { x = 10, y = 10, width = 320, height = 180 }, visible = false }, { source = "cam-faded", rect = { x = 0, y = 0, width = 640, height = 360 }, filters = [{ kind = "opacity", amount = 0.8 }] }, + { source = "cam", rect = { x = 0, y = 0, width = 640, height = 360 }, crop = { left = 0.1, right = 0.1 }, blend = "soft_light", filters = [ + { kind = "chroma_key", color = "#00ff00", similarity = 0.4, smoothness = 0.08, spill = 0.1 }, + { kind = "lut", source = "grade", amount = 0.7 }, + { kind = "mask", source = "bg", mode = "luminance" }, + { kind = "drop_shadow", x = 4, y = 4, blur = 8, color = "#00000080" }, + ] }, ] [[top]] @@ -724,6 +1068,16 @@ rect = { x = 0, y = 680, width = 1280, height = 40 } [&Filter::Opacity { amount: 0.5 }, &Filter::Opacity { amount: 0.8 }], "the source's filters, then the item's" ); + let keyed = &main.items[3]; + assert_eq!(keyed.blend, Blend::SoftLight); + assert_eq!(keyed.crop.right, 0.1); + assert_eq!( + keyed.filters[2], + Filter::Mask { + source: "bg".into(), + mode: MaskMode::Luminance + } + ); assert_eq!(collection.top[0].source, "bg"); } @@ -807,13 +1161,122 @@ rect = { x = 0, y = 680, width = 1280, height = 40 } #[test] fn refuses_a_filter_out_of_range() { - for amount in [-0.1, 1.5, f32::NAN] { + let red = Rgba([255, 0, 0, 255]); + let bad = [ + Filter::Opacity { amount: 1.5 }, + Filter::Opacity { amount: f32::NAN }, + Filter::Brightness { amount: -0.1 }, + Filter::Saturate { amount: f32::INFINITY }, + Filter::Invert { amount: 2.0 }, + Filter::HueRotate { degrees: f32::NAN }, + Filter::Blur { radius: -1.0 }, + Filter::DropShadow { + x: f32::INFINITY, + y: 0.0, + blur: 0.0, + color: red, + }, + Filter::ChromaKey { + color: red, + similarity: 1.5, + smoothness: 0.1, + spill: 0.1, + }, + Filter::LumaKey { + min: 0.8, + max: 0.2, + smoothness: 0.0, + }, + ]; + for filter in bad { + let mut collection = example(); + collection.scenes[0].items[0].filters = vec![filter.clone()]; + assert!(matches!(collection.validate(), Err(Error::Invalid(_))), "{filter:?}"); + } + } + + /// A LUT filter names a LUT, and a mask a source with a picture of its own, + /// since a mask draws its source's picture as it arrives. + #[test] + fn refuses_a_filter_reading_the_wrong_source() { + let lut = |source: &str| Filter::Lut { + source: source.into(), + amount: 1.0, + }; + let mask = |source: &str| Filter::Mask { + source: source.into(), + mode: MaskMode::Alpha, + }; + let mut collection = example(); + collection.sources.insert( + "grade".into(), + Source::Lut { + path: "grade.cube".into(), + }, + ); + collection.scenes[0].items[0].filters = vec![lut("grade"), mask("logo")]; + collection.validate().unwrap(); + + for filter in [lut("logo"), lut("nobody"), mask("grade"), mask("mic"), mask("camera faded")] { + let mut collection = collection.clone(); + collection.video.insert( + "camera".into(), + Video { + filters: vec![filter.clone()], + ..Default::default() + }, + ); + assert!(matches!(collection.validate(), Err(Error::Invalid(_))), "{filter:?}"); + } + } + + #[test] + fn refuses_a_crop_that_leaves_nothing() { + let crops = [ + Crop { + left: 0.6, + right: 0.4, + ..Crop::default() + }, + Crop { + top: -0.1, + ..Crop::default() + }, + Crop { + bottom: f32::NAN, + ..Crop::default() + }, + ]; + for crop in crops { let mut collection = example(); - collection.scenes[0].items[0].filters = vec![Filter::Opacity { amount }]; - assert!(matches!(collection.validate(), Err(Error::Invalid(_))), "{amount}"); + collection.scenes[0].items[0].crop = crop; + assert!(matches!(collection.validate(), Err(Error::Invalid(_))), "{crop:?}"); } } + /// A source's crop then an item's is one crop of the original picture. + #[test] + fn crops_compose() { + let source = Crop { + left: 0.5, + ..Crop::default() + }; + let item = Crop { + left: 0.5, + bottom: 0.25, + ..Crop::default() + }; + assert_eq!( + source.then(&item), + Crop { + left: 0.75, + bottom: 0.25, + ..Crop::default() + } + ); + assert_eq!(Crop::default().then(&item), item); + } + #[test] fn refuses_duplicate_scenes() { let mut collection = example(); diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index a147625..afef8ce 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -2,60 +2,20 @@ //! between them. use std::collections::HashMap; -use std::num::NonZeroU64; use std::time::Duration; use kio::clock::Instant; use moq_video::render::{Renderer, wgpu}; use crate::Error; -use crate::collection::{Canvas, Collection, Filter, Item, Rect, Source}; -use crate::source::{Picture, Sources}; +use crate::collection::{self, Canvas, Collection, Item, Layer as Place, Source}; +use crate::effect::{self, Effects, Frame, Lut, Picture, Prepared}; +use crate::source::{self, Sources}; /// The format of every canvas texture: gamma-encoded RGBA, what /// `moq_video::render` produces and what its exporter hands an encoder. pub const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm; -/// One draw's uniform: where on the canvas, in clip space, and how opaque. -#[repr(C)] -#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] -struct Uniform { - /// Top-left x, top-left y, bottom-right x, bottom-right y, in clip space. - rect: [f32; 4], - /// Opacity in `x`; the rest pads the struct to WGSL's 16-byte vectors. - opacity: [f32; 4], -} - -const SHADER: &str = r#" -struct Item { - rect: vec4, - opacity: vec4, -} - -@group(0) @binding(0) var item: Item; -@group(1) @binding(0) var image: texture_2d; -@group(1) @binding(1) var image_sampler: sampler; - -struct Varyings { - @builtin(position) position: vec4, - @location(0) uv: vec2, -} - -// A four-vertex triangle strip covering the item's rectangle. -@vertex fn vs_main(@builtin(vertex_index) index: u32) -> Varyings { - let corner = vec2(f32(index & 1u), f32((index >> 1u) & 1u)); - var out: Varyings; - out.position = vec4(mix(item.rect.xy, item.rect.zw, corner), 0.0, 1.0); - out.uv = corner; - return out; -} - -@fragment fn fs_main(in: Varyings) -> @location(0) vec4 { - let color = textureSample(image, image_sampler, in.uv); - return vec4(color.rgb, color.a * item.opacity.x); -} -"#; - /// Draws a [`Collection`]'s scenes onto its canvas: one texture for the scene /// in preview, one for the scene on program, and the fade from one to the other. /// @@ -66,18 +26,13 @@ struct Varyings { /// holding up the canvas. /// /// Pixels are blended in the gamma-encoded space sources arrive in, as a -/// broadcast switcher does, so a 50% fade is a 50% mix of the stored values. +/// broadcast switcher and CSS do, so a 50% fade is a 50% mix of the stored +/// values. pub struct Compositor { device: wgpu::Device, queue: wgpu::Queue, canvas: Canvas, - - pipeline: wgpu::RenderPipeline, - image_layout: wgpu::BindGroupLayout, - sampler: wgpu::Sampler, - - uniform_layout: wgpu::BindGroupLayout, - uniforms: Uniforms, + effects: Effects, preview: Target, program: Target, @@ -98,46 +53,38 @@ struct Fade { duration: Duration, } -/// A canvas-sized texture drawn into, and the bind group that samples it. +/// A canvas-sized texture drawn into, and bound to sample. struct Target { texture: wgpu::Texture, - view: wgpu::TextureView, - bind: wgpu::BindGroup, + picture: Picture, } -/// One source's newest picture on the GPU. +/// One source's newest picture or table on the GPU. struct Layer { /// What the picture came from. An edited source starts a new layer, so it /// never shows the old source's last picture. source: Source, /// Converts video frames, created on the first one. renderer: Option, - /// Samples the newest picture; `None` until the first arrives. - bind: Option, + /// The newest picture; `None` until the first arrives. + picture: Option, + /// A LUT source's table; `None` until it is read. + lut: Option, /// Why the newest picture failed to draw, logged when it first does. error: Option, } -/// A growable uniform buffer holding one [`Uniform`] per draw of a render. -struct Uniforms { - buffer: wgpu::Buffer, - bind: wgpu::BindGroup, - /// Bytes between draws, rounded up to the device's dynamic-offset alignment. - stride: u64, - capacity: u64, -} +/// The pictures and tables of the compositor's layers, for filters to read. +struct Library<'a>(&'a HashMap); -/// One textured quad: what to sample, and the uniform that places it. -#[derive(Clone)] -struct Draw { - bind: wgpu::BindGroup, - uniform: Uniform, -} +impl effect::Library for Library<'_> { + fn picture(&self, name: &str) -> Option { + self.0.get(name)?.picture.clone() + } -/// One render pass: a target cleared to black and the quads drawn into it. -struct Pass<'a> { - view: &'a wgpu::TextureView, - draws: Vec, + fn lut(&self, name: &str) -> Option { + self.0.get(name)?.lut.clone() + } } impl Compositor { @@ -153,101 +100,8 @@ impl Compositor { ))); } - let image_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("moq-scene image"), - entries: &[ - wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - sample_type: wgpu::TextureSampleType::Float { filterable: true }, - view_dimension: wgpu::TextureViewDimension::D2, - multisampled: false, - }, - count: None, - }, - wgpu::BindGroupLayoutEntry { - binding: 1, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), - count: None, - }, - ], - }); - let uniform_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("moq-scene item"), - entries: &[wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, - ty: wgpu::BindingType::Buffer { - ty: wgpu::BufferBindingType::Uniform, - has_dynamic_offset: true, - min_binding_size: NonZeroU64::new(size_of::() as u64), - }, - count: None, - }], - }); - let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("moq-scene"), - bind_group_layouts: &[Some(&uniform_layout), Some(&image_layout)], - immediate_size: 0, - }); - let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("moq-scene"), - source: wgpu::ShaderSource::Wgsl(SHADER.into()), - }); - let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("moq-scene"), - layout: Some(&pipeline_layout), - vertex: wgpu::VertexState { - module: &shader, - entry_point: Some("vs_main"), - compilation_options: Default::default(), - buffers: &[], - }, - primitive: wgpu::PrimitiveState { - topology: wgpu::PrimitiveTopology::TriangleStrip, - ..Default::default() - }, - depth_stencil: None, - multisample: Default::default(), - fragment: Some(wgpu::FragmentState { - module: &shader, - entry_point: Some("fs_main"), - compilation_options: Default::default(), - targets: &[Some(wgpu::ColorTargetState { - format: FORMAT, - // Straight alpha over an opaque canvas, which stays opaque. - blend: Some(wgpu::BlendState { - color: wgpu::BlendComponent { - src_factor: wgpu::BlendFactor::SrcAlpha, - dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, - operation: wgpu::BlendOperation::Add, - }, - alpha: wgpu::BlendComponent { - src_factor: wgpu::BlendFactor::One, - dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, - operation: wgpu::BlendOperation::Add, - }, - }), - write_mask: wgpu::ColorWrites::ALL, - })], - }), - multiview_mask: None, - cache: None, - }); - let sampler = device.create_sampler(&wgpu::SamplerDescriptor { - label: Some("moq-scene"), - mag_filter: wgpu::FilterMode::Linear, - min_filter: wgpu::FilterMode::Linear, - ..Default::default() - }); - - let alignment = u64::from(device.limits().min_uniform_buffer_offset_alignment); - let stride = (size_of::() as u64).next_multiple_of(alignment); - let uniforms = Uniforms::new(device, &uniform_layout, stride, 16); - - let target = |label| Target::new(device, &image_layout, &sampler, canvas, label); + let effects = Effects::new(device, queue, canvas); + let target = |label| Target::new(device, &effects, canvas, label); Ok(Self { preview: target("moq-scene preview"), program: target("moq-scene program"), @@ -255,11 +109,7 @@ impl Compositor { device: device.clone(), queue: queue.clone(), canvas, - pipeline, - image_layout, - sampler, - uniform_layout, - uniforms, + effects, layers: HashMap::new(), preview_scene: None, program_scene: None, @@ -345,109 +195,39 @@ impl Compositor { self.fade = None; } + let mut frame = self.effects.frame(); + let mut draw = Draw { + collection, + effects: &mut self.effects, + library: Library(&self.layers), + frame: &mut frame, + prepared: HashMap::new(), + }; + // The top layer goes over both outputs, and over a fade rather than // through it, so it stays on across the switch. - let top = self.items(collection, &collection.top); - let mut preview = self.scene(collection, self.preview_scene.as_deref()); - preview.extend(top.iter().cloned()); - let mut passes = vec![Pass { - view: &self.preview.view, - draws: preview, - }]; - let mut program = match (&self.fade, progress) { + draw.target(&self.preview); + draw.scene(&self.preview, self.preview_scene.as_deref()); + draw.layer(&self.preview, Place::Top); + + match (&self.fade, progress) { (Some(fade), Some(progress)) => { - passes.push(Pass { - view: &self.incoming.view, - draws: self.scene(collection, self.program_scene.as_deref()), - }); - let mut draws = self.scene(collection, fade.from.as_deref()); - draws.push(Draw { - bind: self.incoming.bind.clone(), - uniform: uniform(Rect::fill(&self.canvas), self.canvas, progress), - }); - draws + draw.target(&self.incoming); + draw.scene(&self.incoming, self.program_scene.as_deref()); + draw.target(&self.program); + draw.scene(&self.program, fade.from.as_deref()); + let incoming = draw.effects.fade(&self.incoming.picture, progress); + draw.effects.draw(draw.frame, &self.program.texture, &incoming); } - _ => self.scene(collection, self.program_scene.as_deref()), - }; - program.extend(top); - passes.push(Pass { - view: &self.program.view, - draws: program, - }); - - let count: usize = passes.iter().map(|pass| pass.draws.len()).sum(); - self.uniforms.reserve(&self.device, &self.uniform_layout, count as u64); - let stride = self.uniforms.stride as usize; - let mut bytes = vec![0u8; count * stride]; - for (index, draw) in passes.iter().flat_map(|pass| &pass.draws).enumerate() { - let offset = index * stride; - bytes[offset..offset + size_of::()].copy_from_slice(bytemuck::bytes_of(&draw.uniform)); - } - if !bytes.is_empty() { - self.queue.write_buffer(&self.uniforms.buffer, 0, &bytes); - } - - let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("moq-scene"), - }); - let mut index = 0u32; - for pass in &passes { - let mut render = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("moq-scene"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: pass.view, - depth_slice: None, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), - store: wgpu::StoreOp::Store, - }, - })], - depth_stencil_attachment: None, - timestamp_writes: None, - occlusion_query_set: None, - multiview_mask: None, - }); - render.set_pipeline(&self.pipeline); - for draw in &pass.draws { - render.set_bind_group(0, &self.uniforms.bind, &[index * stride as u32]); - render.set_bind_group(1, &draw.bind, &[]); - render.draw(0..4, 0..1); - index += 1; + _ => { + draw.target(&self.program); + draw.scene(&self.program, self.program_scene.as_deref()); } } - self.queue.submit([encoder.finish()]); - Ok(()) - } - - /// The quads drawing `scene`, back to front. A missing scene draws nothing. - fn scene(&self, collection: &Collection, scene: Option<&str>) -> Vec { - let items = scene - .and_then(|name| collection.scene(name)) - .map(|scene| scene.items.as_slice()); - self.items(collection, items.unwrap_or_default()) - } + draw.layer(&self.program, Place::Top); - /// The quads drawing `items`, back to front, each through its filters. A - /// hidden item or a source with no picture yet draws nothing. - fn items(&self, collection: &Collection, items: &[Item]) -> Vec { - items - .iter() - .filter(|item| item.visible) - .filter_map(|item| { - let bind = self.layers.get(collection.capture(&item.source))?.bind.clone()?; - let opacity = collection - .filters(item) - .map(|filter| match filter { - Filter::Opacity { amount } => *amount, - }) - .product(); - Some(Draw { - bind, - uniform: uniform(item.rect, self.canvas, opacity), - }) - }) - .collect() + self.effects.submit(frame); + Ok(()) } /// Pull the newest picture from every source onto the GPU, and forget the @@ -466,11 +246,12 @@ impl Compositor { let layer = self.layers.entry(name.clone()).or_insert_with(|| Layer { source: source.clone(), renderer: None, - bind: None, + picture: None, + lut: None, error: None, }); let texture = match picture { - Picture::Frame(frame) => { + source::Picture::Frame(frame) => { let renderer = match layer.renderer.as_mut() { Some(renderer) => Ok(renderer), None => Renderer::new(&self.device, &self.queue, Default::default()) @@ -480,14 +261,17 @@ impl Compositor { .and_then(|renderer| renderer.render(&frame)) .map_err(Error::from) } - Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels), + source::Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels), + source::Picture::Lut(cube) => { + layer.lut = Some(self.effects.lut(&cube)); + continue; + } }; match texture { Ok(texture) => { layer.error = None; - let view = texture.create_view(&Default::default()); - layer.bind = Some(image_bind(&self.device, &self.image_layout, &self.sampler, &view)); + layer.picture = Some(self.effects.picture(&texture)); } Err(err) => { if layer.error.is_none() { @@ -500,14 +284,63 @@ impl Compositor { } } -/// Place `rect` (canvas pixels) in clip space. -fn uniform(rect: Rect, canvas: Canvas, opacity: f32) -> Uniform { - let (width, height) = (canvas.width as f32, canvas.height as f32); - let x = |px: f32| px / width * 2.0 - 1.0; - let y = |px: f32| 1.0 - px / height * 2.0; - Uniform { - rect: [x(rect.x), y(rect.y), x(rect.x + rect.width), y(rect.y + rect.height)], - opacity: [opacity, 0.0, 0.0, 0.0], +/// One render's draws onto its targets. +struct Draw<'a> { + collection: &'a Collection, + effects: &'a mut Effects, + library: Library<'a>, + frame: &'a mut Frame, + /// Each item, filtered once though it is drawn onto several targets. + prepared: HashMap<(Place, usize), Option>, +} + +impl Draw<'_> { + /// Start drawing `target`. + fn target(&mut self, target: &Target) { + self.frame.target(&target.picture); + } + + /// Draw `scene`'s items onto `target`. A missing scene draws nothing. + fn scene(&mut self, target: &Target, scene: Option<&str>) { + if let Some(scene) = scene.filter(|name| self.collection.scene(name).is_some()) { + self.layer(target, Place::Scene(scene.to_string())); + } + } + + /// Draw `place`'s items onto `target`, back to front, each through its crop, + /// filters, and blend mode. A hidden item, a source with no picture yet, or + /// a filter whose source has nothing yet draws nothing. + fn layer(&mut self, target: &Target, place: Place) { + let items = self.collection.items(&place).map_or(&[][..], Vec::as_slice); + for (index, item) in items.iter().enumerate().filter(|(_, item)| item.visible) { + let key = (place.clone(), index); + let prepared = match self.prepared.get(&key) { + Some(prepared) => prepared.clone(), + None => { + let prepared = self.prepare(item); + self.prepared.insert(key, prepared.clone()); + prepared + } + }; + if let Some(prepared) = prepared { + self.effects.draw(self.frame, &target.texture, &prepared); + } + } + } + + fn prepare(&mut self, item: &Item) -> Option { + let collection = self.collection; + let picture = self.library.0.get(collection.capture(&item.source))?.picture.as_ref()?; + let video = collection.video.get(&item.source); + let crop = video.map_or(collection::Crop::default(), |video| video.crop); + let item = effect::Item { + picture, + rect: item.rect, + crop: crop.then(&item.crop), + filters: collection.filters(item).collect(), + blend: item.blend, + }; + self.effects.prepare(self.frame, item, &self.library) } } @@ -558,36 +391,8 @@ fn upload( Ok(texture) } -fn image_bind( - device: &wgpu::Device, - layout: &wgpu::BindGroupLayout, - sampler: &wgpu::Sampler, - view: &wgpu::TextureView, -) -> wgpu::BindGroup { - device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("moq-scene image"), - layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::Sampler(sampler), - }, - ], - }) -} - impl Target { - fn new( - device: &wgpu::Device, - layout: &wgpu::BindGroupLayout, - sampler: &wgpu::Sampler, - canvas: Canvas, - label: &str, - ) -> Self { + fn new(device: &wgpu::Device, effects: &Effects, canvas: Canvas, label: &str) -> Self { let texture = device.create_texture(&wgpu::TextureDescriptor { label: Some(label), size: wgpu::Extent3d { @@ -599,57 +404,21 @@ impl Target { sample_count: 1, dimension: wgpu::TextureDimension::D2, format: FORMAT, + // A blend mode copies what is under it out of the target. usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_SRC, view_formats: &[], }); - let view = texture.create_view(&Default::default()); - let bind = image_bind(device, layout, sampler, &view); - Self { texture, view, bind } - } -} - -impl Uniforms { - fn new(device: &wgpu::Device, layout: &wgpu::BindGroupLayout, stride: u64, capacity: u64) -> Self { - let buffer = device.create_buffer(&wgpu::BufferDescriptor { - label: Some("moq-scene items"), - size: stride * capacity, - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - mapped_at_creation: false, - }); - let bind = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("moq-scene items"), - layout, - entries: &[wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { - buffer: &buffer, - offset: 0, - size: NonZeroU64::new(size_of::() as u64), - }), - }], - }); - Self { - buffer, - bind, - stride, - capacity, - } - } - - /// Grow to hold `count` draws, doubling so a growing scene reallocates rarely. - fn reserve(&mut self, device: &wgpu::Device, layout: &wgpu::BindGroupLayout, count: u64) { - if count > self.capacity { - *self = Self::new(device, layout, self.stride, count.next_power_of_two()); - } + let picture = effects.picture(&texture); + Self { texture, picture } } } #[cfg(test)] mod tests { use super::*; - use crate::collection::{Rgba, Scene}; + use crate::collection::{Blend, Filter, MaskMode, Rect, Rgba, Scene}; use crate::source::test::{context, local, start}; const RED: [u8; 4] = [255, 0, 0, 255]; @@ -683,12 +452,7 @@ mod tests { collection .sources .insert("blue".into(), Source::Color { color: Rgba(BLUE) }); - let item = |source: &str, rect| Item { - source: source.into(), - rect, - visible: true, - filters: Vec::new(), - }; + let item = Item::new; let fill = Rect::fill(&collection.canvas); let corner = Rect { x: 4.0, @@ -723,8 +487,12 @@ mod tests { impl Studio { async fn new(collection: Collection) -> Self { + Self::with_files(collection, tempfile::tempdir().unwrap()).await + } + + /// Over sources whose files are in `dir`. + async fn with_files(collection: Collection, dir: tempfile::TempDir) -> Self { let (device, queue) = gpu().await; - let dir = tempfile::tempdir().unwrap(); let (mut context, origin) = context(dir.path()); context.output = moq_video::render::output(&device); let mut sources = Sources::new(context); @@ -897,6 +665,7 @@ mod tests { "blue faded".into(), crate::Video { filters: vec![Filter::Opacity { amount: 0.5 }], + ..Default::default() }, ); collection.scenes[0].items.push(Item { @@ -1031,4 +800,394 @@ mod tests { }) .await; } + + /// The program of `collection` with scene "red" cut to it, its files in `dir`. + async fn drawn(collection: Collection, dir: tempfile::TempDir) -> Vec { + let mut studio = Studio::with_files(collection, dir).await; + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + studio.render(Instant::now()); + studio.program() + } + + /// Scene "red" as `items` alone, over black. + fn showing(items: Vec) -> Collection { + let mut collection = collection(); + collection.scenes[0].items = items; + collection.validate().unwrap(); + collection + } + + fn rgb(rgb: [u8; 3]) -> [u8; 4] { + [rgb[0], rgb[1], rgb[2], 255] + } + + /// Each CSS color filter maps a color as the spec's matrix does, clamping + /// after each filter rather than once at the end. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn color_filters_follow_css() { + let cases = [ + (vec![Filter::Grayscale { amount: 1.0 }], [118, 118, 118]), + (vec![Filter::Sepia { amount: 1.0 }], [165, 147, 114]), + (vec![Filter::Invert { amount: 1.0 }], [55, 155, 205]), + (vec![Filter::Brightness { amount: 0.5 }], [100, 50, 25]), + (vec![Filter::Contrast { amount: 0.0 }], [128, 128, 128]), + (vec![Filter::Saturate { amount: 0.0 }], [118, 118, 118]), + (vec![Filter::HueRotate { degrees: 180.0 }], [35, 135, 185]), + (vec![Filter::Opacity { amount: 0.5 }], [100, 50, 25]), + ( + vec![Filter::Brightness { amount: 2.0 }, Filter::Brightness { amount: 0.5 }], + [128, 100, 50], + ), + ]; + local(async { + for (filters, expected) in cases { + let mut collection = collection(); + collection.sources.insert( + "tan".into(), + Source::Color { + color: Rgba([200, 100, 50, 255]), + }, + ); + collection.scenes[0].items = vec![Item { + filters: filters.clone(), + ..Item::new("tan", Rect::fill(&collection.canvas)) + }]; + let program = drawn(collection, tempfile::tempdir().unwrap()).await; + let actual = pixel(&program, 3, 3); + assert!(near(actual, rgb(expected)), "{filters:?}: {actual:?}"); + } + }) + .await; + } + + /// A crop trims the picture and stretches the rest over the item: an image + /// red on its left half and blue on its right shows blue where red was. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_crop_fills_the_item() { + local(async { + let dir = tempfile::tempdir().unwrap(); + image::RgbaImage::from_fn(8, 8, |x, _| image::Rgba(if x < 4 { RED } else { BLUE })) + .save(dir.path().join("halves.png")) + .unwrap(); + let mut collection = collection(); + collection.sources.insert( + "halves".into(), + Source::Image { + path: "halves.png".into(), + }, + ); + let fill = Rect::fill(&collection.canvas); + collection.scenes[0].items = vec![Item { + crop: crate::Crop { + left: 0.5, + ..Default::default() + }, + ..Item::new("halves", fill) + }]; + let program = drawn(collection, dir).await; + assert_eq!(pixel(&program, 3, 4), BLUE); + }) + .await; + } + + /// A blur is a Gaussian of the picture that fades into transparency past + /// its edges, as CSS's edge mode none: a blue square bleeds onto the black + /// around it by the discrete Gaussian's weights. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_blur_spreads_past_the_edges() { + local(async { + let square = Rect { + x: 2.0, + y: 2.0, + width: 4.0, + height: 4.0, + }; + let program = drawn( + showing(vec![Item { + filters: vec![Filter::Blur { radius: 1.0 }], + ..Item::new("blue", square) + }]), + tempfile::tempdir().unwrap(), + ) + .await; + // The weights of offsets 0 to 3 at a standard deviation of 1. + let weight = |offset: i32| (-(offset * offset) as f32 / 2.0).exp(); + let total: f32 = (-3..=3).map(weight).sum(); + let covered = |offsets: std::ops::RangeInclusive| offsets.map(weight).sum::() / total; + // One pixel left of the square, on its second row. + let expected = covered(1..=3) * covered(-1..=2) * 255.0; + let actual = pixel(&program, 1, 3); + assert!( + near(actual, [0, 0, expected.round() as u8, 255]), + "{actual:?} vs {expected}" + ); + assert_eq!(pixel(&program, 1, 3), pixel(&program, 6, 4), "symmetric"); + }) + .await; + } + + /// A blur many pixels wide is drawn on halved rasters, still matching the + /// Gaussian of a square to within a few levels. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_wide_blur_matches_the_gaussian() { + local(async { + let mut collection = collection(); + collection.canvas = Canvas { + width: 64, + height: 64, + framerate: 30, + }; + collection.scenes[0].items = vec![Item { + filters: vec![Filter::Blur { radius: 10.0 }], + ..Item::new("blue", Rect::fill(&collection.canvas)) + }]; + let program = drawn(collection, tempfile::tempdir().unwrap()).await; + let at = |x: usize, y: usize| program[(y * 64 + x) * 4 + 2]; + // The share of a Gaussian of deviation 10, centered half a pixel + // inside an edge, that falls inside: Φ(0.05). + let edge = 0.52; + let cases = [((32, 32), 255.0), ((0, 32), edge * 255.0), ((0, 0), edge * edge * 255.0)]; + for ((x, y), expected) in cases { + let actual = f32::from(at(x, y)); + assert!((actual - expected).abs() <= 4.0, "({x}, {y}): {actual} vs {expected}"); + } + }) + .await; + } + + /// A drop shadow is the picture's shape, offset and filled, under it. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_drop_shadow_falls_under() { + local(async { + let square = Rect { + x: 2.0, + y: 2.0, + width: 2.0, + height: 2.0, + }; + let program = drawn( + showing(vec![Item { + filters: vec![Filter::DropShadow { + x: 1.0, + y: 2.0, + blur: 0.0, + color: Rgba([255, 0, 0, 128]), + }], + ..Item::new("blue", square) + }]), + tempfile::tempdir().unwrap(), + ) + .await; + assert_eq!(pixel(&program, 3, 3), BLUE, "the picture over its shadow"); + let actual = pixel(&program, 4, 5); + assert!(near(actual, [128, 0, 0, 255]), "the shadow alone: {actual:?}"); + assert_eq!(pixel(&program, 4, 2), BLACK, "beside both"); + }) + .await; + } + + /// A chroma key hides its color and keeps the rest; a luma key hides what + /// is out of its range. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn keys_hide_what_they_key() { + local(async { + let mut collection = collection(); + let green = Rgba([0, 255, 0, 255]); + for (name, color) in [("green", green), ("white", Rgba([255; 4]))] { + collection.sources.insert(name.into(), Source::Color { color }); + } + let chroma = Filter::ChromaKey { + color: green, + similarity: 0.4, + smoothness: 0.08, + spill: 0.1, + }; + let luma = Filter::LumaKey { + min: 0.0, + max: 0.5, + smoothness: 0.0, + }; + let left = Rect { + x: 0.0, + y: 0.0, + width: 4.0, + height: 8.0, + }; + let right = Rect { x: 4.0, ..left }; + let keyed = |source: &str, rect, filter: &Filter| Item { + filters: vec![filter.clone()], + ..Item::new(source, rect) + }; + collection.scenes[0].items = vec![ + Item::new("red", Rect::fill(&collection.canvas)), + keyed("green", left, &chroma), + keyed("blue", right, &chroma), + keyed("white", Rect { height: 4.0, ..left }, &luma), + ]; + let program = drawn(collection, tempfile::tempdir().unwrap()).await; + assert_eq!(pixel(&program, 1, 6), RED, "green is keyed out"); + assert_eq!(pixel(&program, 6, 6), BLUE, "blue is kept"); + assert_eq!(pixel(&program, 1, 1), RED, "white is too bright"); + }) + .await; + } + + /// A LUT maps each color through its table, interpolating between points. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_lut_grades() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let mut cube = String::from("LUT_3D_SIZE 2\n"); + for b in 0..2 { + for g in 0..2 { + for r in 0..2 { + cube += &format!("{} {} {}\n", 1 - r, 1 - g, 1 - b); + } + } + } + std::fs::write(dir.path().join("invert.cube"), cube).unwrap(); + let mut collection = collection(); + collection.sources.insert( + "invert".into(), + Source::Lut { + path: "invert.cube".into(), + }, + ); + collection.sources.insert( + "tan".into(), + Source::Color { + color: Rgba([200, 100, 50, 255]), + }, + ); + let lut = |amount| Filter::Lut { + source: "invert".into(), + amount, + }; + let half = Rect { + x: 0.0, + y: 0.0, + width: 8.0, + height: 4.0, + }; + collection.scenes[0].items = vec![ + Item { + filters: vec![lut(1.0)], + ..Item::new("tan", half) + }, + Item { + filters: vec![lut(0.5)], + ..Item::new("blue", Rect { y: 4.0, ..half }) + }, + ]; + let program = drawn(collection, dir).await; + let actual = pixel(&program, 3, 1); + assert!(near(actual, rgb([55, 155, 205])), "{actual:?}"); + let actual = pixel(&program, 3, 6); + assert!(near(actual, rgb([128, 128, 128])), "half graded: {actual:?}"); + }) + .await; + } + + /// A mask keeps the picture where its source is opaque, or bright in + /// luminance mode, and an item whose mask has no picture yet draws nothing. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_mask_keeps_what_it_covers() { + local(async { + let dir = tempfile::tempdir().unwrap(); + image::RgbaImage::from_fn(8, 8, |x, _| { + image::Rgba(if x < 4 { [255; 4] } else { [255, 255, 255, 0] }) + }) + .save(dir.path().join("alpha.png")) + .unwrap(); + image::RgbaImage::from_fn(8, 8, |x, _| image::Rgba(if x < 4 { [255; 4] } else { BLACK })) + .save(dir.path().join("luma.png")) + .unwrap(); + let mut collection = collection(); + for (name, path) in [("alpha", "alpha.png"), ("luma", "luma.png")] { + collection + .sources + .insert(name.into(), Source::Image { path: path.into() }); + } + collection.sources.insert( + "offline".into(), + Source::Broadcast { + path: "room/offline".into(), + }, + ); + let masked = |mask: &str, mode, y| Item { + filters: vec![Filter::Mask { + source: mask.into(), + mode, + }], + ..Item::new( + "blue", + Rect { + x: 0.0, + y, + width: 8.0, + height: 2.0, + }, + ) + }; + collection.scenes[0].items = vec![ + Item::new("red", Rect::fill(&collection.canvas)), + masked("alpha", MaskMode::Alpha, 0.0), + masked("luma", MaskMode::Luminance, 2.0), + masked("offline", MaskMode::Alpha, 4.0), + ]; + let program = drawn(collection, dir).await; + for y in [0, 2] { + assert_eq!(pixel(&program, 1, y), BLUE, "row {y}"); + assert_eq!(pixel(&program, 6, y), RED, "row {y}"); + } + assert_eq!(pixel(&program, 1, 4), RED, "an offline mask hides the item"); + }) + .await; + } + + /// A blend mode mixes an item with what is under it by the CSS formula, + /// weighted by the item's opacity. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn blend_modes_mix_with_what_is_under() { + let cases = [ + (Blend::Multiply, 255, [0, 0, 0]), + (Blend::Screen, 255, [255, 0, 255]), + (Blend::Difference, 255, [255, 0, 255]), + (Blend::Luminosity, 255, [94, 0, 0]), + (Blend::Multiply, 128, [127, 0, 0]), + ]; + local(async { + for (blend, alpha, expected) in cases { + let mut collection = collection(); + collection.sources.insert( + "tint".into(), + Source::Color { + color: Rgba([0, 0, 255, alpha]), + }, + ); + let fill = Rect::fill(&collection.canvas); + collection.scenes[0].items = vec![ + Item::new("red", fill), + Item { + blend, + ..Item::new("tint", fill) + }, + ]; + let program = drawn(collection, tempfile::tempdir().unwrap()).await; + let actual = pixel(&program, 3, 3); + assert!(near(actual, rgb(expected)), "{blend:?} at {alpha}: {actual:?}"); + } + }) + .await; + } } diff --git a/rs/moq-scene/src/edit.rs b/rs/moq-scene/src/edit.rs index 8674538..d6ebbf2 100644 --- a/rs/moq-scene/src/edit.rs +++ b/rs/moq-scene/src/edit.rs @@ -14,7 +14,7 @@ use std::collections::VecDeque; use serde::{Deserialize, Serialize}; use crate::Error; -use crate::collection::{Audio, Collection, Item, Layer, Scene, Source, Transition, Video}; +use crate::collection::{Audio, Collection, Filter, Item, Layer, Scene, Source, Transition, Video}; /// One part of a collection, before and after an edit. /// @@ -378,7 +378,8 @@ impl Draft { } /// Remove the source called `name` and everything that refers to it: its - /// clones, the items showing either, and their audio and video settings. + /// clones, the items showing either, their audio and video settings, and + /// the filters reading it, such as a mask. pub fn remove_source(&mut self, name: &str) -> Result<(), Error> { let clones: Vec = self .collection @@ -403,6 +404,36 @@ impl Draft { for (layer, index) in placed.into_iter().rev() { self.remove_item(&layer, index)?; } + + let reads = |filter: &Filter| filter.source() == Some(name); + let items: Vec<(Layer, usize, Item)> = self + .collection + .layers() + .flat_map(|(layer, items)| { + let reading = items + .iter() + .enumerate() + .filter(|(_, item)| item.filters.iter().any(reads)); + reading + .map(move |(index, item)| (layer.clone(), index, item.clone())) + .collect::>() + }) + .collect(); + for (layer, index, mut item) in items { + item.filters.retain(|filter| !reads(filter)); + self.set_item(&layer, index, item)?; + } + let videos: Vec<(String, Video)> = self + .collection + .video + .iter() + .filter(|(_, video)| video.filters.iter().any(reads)) + .map(|(name, video)| (name.clone(), video.clone())) + .collect(); + for (other, mut video) in videos { + video.filters.retain(|filter| !reads(filter)); + self.set_video(&other, Some(video))?; + } if self.collection.audio.contains_key(name) { self.set_audio(name, None)?; } @@ -780,6 +811,7 @@ mod tests { "red copy".into(), Video { filters: vec![Filter::Opacity { amount: 0.5 }], + ..Default::default() }, ); collection @@ -804,6 +836,43 @@ mod tests { assert_eq!(collection, before); } + /// A removed source's filters go with it, from items and other sources + /// alike, leaving their other filters. + #[test] + fn removing_a_source_removes_the_filters_reading_it() { + let mut collection = collection(); + collection.sources.insert("matte".into(), color([0, 0, 0, 128])); + collection.sources.insert( + "grade".into(), + Source::Lut { + path: "grade.cube".into(), + }, + ); + let mask = Filter::Mask { + source: "matte".into(), + mode: Default::default(), + }; + let lut = Filter::Lut { + source: "grade".into(), + amount: 1.0, + }; + collection.scenes[0].items[0].filters = vec![mask.clone(), lut.clone()]; + collection.video.insert( + "red".into(), + Video { + filters: vec![lut.clone(), mask], + ..Default::default() + }, + ); + collection.validate().unwrap(); + + let mut draft = collection.draft("Remove matte"); + draft.remove_source("matte").unwrap(); + collection.apply(&draft.finish().unwrap()).unwrap(); + assert_eq!(collection.scenes[0].items[0].filters, std::slice::from_ref(&lut)); + assert_eq!(collection.video["red"].filters, [lut]); + } + /// An edit that leaves the collection invalid is refused whole. #[test] fn refuses_an_edit_that_leaves_it_invalid() { diff --git a/rs/moq-scene/src/effect.rs b/rs/moq-scene/src/effect.rs new file mode 100644 index 0000000..3ceb6ab --- /dev/null +++ b/rs/moq-scene/src/effect.rs @@ -0,0 +1,1241 @@ +//! The GPU passes that draw an item through its crop, filters, and blend mode. +//! +//! Per-pixel filters (color, keys, LUTs, masks) run in the shader that draws +//! the item, so an item without a blur or shadow costs one draw however many +//! it has. A blur or drop shadow needs the pixels around each one, so the +//! picture up to it is drawn into a raster first. A raster covers only what +//! can still reach the canvas, and never more than twice the canvas each way, +//! so its memory is bounded by the canvas rather than by the item or filter. A +//! wide blur halves its raster until it is a few texels across, so its cost is +//! bounded too. + +use std::num::NonZeroU64; + +use moq_video::render::wgpu; + +use crate::collection::{Blend, Canvas, Crop, Filter, MaskMode, Rect, Rgba}; +use crate::compositor::FORMAT; +use crate::lut::Cube; + +const SHADER: &str = include_str!("effect.wgsl"); + +/// Per-pixel filters one draw runs; a longer run is split across rasters. +const OPS: usize = 8; + +/// What rasters between passes hold: premultiplied, and fine enough that a +/// blur's faint tail does not band. +const RASTER: wgpu::TextureFormat = wgpu::TextureFormat::Rgba16Float; + +/// The widest blur, in texels, drawn without first halving its raster. +const SIGMA: f32 = 4.0; + +/// Texture coordinates covering a whole texture. +const UNIT: [f32; 4] = [0.0, 0.0, 1.0, 1.0]; + +/// Clip space covering a whole target, top-left then bottom-right. +const FULL: [f32; 4] = [-1.0, 1.0, 1.0, -1.0]; + +/// One per-pixel filter, as `effect.wgsl`'s `Op`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, bytemuck::Pod, bytemuck::Zeroable)] +struct Op { + kind: [u32; 4], + params: [f32; 4], + /// Column-major, as WGSL's `mat4x4`. + matrix: [[f32; 4]; 4], + offset: [f32; 4], +} + +impl Op { + const MATRIX: u32 = 0; + const CHROMA_KEY: u32 = 1; + const LUMA_KEY: u32 = 2; + const LUT: u32 = 3; + const MASK: u32 = 4; + + /// `rows` times straight RGBA, plus `offset`, clamped. + fn matrix(rows: [[f32; 4]; 4], offset: [f32; 4]) -> Self { + let mut matrix = [[0.0; 4]; 4]; + for (row, values) in rows.iter().enumerate() { + for (column, value) in values.iter().enumerate() { + matrix[column][row] = *value; + } + } + Self { + kind: [Self::MATRIX, 0, 0, 0], + matrix, + offset, + ..Default::default() + } + } + + /// A 3x3 color matrix, leaving alpha alone. + fn color(rgb: [[f32; 3]; 3], offset: f32) -> Self { + let row = |r: [f32; 3]| [r[0], r[1], r[2], 0.0]; + Self::matrix( + [row(rgb[0]), row(rgb[1]), row(rgb[2]), [0.0, 0.0, 0.0, 1.0]], + [offset, offset, offset, 0.0], + ) + } + + /// Each color channel times `slope`, plus `intercept`. + fn linear(slope: f32, intercept: f32) -> Self { + Self::color([[slope, 0.0, 0.0], [0.0, slope, 0.0], [0.0, 0.0, slope]], intercept) + } + + /// The per-pixel filters that read nothing but the pixel, by the CSS Filter + /// Effects spec's equivalents; `None` for every other filter. + fn of(filter: &Filter) -> Option { + Some(match *filter { + Filter::Opacity { amount } => { + let mut rows = IDENTITY; + rows[3][3] = amount; + Self::matrix(rows, [0.0; 4]) + } + Filter::Brightness { amount } => Self::linear(amount, 0.0), + Filter::Contrast { amount } => Self::linear(amount, 0.5 - 0.5 * amount), + Filter::Invert { amount } => Self::linear(1.0 - 2.0 * amount, amount), + Filter::Saturate { amount: s } => Self::color( + [ + [0.213 + 0.787 * s, 0.715 - 0.715 * s, 0.072 - 0.072 * s], + [0.213 - 0.213 * s, 0.715 + 0.285 * s, 0.072 - 0.072 * s], + [0.213 - 0.213 * s, 0.715 - 0.715 * s, 0.072 + 0.928 * s], + ], + 0.0, + ), + Filter::Grayscale { amount } => { + let a = 1.0 - amount; + Self::color( + [ + [0.2126 + 0.7874 * a, 0.7152 - 0.7152 * a, 0.0722 - 0.0722 * a], + [0.2126 - 0.2126 * a, 0.7152 + 0.2848 * a, 0.0722 - 0.0722 * a], + [0.2126 - 0.2126 * a, 0.7152 - 0.7152 * a, 0.0722 + 0.9278 * a], + ], + 0.0, + ) + } + Filter::Sepia { amount } => { + let a = 1.0 - amount; + Self::color( + [ + [0.393 + 0.607 * a, 0.769 - 0.769 * a, 0.189 - 0.189 * a], + [0.349 - 0.349 * a, 0.686 + 0.314 * a, 0.168 - 0.168 * a], + [0.272 - 0.272 * a, 0.534 - 0.534 * a, 0.131 + 0.869 * a], + ], + 0.0, + ) + } + Filter::HueRotate { degrees } => { + let (sin, cos) = degrees.to_radians().sin_cos(); + Self::color( + [ + [ + 0.213 + cos * 0.787 - sin * 0.213, + 0.715 - cos * 0.715 - sin * 0.715, + 0.072 - cos * 0.072 + sin * 0.928, + ], + [ + 0.213 - cos * 0.213 + sin * 0.143, + 0.715 + cos * 0.285 + sin * 0.140, + 0.072 - cos * 0.072 - sin * 0.283, + ], + [ + 0.213 - cos * 0.213 - sin * 0.787, + 0.715 - cos * 0.715 + sin * 0.715, + 0.072 + cos * 0.928 + sin * 0.072, + ], + ], + 0.0, + ) + } + Filter::ChromaKey { + color, + similarity, + smoothness, + spill, + } => { + let [r, g, b, _] = color.0.map(|channel| f32::from(channel) / 255.0); + let cb = -0.1146 * r - 0.3854 * g + 0.5 * b; + let cr = 0.5 * r - 0.4542 * g - 0.0458 * b; + Self { + kind: [Self::CHROMA_KEY, 0, 0, 0], + params: [cb, cr, similarity, smoothness], + offset: [spill, 0.0, 0.0, 0.0], + ..Default::default() + } + } + Filter::LumaKey { min, max, smoothness } => Self { + kind: [Self::LUMA_KEY, 0, 0, 0], + params: [min, max, smoothness, 0.0], + ..Default::default() + }, + Filter::Blur { .. } | Filter::DropShadow { .. } | Filter::Lut { .. } | Filter::Mask { .. } => return None, + }) + } +} + +const IDENTITY: [[f32; 4]; 4] = [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], +]; + +/// One draw's uniform, as `effect.wgsl`'s `Draw`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, bytemuck::Pod, bytemuck::Zeroable)] +struct Uniform { + rect: [f32; 4], + input: [f32; 4], + valid: [f32; 4], + item: [f32; 4], + aux: [f32; 4], + mode: [u32; 4], + blur: [f32; 4], + color: [f32; 4], + ops: [Op; OPS], +} + +/// A region of the canvas, in canvas pixels: left, top, right, bottom. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Area { + x0: f32, + y0: f32, + x1: f32, + y1: f32, +} + +impl Area { + fn of(rect: Rect) -> Self { + Self { + x0: rect.x, + y0: rect.y, + x1: rect.x + rect.width, + y1: rect.y + rect.height, + } + } + + fn canvas(canvas: Canvas) -> Self { + Self::of(Rect::fill(&canvas)) + } + + fn width(&self) -> f32 { + self.x1 - self.x0 + } + + fn height(&self) -> f32 { + self.y1 - self.y0 + } + + fn is_empty(&self) -> bool { + !(self.x1 > self.x0 && self.y1 > self.y0) + } + + fn grow(&self, x: f32, y: f32) -> Self { + Self { + x0: self.x0 - x, + y0: self.y0 - y, + x1: self.x1 + x, + y1: self.y1 + y, + } + } + + fn shift(&self, x: f32, y: f32) -> Self { + Self { + x0: self.x0 + x, + y0: self.y0 + y, + x1: self.x1 + x, + y1: self.y1 + y, + } + } + + fn intersect(&self, other: &Self) -> Self { + Self { + x0: self.x0.max(other.x0), + y0: self.y0.max(other.y0), + x1: self.x1.min(other.x1), + y1: self.y1.min(other.y1), + } + } + + fn union(&self, other: &Self) -> Self { + Self { + x0: self.x0.min(other.x0), + y0: self.y0.min(other.y0), + x1: self.x1.max(other.x1), + y1: self.y1.max(other.y1), + } + } + + /// The whole pixels covering this area, or `None` if it covers none. + fn pixels(&self) -> Option { + let area = Self { + x0: self.x0.floor(), + y0: self.y0.floor(), + x1: self.x1.ceil(), + y1: self.y1.ceil(), + }; + (!self.is_empty()).then_some(area) + } + + /// What coordinates `uv`, given at this area's corners, are at `other`'s. + fn map(&self, uv: [f32; 4], other: &Self) -> [f32; 4] { + let u = |x: f32| uv[0] + (x - self.x0) / self.width() * (uv[2] - uv[0]); + let v = |y: f32| uv[1] + (y - self.y0) / self.height() * (uv[3] - uv[1]); + [u(other.x0), v(other.y0), u(other.x1), v(other.y1)] + } +} + +/// A filter's place in the passes: per pixel, or reading around each pixel. +#[derive(Clone, Copy)] +enum Stage<'a> { + Pixel(&'a Filter), + Blur(f32), + Shadow { x: f32, y: f32, blur: f32, color: Rgba }, +} + +impl<'a> Stage<'a> { + fn of(filter: &'a Filter) -> Self { + match *filter { + Filter::Blur { radius } => Self::Blur(radius), + Filter::DropShadow { x, y, blur, color } => Self::Shadow { x, y, blur, color }, + _ => Self::Pixel(filter), + } + } + + /// How far, in canvas pixels, an output pixel reads from. + fn reach(&self) -> (f32, f32) { + match *self { + Self::Pixel(_) => (0.0, 0.0), + Self::Blur(sigma) => (reach(sigma), reach(sigma)), + Self::Shadow { x, y, blur, .. } => (x.abs() + reach(blur), y.abs() + reach(blur)), + } + } +} + +/// Where a Gaussian of `sigma` is negligible: three standard deviations. +fn reach(sigma: f32) -> f32 { + (3.0 * sigma).ceil() +} + +/// A source's newest picture, bound to sample. +#[derive(Clone)] +pub(crate) struct Picture { + view: wgpu::TextureView, + bind: wgpu::BindGroup, +} + +/// A LUT source's table on the GPU. +#[derive(Clone)] +pub(crate) struct Lut { + view: wgpu::TextureView, + size: u32, + min: [f32; 3], + max: [f32; 3], +} + +/// What a draw samples: a picture covering `area` of the canvas, at texture +/// coordinates `uv` at its corners, and transparent outside `valid`. +#[derive(Clone)] +struct Input { + picture: Picture, + area: Area, + uv: [f32; 4], + valid: [f32; 4], + premultiplied: bool, +} + +/// Per-pixel filters waiting to run when an input is sampled. +#[derive(Clone, Default)] +struct Segment { + ops: Vec, + lut: Option, + mask: Option, +} + +/// An item ready to draw onto a canvas-sized target: its last input, the +/// filters still to run, and where on the canvas it can show. +#[derive(Clone)] +pub(crate) struct Prepared { + input: Input, + segment: Segment, + item: Area, + bounds: Area, + blend: Blend, +} + +/// Which pipeline a draw uses. +#[derive(Clone, Copy)] +enum Kind { + /// Per-pixel filters, over a canvas. + Canvas, + /// Per-pixel filters, into a raster. + Raster, + /// Per-pixel filters, mixed with a backdrop by a blend mode. + Blend, + /// One direction of a blur, or a halving. + Blur, + /// A picture over its shadow. + Shadow, +} + +struct Draw { + kind: Kind, + input: wgpu::BindGroup, + extra: wgpu::BindGroup, + uniform: Uniform, +} + +enum Command { + /// Draws into `view`, first cleared to `clear` unless `None`. + Pass { + view: wgpu::TextureView, + clear: Option, + draws: Vec, + }, + /// Copy `area` of a target into the backdrop, for a blend mode to read. + Backdrop { texture: wgpu::Texture, area: Area }, +} + +/// One render's passes: the rasters items are filtered through, then the +/// targets, in the order they are drawn. +#[derive(Default)] +pub(crate) struct Frame { + rasters: Vec, + targets: Vec, +} + +impl Frame { + /// Start drawing a canvas-sized target, cleared to opaque black. + pub fn target(&mut self, target: &Picture) { + self.targets.push(Command::Pass { + view: target.view.clone(), + clear: Some(wgpu::Color::BLACK), + draws: Vec::new(), + }); + } + + fn draw(&mut self, draw: Draw) { + match self.targets.last_mut() { + Some(Command::Pass { draws, .. }) => draws.push(draw), + _ => unreachable!("a draw follows its target"), + } + } +} + +/// A raster texture, reused across renders while one of its size is needed. +#[derive(Clone)] +struct Raster { + size: (u32, u32), + picture: Picture, +} + +/// What an item draws, as [`Effects::prepare`] takes it. +pub(crate) struct Item<'a> { + /// Its source's newest picture. + pub picture: &'a Picture, + /// Where on the canvas. + pub rect: Rect, + /// Its source's crop, then its own. + pub crop: Crop, + /// Its source's filters, then its own. + pub filters: Vec<&'a Filter>, + /// How it mixes with what is under it. + pub blend: Blend, +} + +/// The pictures and tables filters read, by source name. +pub(crate) trait Library { + /// The newest picture of the source called `name`. + fn picture(&self, name: &str) -> Option; + /// The table of the LUT source called `name`. + fn lut(&self, name: &str) -> Option; +} + +/// The GPU objects every effect draws with. +pub(crate) struct Effects { + device: wgpu::Device, + queue: wgpu::Queue, + canvas: Canvas, + + input_layout: wgpu::BindGroupLayout, + extra_layout: wgpu::BindGroupLayout, + uniform_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + canvas_pipeline: wgpu::RenderPipeline, + raster_pipeline: wgpu::RenderPipeline, + blend_pipeline: wgpu::RenderPipeline, + blur_pipeline: wgpu::RenderPipeline, + shadow_pipeline: wgpu::RenderPipeline, + + uniforms: Uniforms, + /// Group 2 for a draw that reads no LUT, mask, or aux texture. + none: wgpu::BindGroup, + empty_2d: wgpu::TextureView, + empty_3d: wgpu::TextureView, + /// What a blend mode mixes with: a copy of its target under it. + backdrop: wgpu::Texture, + backdrop_view: wgpu::TextureView, + + /// Rasters used by the last render, reused by this one. + free: Vec, + /// Rasters used by this render. + used: Vec, +} + +impl Effects { + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, canvas: Canvas) -> Self { + let texture = |binding, sample_type, view_dimension| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type, + view_dimension, + multisampled: false, + }, + count: None, + }; + let float = wgpu::TextureSampleType::Float { filterable: true }; + let input_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene input"), + entries: &[ + texture(0, float, wgpu::TextureViewDimension::D2), + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let extra_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene extra"), + entries: &[ + // Rgba32Float, read with textureLoad: not every device filters it. + texture( + 0, + wgpu::TextureSampleType::Float { filterable: false }, + wgpu::TextureViewDimension::D3, + ), + texture(1, float, wgpu::TextureViewDimension::D2), + texture(2, float, wgpu::TextureViewDimension::D2), + ], + }); + let uniform_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene draw"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: true, + min_binding_size: NonZeroU64::new(size_of::() as u64), + }, + count: None, + }], + }); + let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("moq-scene"), + bind_group_layouts: &[Some(&uniform_layout), Some(&input_layout), Some(&extra_layout)], + immediate_size: 0, + }); + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("moq-scene"), + source: wgpu::ShaderSource::Wgsl(SHADER.into()), + }); + let pipeline = |entry: &str, format, blend| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(entry), + layout: Some(&layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some(entry), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format, + blend, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }) + }; + // Premultiplied over an opaque canvas, which stays opaque. + let over = Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING); + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("moq-scene"), + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + let empty = |dimension, format| { + device + .create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene empty"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) + .create_view(&Default::default()) + }; + let empty_2d = empty(wgpu::TextureDimension::D2, FORMAT); + let empty_3d = empty(wgpu::TextureDimension::D3, wgpu::TextureFormat::Rgba32Float); + let backdrop = device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene backdrop"), + size: wgpu::Extent3d { + width: canvas.width, + height: canvas.height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + + let alignment = u64::from(device.limits().min_uniform_buffer_offset_alignment); + let stride = (size_of::() as u64).next_multiple_of(alignment); + Self { + canvas_pipeline: pipeline("fs_effect", FORMAT, over), + raster_pipeline: pipeline("fs_effect", RASTER, None), + blend_pipeline: pipeline("fs_blend", FORMAT, None), + blur_pipeline: pipeline("fs_blur", RASTER, None), + shadow_pipeline: pipeline("fs_shadow", RASTER, None), + uniforms: Uniforms::new(device, &uniform_layout, stride, 16), + none: extra_bind(device, &extra_layout, [&empty_3d, &empty_2d, &empty_2d]), + device: device.clone(), + queue: queue.clone(), + canvas, + input_layout, + extra_layout, + uniform_layout, + sampler, + empty_2d, + empty_3d, + backdrop_view: backdrop.create_view(&Default::default()), + backdrop, + free: Vec::new(), + used: Vec::new(), + } + } + + /// `texture`, a source's picture or a canvas-sized target, bound to sample. + pub fn picture(&self, texture: &wgpu::Texture) -> Picture { + let view = texture.create_view(&Default::default()); + let bind = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene input"), + layout: &self.input_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + Picture { view, bind } + } + + /// Upload a LUT's table. + pub fn lut(&self, cube: &Cube) -> Lut { + let extent = wgpu::Extent3d { + width: cube.size, + height: cube.size, + depth_or_array_layers: cube.size, + }; + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene LUT"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D3, + format: wgpu::TextureFormat::Rgba32Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let texels: Vec<[f32; 4]> = cube.table.iter().map(|&[r, g, b]| [r, g, b, 1.0]).collect(); + self.queue.write_texture( + texture.as_image_copy(), + bytemuck::cast_slice(&texels), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(cube.size * 16), + rows_per_image: Some(cube.size), + }, + extent, + ); + Lut { + view: texture.create_view(&Default::default()), + size: cube.size, + min: cube.min, + max: cube.max, + } + } + + /// Start a render's passes. + pub fn frame(&mut self) -> Frame { + // Rasters the last render did not use are dropped. + self.free = std::mem::take(&mut self.used); + Frame::default() + } + + /// Run `frame`'s passes. + pub fn submit(&mut self, frame: Frame) { + let commands: Vec = frame.rasters.into_iter().chain(frame.targets).collect(); + let draws = commands.iter().flat_map(|command| match command { + Command::Pass { draws, .. } => draws.as_slice(), + Command::Backdrop { .. } => &[], + }); + let count = draws.clone().count(); + self.uniforms.reserve(&self.device, &self.uniform_layout, count as u64); + let stride = self.uniforms.stride as usize; + let mut bytes = vec![0u8; count * stride]; + for (index, draw) in draws.enumerate() { + let offset = index * stride; + bytes[offset..offset + size_of::()].copy_from_slice(bytemuck::bytes_of(&draw.uniform)); + } + if !bytes.is_empty() { + self.queue.write_buffer(&self.uniforms.buffer, 0, &bytes); + } + + let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("moq-scene"), + }); + let mut index = 0u32; + for command in &commands { + let (view, clear, draws) = match command { + Command::Pass { view, clear, draws } => (view, clear, draws), + Command::Backdrop { texture, area } => { + let origin = wgpu::Origin3d { + x: area.x0 as u32, + y: area.y0 as u32, + z: 0, + }; + encoder.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { + origin, + ..texture.as_image_copy() + }, + wgpu::TexelCopyTextureInfo { + origin, + ..self.backdrop.as_image_copy() + }, + wgpu::Extent3d { + width: area.width() as u32, + height: area.height() as u32, + depth_or_array_layers: 1, + }, + ); + continue; + } + }; + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("moq-scene"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: clear.map_or(wgpu::LoadOp::Load, wgpu::LoadOp::Clear), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + for draw in draws { + pass.set_pipeline(match draw.kind { + Kind::Canvas => &self.canvas_pipeline, + Kind::Raster => &self.raster_pipeline, + Kind::Blend => &self.blend_pipeline, + Kind::Blur => &self.blur_pipeline, + Kind::Shadow => &self.shadow_pipeline, + }); + pass.set_bind_group(0, &self.uniforms.bind, &[index * stride as u32]); + pass.set_bind_group(1, &draw.input, &[]); + pass.set_bind_group(2, &draw.extra, &[]); + pass.draw(0..4, 0..1); + index += 1; + } + } + self.queue.submit([encoder.finish()]); + } + + /// `item` through its crop and filters, ready to draw onto targets: `None` + /// if nothing of it reaches the canvas, or a source a filter reads has + /// nothing in `library` yet. + pub fn prepare(&mut self, frame: &mut Frame, item: Item<'_>, library: &dyn Library) -> Option { + let area = Area::of(item.rect); + let canvas = Area::canvas(self.canvas); + if area.is_empty() { + return None; + } + let crop = [ + item.crop.left, + item.crop.top, + 1.0 - item.crop.right, + 1.0 - item.crop.bottom, + ]; + let mut input = Input { + picture: item.picture.clone(), + area, + uv: crop, + valid: crop, + premultiplied: false, + }; + + let stages: Vec = item.filters.iter().map(|filter| Stage::of(filter)).collect(); + // How far past the canvas each stage's input must reach, for the stages + // from it on. + let mut margins = vec![(0.0, 0.0); stages.len() + 1]; + for (index, stage) in stages.iter().enumerate().rev() { + let (x, y) = stage.reach(); + margins[index] = (margins[index + 1].0 + x, margins[index + 1].1 + y); + } + let clip = |index: usize| canvas.grow(margins[index].0, margins[index].1); + + let mut bounds = area; + let mut segment = Segment::default(); + for (index, stage) in stages.iter().enumerate() { + match *stage { + Stage::Pixel(filter) => { + let (op, lut, mask) = match filter { + Filter::Lut { source, amount } => { + let lut = library.lut(source)?; + let op = Op { + kind: [Op::LUT, 0, 0, 0], + params: [*amount, lut.size as f32, 0.0, 0.0], + matrix: [[lut.max[0], lut.max[1], lut.max[2], 0.0], [0.0; 4], [0.0; 4], [0.0; 4]], + offset: [lut.min[0], lut.min[1], lut.min[2], 0.0], + }; + (op, Some(lut.view), None) + } + Filter::Mask { source, mode } => { + let mode = match mode { + MaskMode::Alpha => 0.0, + MaskMode::Luminance => 1.0, + }; + let op = Op { + kind: [Op::MASK, 0, 0, 0], + params: [mode, 0.0, 0.0, 0.0], + ..Default::default() + }; + (op, None, Some(library.picture(source)?.view)) + } + filter => (Op::of(filter).expect("a per-pixel filter"), None, None), + }; + let full = segment.ops.len() == OPS + || (lut.is_some() && segment.lut.is_some()) + || (mask.is_some() && segment.mask.is_some()); + if full { + let region = bounds.intersect(&clip(index)).pixels()?; + input = self.raster(frame, &input, &segment, area, region); + segment = Segment::default(); + } + segment.ops.push(op); + if mask.is_some() { + // Nothing shows outside the mask, which covers the item. + bounds = bounds.intersect(&area); + segment.mask = mask; + } + segment.lut = lut.or(segment.lut); + } + Stage::Blur(sigma) => { + if sigma == 0.0 { + continue; + } + let spread = reach(sigma); + let region = bounds.grow(spread, spread).intersect(&clip(index)).pixels()?; + input = self.blur(frame, &input, &segment, area, region, sigma); + segment = Segment::default(); + bounds = region; + } + Stage::Shadow { x, y, blur, color } => { + let spread = reach(blur); + let source = bounds.intersect(&clip(index)).pixels()?; + let picture = self.raster(frame, &input, &segment, area, source); + let shape = if blur == 0.0 { + picture.clone() + } else { + let region = bounds.grow(spread, spread).intersect(&clip(index)).pixels()?; + self.blur(frame, &picture, &Segment::default(), area, region, blur) + }; + let shadow = bounds.shift(x, y).grow(spread, spread); + let region = bounds.union(&shadow).intersect(&clip(index + 1)).pixels()?; + input = self.shadow(frame, &picture, &shape, region, (x, y), color); + segment = Segment::default(); + bounds = region; + } + } + } + + let bounds = bounds.intersect(&canvas); + if bounds.is_empty() { + return None; + } + Some(Prepared { + input, + segment, + item: area, + bounds, + blend: item.blend, + }) + } + + /// A whole canvas-sized `picture` faded to `opacity`, to draw over another. + pub fn fade(&self, picture: &Picture, opacity: f32) -> Prepared { + let canvas = Area::canvas(self.canvas); + Prepared { + input: Input { + picture: picture.clone(), + area: canvas, + uv: UNIT, + valid: UNIT, + premultiplied: false, + }, + segment: Segment { + ops: vec![Op::of(&Filter::Opacity { amount: opacity }).expect("per pixel")], + ..Default::default() + }, + item: canvas, + bounds: canvas, + blend: Blend::Normal, + } + } + + /// Draw `prepared` onto the target `frame` last started, which is `texture`. + pub fn draw(&self, frame: &mut Frame, texture: &wgpu::Texture, prepared: &Prepared) { + let Prepared { + input, + segment, + item, + bounds, + blend, + } = prepared; + let (width, height) = (self.canvas.width as f32, self.canvas.height as f32); + let x = |px: f32| px / width * 2.0 - 1.0; + let y = |px: f32| 1.0 - px / height * 2.0; + let mut uniform = self.uniform(input, segment, item, bounds); + uniform.rect = [x(bounds.x0), y(bounds.y0), x(bounds.x1), y(bounds.y1)]; + uniform.mode[2] = *blend as u32; + + if *blend == Blend::Normal { + frame.draw(Draw { + kind: Kind::Canvas, + input: input.picture.bind.clone(), + extra: self.extra(segment.lut.as_ref(), segment.mask.as_ref(), None), + uniform, + }); + return; + } + + // A blend mode reads what is under it, which a pass cannot while drawing + // it: copy that much aside, then carry on in a new pass. + let area = bounds.pixels().expect("prepared bounds are not empty"); + let Some(Command::Pass { view, .. }) = frame.targets.last() else { + unreachable!("a draw follows its target"); + }; + let view = view.clone(); + frame.targets.push(Command::Backdrop { + texture: texture.clone(), + area, + }); + frame.targets.push(Command::Pass { + view, + clear: None, + draws: Vec::new(), + }); + frame.draw(Draw { + kind: Kind::Blend, + input: input.picture.bind.clone(), + extra: self.extra(segment.lut.as_ref(), segment.mask.as_ref(), Some(&self.backdrop_view)), + uniform, + }); + } + + /// The uniform sampling `input` through `segment` over `quad` of the canvas. + fn uniform(&self, input: &Input, segment: &Segment, item: &Area, quad: &Area) -> Uniform { + let mut ops = [Op::default(); OPS]; + ops[..segment.ops.len()].copy_from_slice(&segment.ops); + Uniform { + rect: FULL, + input: input.area.map(input.uv, quad), + valid: input.valid, + item: item.map(UNIT, quad), + mode: [segment.ops.len() as u32, input.premultiplied.into(), 0, 0], + ops, + ..Default::default() + } + } + + /// `input` through `segment`, drawn into a raster covering `area`. + fn raster(&mut self, frame: &mut Frame, input: &Input, segment: &Segment, item: Area, area: Area) -> Input { + let size = self.texels(&area); + let raster = self.acquire(size); + let draw = Draw { + kind: Kind::Raster, + input: input.picture.bind.clone(), + extra: self.extra(segment.lut.as_ref(), segment.mask.as_ref(), None), + uniform: self.uniform(input, segment, &item, &area), + }; + frame.rasters.push(Command::Pass { + view: raster.picture.view.clone(), + clear: Some(wgpu::Color::TRANSPARENT), + draws: vec![draw], + }); + Input { + picture: raster.picture, + area, + uv: UNIT, + valid: UNIT, + premultiplied: true, + } + } + + /// `input` through `segment`, blurred by `sigma` canvas pixels over `area`. + fn blur(&mut self, frame: &mut Frame, input: &Input, segment: &Segment, item: Area, area: Area, sigma: f32) -> Input { + let mut current = self.raster(frame, input, segment, item, area); + let mut size = self.texels(&area); + let texels = |size: (u32, u32)| (size.0 as f32 / area.width(), size.1 as f32 / area.height()); + // Halve until the blur is a few texels wide; each texel of a halving + // averages four, so nothing aliases. + while sigma * texels(size).0.max(texels(size).1) > SIGMA && (size.0 > 1 || size.1 > 1) { + size = (size.0.div_ceil(2), size.1.div_ceil(2)); + current = self.pass(frame, Kind::Blur, ¤t, size, [0.0, 0.0, 1.0, 0.0]); + } + let (x, y) = texels(size); + current = self.pass( + frame, + Kind::Blur, + ¤t, + size, + [1.0 / size.0 as f32, 0.0, sigma * x, reach(sigma * x)], + ); + self.pass( + frame, + Kind::Blur, + ¤t, + size, + [0.0, 1.0 / size.1 as f32, sigma * y, reach(sigma * y)], + ) + } + + /// A whole raster `input` drawn into a new raster of `size` by `kind`. + fn pass(&mut self, frame: &mut Frame, kind: Kind, input: &Input, size: (u32, u32), blur: [f32; 4]) -> Input { + let raster = self.acquire(size); + let uniform = Uniform { + rect: FULL, + input: UNIT, + valid: UNIT, + mode: [0, 1, 0, 0], + blur, + ..Default::default() + }; + frame.rasters.push(Command::Pass { + view: raster.picture.view.clone(), + clear: Some(wgpu::Color::TRANSPARENT), + draws: vec![Draw { + kind, + input: input.picture.bind.clone(), + extra: self.none.clone(), + uniform, + }], + }); + Input { + picture: raster.picture, + ..input.clone() + } + } + + /// `picture` over its `shape` moved by `offset` and filled with `color`, + /// in a raster covering `area`. + fn shadow( + &mut self, + frame: &mut Frame, + picture: &Input, + shape: &Input, + area: Area, + offset: (f32, f32), + color: Rgba, + ) -> Input { + let raster = self.acquire(self.texels(&area)); + let uniform = Uniform { + rect: FULL, + input: picture.area.map(UNIT, &area), + valid: UNIT, + aux: shape.area.map(UNIT, &area.shift(-offset.0, -offset.1)), + mode: [0, 1, 0, 0], + color: color.0.map(|channel| f32::from(channel) / 255.0), + ..Default::default() + }; + frame.rasters.push(Command::Pass { + view: raster.picture.view.clone(), + clear: Some(wgpu::Color::TRANSPARENT), + draws: vec![Draw { + kind: Kind::Shadow, + input: picture.picture.bind.clone(), + extra: self.extra(None, None, Some(&shape.picture.view)), + uniform, + }], + }); + Input { + picture: raster.picture, + area, + uv: UNIT, + valid: UNIT, + premultiplied: true, + } + } + + /// The texels a raster covering `area` gets: one per canvas pixel, unless + /// that is more than twice the canvas or the device allows either way. + fn texels(&self, area: &Area) -> (u32, u32) { + let limit = self.device.limits().max_texture_dimension_2d; + let most = |canvas: u32| (canvas * 2).min(limit) as f32; + let scale = 1f32 + .min(most(self.canvas.width) / area.width()) + .min(most(self.canvas.height) / area.height()); + let texels = |length: f32| ((length * scale).ceil() as u32).max(1); + (texels(area.width()), texels(area.height())) + } + + /// A raster of `size` for this render, reusing one the last render used. + fn acquire(&mut self, size: (u32, u32)) -> Raster { + let raster = match self.free.iter().position(|raster| raster.size == size) { + Some(index) => self.free.swap_remove(index), + None => { + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene raster"), + size: wgpu::Extent3d { + width: size.0, + height: size.1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: RASTER, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + Raster { + size, + picture: self.picture(&texture), + } + } + }; + self.used.push(raster.clone()); + raster + } + + /// Group 2: the LUT, mask, and aux textures a draw reads. + fn extra( + &self, + lut: Option<&wgpu::TextureView>, + mask: Option<&wgpu::TextureView>, + aux: Option<&wgpu::TextureView>, + ) -> wgpu::BindGroup { + if lut.is_none() && mask.is_none() && aux.is_none() { + return self.none.clone(); + } + extra_bind( + &self.device, + &self.extra_layout, + [ + lut.unwrap_or(&self.empty_3d), + mask.unwrap_or(&self.empty_2d), + aux.unwrap_or(&self.empty_2d), + ], + ) + } +} + +fn extra_bind(device: &wgpu::Device, layout: &wgpu::BindGroupLayout, views: [&wgpu::TextureView; 3]) -> wgpu::BindGroup { + let entries: Vec = views + .iter() + .enumerate() + .map(|(binding, view)| wgpu::BindGroupEntry { + binding: binding as u32, + resource: wgpu::BindingResource::TextureView(view), + }) + .collect(); + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene extra"), + layout, + entries: &entries, + }) +} + +/// A growable uniform buffer holding one [`Uniform`] per draw of a render. +struct Uniforms { + buffer: wgpu::Buffer, + bind: wgpu::BindGroup, + /// Bytes between draws, rounded up to the device's dynamic-offset alignment. + stride: u64, + capacity: u64, +} + +impl Uniforms { + fn new(device: &wgpu::Device, layout: &wgpu::BindGroupLayout, stride: u64, capacity: u64) -> Self { + let buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("moq-scene draws"), + size: stride * capacity, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let bind = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene draws"), + layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { + buffer: &buffer, + offset: 0, + size: NonZeroU64::new(size_of::() as u64), + }), + }], + }); + Self { + buffer, + bind, + stride, + capacity, + } + } + + /// Grow to hold `count` draws, doubling so a growing scene reallocates rarely. + fn reserve(&mut self, device: &wgpu::Device, layout: &wgpu::BindGroupLayout, count: u64) { + if count > self.capacity { + *self = Self::new(device, layout, self.stride, count.next_power_of_two()); + } + } +} diff --git a/rs/moq-scene/src/effect.wgsl b/rs/moq-scene/src/effect.wgsl new file mode 100644 index 0000000..043d1ba --- /dev/null +++ b/rs/moq-scene/src/effect.wgsl @@ -0,0 +1,343 @@ +// Every draw the compositor makes: a quad sampling one picture, through a list +// of per-pixel filters, a blur, or a drop shadow. Colors are gamma-encoded +// sRGB, as CSS filters see them. Rasters between passes hold premultiplied +// alpha, so a blur weighs a transparent pixel as nothing. + +const OPS: u32 = 8u; + +// Kinds of `Op`. +const MATRIX: u32 = 0u; +const CHROMA_KEY: u32 = 1u; +const LUMA_KEY: u32 = 2u; +const LUT: u32 = 3u; +const MASK: u32 = 4u; + +// Rec. 709 luma. +const LUMA: vec3 = vec3(0.2126, 0.7152, 0.0722); + +// One per-pixel filter. +struct Op { + kind: vec4, + // MATRIX: unused. CHROMA_KEY: key Cb, key Cr, similarity, smoothness. + // LUMA_KEY: min, max, smoothness. LUT: amount, size. MASK: mode. + params: vec4, + // MATRIX: applied to straight RGBA. LUT: the domain's max in column 0. + matrix: mat4x4, + // MATRIX: added after. CHROMA_KEY: spill in x. LUT: the domain's min. + offset: vec4, +} + +struct Draw { + // The quad: top-left and bottom-right, in clip space. + rect: vec4, + // The input's texture coordinates at the quad's corners. + input: vec4, + // Where the input has a picture, in texture coordinates; zero outside. + valid: vec4, + // The item's own coordinates at the quad's corners, 0 to 1 over the item. + item: vec4, + // The aux texture's coordinates at the quad's corners. + aux: vec4, + // Ops in x, whether the input is premultiplied in y, the blend mode in z. + mode: vec4, + // A blur's step between taps in xy, its standard deviation in z, and its + // taps each side in w, all in texels. + blur: vec4, + // A shadow's straight color. + color: vec4, + ops: array, +} + +@group(0) @binding(0) var draw: Draw; +@group(1) @binding(0) var input: texture_2d; +@group(1) @binding(1) var bilinear: sampler; +@group(2) @binding(0) var lut: texture_3d; +@group(2) @binding(1) var mask: texture_2d; +// A drop shadow's blurred shape, or the backdrop a blend mode mixes with. +@group(2) @binding(2) var aux: texture_2d; + +struct Varyings { + @builtin(position) position: vec4, + @location(0) input: vec2, + @location(1) item: vec2, + @location(2) aux: vec2, +} + +// A four-vertex triangle strip covering the quad. +@vertex fn vs_main(@builtin(vertex_index) index: u32) -> Varyings { + let corner = vec2(f32(index & 1u), f32((index >> 1u) & 1u)); + var out: Varyings; + out.position = vec4(mix(draw.rect.xy, draw.rect.zw, corner), 0.0, 1.0); + out.input = mix(draw.input.xy, draw.input.zw, corner); + out.item = mix(draw.item.xy, draw.item.zw, corner); + out.aux = mix(draw.aux.xy, draw.aux.zw, corner); + return out; +} + +fn within(uv: vec2, low: vec2, high: vec2) -> bool { + return all(uv >= low) && all(uv <= high); +} + +// The input at `uv`, transparent outside its picture, as CSS's edge mode none. +fn fetch(uv: vec2) -> vec4 { + if !within(uv, draw.valid.xy, draw.valid.zw) { + return vec4(0.0); + } + return textureSampleLevel(input, bilinear, uv, 0.0); +} + +// The input at `uv`, with straight alpha. +fn straight(uv: vec2) -> vec4 { + let color = fetch(uv); + if draw.mode.y == 0u { + return color; + } + if color.a <= 0.0 { + return vec4(0.0); + } + return vec4(color.rgb / color.a, color.a); +} + +fn chroma_key(color: vec4, op: Op) -> vec4 { + let chroma = vec2( + dot(color.rgb, vec3(-0.1146, -0.3854, 0.5)), + dot(color.rgb, vec3(0.5, -0.4542, -0.0458)), + ); + let past = distance(chroma, op.params.xy) - op.params.z; + let alpha = pow(clamp(past / max(op.params.w, 1e-4), 0.0, 1.0), 1.5); + let kept = pow(clamp(past / max(op.offset.x, 1e-4), 0.0, 1.0), 1.5); + let gray = vec3(dot(color.rgb, LUMA)); + return vec4(mix(gray, color.rgb, kept), color.a * alpha); +} + +fn luma_key(color: vec4, op: Op) -> vec4 { + let luma = dot(color.rgb, LUMA); + let soft = max(op.params.z, 1e-4); + let low = clamp((luma - op.params.x + soft) / soft, 0.0, 1.0); + let high = clamp((op.params.y + soft - luma) / soft, 0.0, 1.0); + return vec4(color.rgb, color.a * low * high); +} + +// The table's color for `rgb`, interpolated between its eight nearest points. +fn lookup(rgb: vec3, op: Op) -> vec3 { + let size = i32(op.params.y); + let low = op.offset.xyz; + let high = op.matrix[0].xyz; + let at = clamp((rgb - low) / (high - low), vec3(0.0), vec3(1.0)) * f32(size - 1); + let base = min(vec3(floor(at)), vec3(size - 2)); + let fraction = at - vec3(base); + var out = vec3(0.0); + for (var corner = 0u; corner < 8u; corner++) { + let corner_step = vec3(corner & 1u, (corner >> 1u) & 1u, (corner >> 2u) & 1u); + let weights = select(1.0 - fraction, fraction, corner_step == vec3(1u)); + let weight = weights.x * weights.y * weights.z; + out += weight * textureLoad(lut, base + vec3(corner_step), 0).rgb; + } + return out; +} + +// The mask's coverage at `item`, stretched over the item and empty outside it. +fn coverage(item: vec2, op: Op) -> f32 { + if !within(item, vec2(0.0), vec2(1.0)) { + return 0.0; + } + let color = textureSampleLevel(mask, bilinear, item, 0.0); + if op.params.x > 0.5 { + return dot(color.rgb, LUMA) * color.a; + } + return color.a; +} + +// Run the draw's ops, in order, on a straight-alpha color. +fn apply(start: vec4, item: vec2) -> vec4 { + var color = start; + for (var index = 0u; index < draw.mode.x; index++) { + let op = draw.ops[index]; + switch op.kind.x { + case MATRIX: { + color = clamp(op.matrix * color + op.offset, vec4(0.0), vec4(1.0)); + } + case CHROMA_KEY: { + color = chroma_key(color, op); + } + case LUMA_KEY: { + color = luma_key(color, op); + } + case LUT: { + color = vec4(mix(color.rgb, lookup(color.rgb, op), op.params.x), color.a); + } + case MASK: { + color.a *= coverage(item, op); + } + default: {} + } + } + return color; +} + +// The input through the ops, premultiplied: drawn over a canvas or into a raster. +@fragment fn fs_effect(in: Varyings) -> @location(0) vec4 { + let color = apply(straight(in.input), in.item); + return vec4(color.rgb * color.a, color.a); +} + +fn hard_light(backdrop: f32, source: f32) -> f32 { + if source <= 0.5 { + return backdrop * 2.0 * source; + } + let screen = 2.0 * source - 1.0; + return backdrop + screen - backdrop * screen; +} + +fn soft_light(backdrop: f32, source: f32) -> f32 { + if source <= 0.5 { + return backdrop - (1.0 - 2.0 * source) * backdrop * (1.0 - backdrop); + } + var d = sqrt(backdrop); + if backdrop <= 0.25 { + d = ((16.0 * backdrop - 12.0) * backdrop + 4.0) * backdrop; + } + return backdrop + (2.0 * source - 1.0) * (d - backdrop); +} + +fn color_dodge(backdrop: f32, source: f32) -> f32 { + if backdrop <= 0.0 { + return 0.0; + } + if source >= 1.0 { + return 1.0; + } + return min(1.0, backdrop / (1.0 - source)); +} + +fn color_burn(backdrop: f32, source: f32) -> f32 { + if backdrop >= 1.0 { + return 1.0; + } + if source <= 0.0 { + return 0.0; + } + return 1.0 - min(1.0, (1.0 - backdrop) / source); +} + +// The non-separable modes' luminosity and saturation, from the Compositing +// and Blending spec. +const LUM: vec3 = vec3(0.3, 0.59, 0.11); + +fn clip_color(color: vec3) -> vec3 { + let l = dot(color, LUM); + let n = min(color.r, min(color.g, color.b)); + let x = max(color.r, max(color.g, color.b)); + var out = color; + if n < 0.0 { + out = l + (out - l) * l / (l - n); + } + if x > 1.0 { + out = l + (out - l) * (1.0 - l) / (x - l); + } + return out; +} + +fn set_lum(color: vec3, l: f32) -> vec3 { + return clip_color(color + (l - dot(color, LUM))); +} + +fn sat(color: vec3) -> f32 { + return max(color.r, max(color.g, color.b)) - min(color.r, min(color.g, color.b)); +} + +fn set_sat(color: vec3, s: f32) -> vec3 { + let low = min(color.r, min(color.g, color.b)); + let range = sat(color); + if range <= 0.0 { + return vec3(0.0); + } + return (color - low) * s / range; +} + +fn mix_blend(backdrop: vec3, source: vec3, mode: u32) -> vec3 { + switch mode { + case 1u: { return backdrop * source; } + case 2u: { return backdrop + source - backdrop * source; } + case 3u: { + return vec3( + hard_light(source.r, backdrop.r), + hard_light(source.g, backdrop.g), + hard_light(source.b, backdrop.b), + ); + } + case 4u: { return min(backdrop, source); } + case 5u: { return max(backdrop, source); } + case 6u: { + return vec3( + color_dodge(backdrop.r, source.r), + color_dodge(backdrop.g, source.g), + color_dodge(backdrop.b, source.b), + ); + } + case 7u: { + return vec3( + color_burn(backdrop.r, source.r), + color_burn(backdrop.g, source.g), + color_burn(backdrop.b, source.b), + ); + } + case 8u: { + return vec3( + hard_light(backdrop.r, source.r), + hard_light(backdrop.g, source.g), + hard_light(backdrop.b, source.b), + ); + } + case 9u: { + return vec3( + soft_light(backdrop.r, source.r), + soft_light(backdrop.g, source.g), + soft_light(backdrop.b, source.b), + ); + } + case 10u: { return abs(backdrop - source); } + case 11u: { return backdrop + source - 2.0 * backdrop * source; } + case 12u: { return set_lum(set_sat(source, sat(backdrop)), dot(backdrop, LUM)); } + case 13u: { return set_lum(set_sat(backdrop, sat(source)), dot(backdrop, LUM)); } + case 14u: { return set_lum(source, dot(backdrop, LUM)); } + case 15u: { return set_lum(backdrop, dot(source, LUM)); } + default: { return source; } + } +} + +// The input through the ops, mixed with the opaque backdrop in `aux` by the +// draw's blend mode: (1 - αs) Cb + αs B(Cb, Cs). +@fragment fn fs_blend(in: Varyings) -> @location(0) vec4 { + let source = apply(straight(in.input), in.item); + let backdrop = textureLoad(aux, vec2(in.position.xy), 0).rgb; + let mixed = mix_blend(backdrop, source.rgb, draw.mode.z); + return vec4(mix(backdrop, mixed, source.a), 1.0); +} + +// One direction of a separable Gaussian blur of a premultiplied raster. With +// no taps, one bilinear sample: a 2x2 average when halving a raster. +@fragment fn fs_blur(in: Varyings) -> @location(0) vec4 { + let sigma = draw.blur.z; + let taps = i32(draw.blur.w); + var sum = vec4(0.0); + var total = 0.0; + for (var tap = -taps; tap <= taps; tap++) { + let weight = exp(-f32(tap * tap) / (2.0 * sigma * sigma)); + sum += weight * fetch(in.input + f32(tap) * draw.blur.xy); + total += weight; + } + return sum / total; +} + +// The premultiplied input over its shadow: the blurred shape in `aux`, filled +// with the shadow's color. +@fragment fn fs_shadow(in: Varyings) -> @location(0) vec4 { + let picture = fetch(in.input); + var shape = 0.0; + if within(in.aux, vec2(0.0), vec2(1.0)) { + shape = textureSampleLevel(aux, bilinear, in.aux, 0.0).a; + } + let shadow = vec4(draw.color.rgb, 1.0) * draw.color.a * shape; + return picture + shadow * (1.0 - picture.a); +} diff --git a/rs/moq-scene/src/lib.rs b/rs/moq-scene/src/lib.rs index 1d2e26a..99f1be8 100644 --- a/rs/moq-scene/src/lib.rs +++ b/rs/moq-scene/src/lib.rs @@ -30,11 +30,15 @@ pub mod clip; mod collection; mod compositor; mod edit; +mod effect; +mod lut; pub mod publish; pub mod source; pub use clip::Clip; -pub use collection::{Audio, Canvas, Collection, Filter, Item, Layer, Rect, Rgba, Scene, Source, Transition, Video}; +pub use collection::{ + Audio, Blend, Canvas, Collection, Crop, Filter, Item, Layer, MaskMode, Rect, Rgba, Scene, Source, Transition, Video, +}; pub use compositor::{Compositor, FORMAT}; pub use edit::{Change, Draft, Edit, History}; @@ -73,6 +77,10 @@ pub enum Error { #[error("image: {0}")] Image(String), + /// A LUT source could not be read or parsed. + #[error("LUT: {0}")] + Lut(String), + /// This build or platform cannot open the source. #[error("unsupported source: {0}")] Unsupported(String), diff --git a/rs/moq-scene/src/lut.rs b/rs/moq-scene/src/lut.rs new file mode 100644 index 0000000..32092c0 --- /dev/null +++ b/rs/moq-scene/src/lut.rs @@ -0,0 +1,154 @@ +//! 3D color lookup tables, read from Adobe and Resolve `.cube` files. + +/// A 3D lookup table: an output color for each point of a cube of inputs. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Cube { + /// Points along each edge of the cube. + pub size: u32, + /// The input color at the cube's first corner. + pub min: [f32; 3], + /// The input color at the cube's last corner. + pub max: [f32; 3], + /// `size³` output colors, red varying fastest, then green, then blue. + pub table: Vec<[f32; 3]>, +} + +impl Cube { + /// The most points along an edge the format allows. + const MAX_SIZE: u32 = 256; + + /// Parse a `.cube` file. A 1D table is refused rather than misread as 3D. + pub fn parse(text: &str) -> Result { + let invalid = |line: usize, reason: &str| format!("line {}: {reason}", line + 1); + let mut size = None; + let mut min = [0.0; 3]; + let mut max = [1.0; 3]; + let mut table = Vec::new(); + + for (number, line) in text.lines().enumerate() { + let line = line.split('#').next().unwrap_or_default().trim(); + let words: Vec<&str> = line.split_whitespace().collect(); + let Some((&first, rest)) = words.split_first() else { continue }; + let floats = |words: &[&str]| { + words + .iter() + .map(|word| word.parse::().ok().filter(|value| value.is_finite())) + .collect::>>() + .ok_or_else(|| invalid(number, "expected numbers")) + }; + let keyword = first.starts_with(|c: char| c.is_ascii_alphabetic()); + if keyword && !table.is_empty() { + return Err(invalid(number, "a keyword after the table")); + } + match first { + "TITLE" => {} + "LUT_3D_SIZE" => { + let value = floats(rest)?; + let edge = match value[..] { + [edge] if edge.fract() == 0.0 && (2.0..=Self::MAX_SIZE as f32).contains(&edge) => edge as u32, + _ => return Err(invalid(number, "LUT_3D_SIZE must be from 2 to 256")), + }; + size = Some(edge); + } + "LUT_1D_SIZE" | "LUT_1D_INPUT_RANGE" => { + return Err(invalid(number, "1D LUTs are not supported; export a 3D one")); + } + "DOMAIN_MIN" | "DOMAIN_MAX" => { + let value: [f32; 3] = floats(rest)? + .try_into() + .map_err(|_| invalid(number, "expected three numbers"))?; + if first == "DOMAIN_MIN" { + min = value + } else { + max = value + } + } + "LUT_3D_INPUT_RANGE" => { + let [low, high] = floats(rest)?[..] else { + return Err(invalid(number, "expected two numbers")); + }; + min = [low; 3]; + max = [high; 3]; + } + _ if keyword => return Err(invalid(number, &format!("unknown keyword `{first}`"))), + _ => { + let Some(size) = size else { + return Err(invalid(number, "a table entry before LUT_3D_SIZE")); + }; + if table.len() == (size as usize).pow(3) { + return Err(invalid(number, "more entries than LUT_3D_SIZE allows")); + } + let value: [f32; 3] = floats(&words)? + .try_into() + .map_err(|_| invalid(number, "expected an R G B triple"))?; + table.push(value); + } + } + } + + let size = size.ok_or("no LUT_3D_SIZE")?; + let expected = (size as usize).pow(3); + if table.len() != expected { + return Err(format!( + "{} entries, but LUT_3D_SIZE {size} needs {expected}", + table.len() + )); + } + if (0..3).any(|channel| min[channel] >= max[channel]) { + return Err(format!("the domain {min:?} to {max:?} is empty")); + } + Ok(Self { size, min, max, table }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An identity table of size 2, as written by hand. + const IDENTITY: &str = "# made by hand +TITLE \"identity\" +LUT_3D_SIZE 2 +DOMAIN_MIN 0 0 0 +DOMAIN_MAX 1 1 1 + +0 0 0 +1 0 0 +0 1 0 +1 1 0 +0 0 1 +1 0 1 +0 1 1 +1 1 1 +"; + + #[test] + fn reads_a_table() { + let cube = Cube::parse(IDENTITY).unwrap(); + assert_eq!(cube.size, 2); + assert_eq!(cube.table[1], [1.0, 0.0, 0.0], "red varies fastest"); + assert_eq!(cube.table[2], [0.0, 1.0, 0.0]); + + let resolve = IDENTITY + .replace("DOMAIN_MIN 0 0 0\n", "") + .replace("DOMAIN_MAX 1 1 1", "LUT_3D_INPUT_RANGE 0 2"); + assert_eq!(Cube::parse(&resolve).unwrap().max, [2.0; 3]); + } + + /// A file that is not a whole 3D table fails rather than drawing a guess. + #[test] + fn refuses_what_it_cannot_read() { + let short = IDENTITY.replace("1 1 1\n", ""); + let long = format!("{IDENTITY}1 1 1\n"); + let late = format!("{IDENTITY}DOMAIN_MAX 2 2 2\n"); + let one = IDENTITY.replace("LUT_3D_SIZE 2", "LUT_1D_SIZE 2"); + let huge = IDENTITY.replace("LUT_3D_SIZE 2", "LUT_3D_SIZE 257"); + let pair = IDENTITY.replace("0 1 1", "0 1"); + let empty = IDENTITY.replace("DOMAIN_MAX 1 1 1", "DOMAIN_MAX 0 1 1"); + let unknown = IDENTITY.replace("TITLE", "LUT_FANCY 3\nTITLE"); + let nan = IDENTITY.replace("1 0 1", "1 NaN 1"); + for text in [&short, &long, &late, &one, &huge, &pair, &empty, &unknown, &nan, "hello"] { + assert!(Cube::parse(text).is_err(), "{text}"); + } + } +} diff --git a/rs/moq-scene/src/source.rs b/rs/moq-scene/src/source.rs index 0b26ba0..61a05cf 100644 --- a/rs/moq-scene/src/source.rs +++ b/rs/moq-scene/src/source.rs @@ -22,6 +22,7 @@ use moq_mux::catalog::Stream; use crate::Error; use crate::collection::{Collection, Source}; +use crate::lut::Cube; use crate::publish::{Claim, Listing, Publish, Still}; /// A source's newest picture, waiting for the compositor to draw it. @@ -30,6 +31,8 @@ pub(crate) enum Picture { Frame(moq_video::Frame), /// Tightly packed RGBA with straight alpha, such as a still image. Rgba { size: moq_video::Size, pixels: Vec }, + /// A LUT source's table, which is not drawn but looked up. + Lut(Cube), } /// The one picture a source holds for the compositor. A newer picture replaces @@ -69,7 +72,7 @@ pub struct Context { pub bus: Bus, /// Where broadcast sources are announced: the studio's relay connection. pub origin: moq_net::origin::Consumer, - /// What a relative image path is relative to: the scene file's directory. + /// What a relative image or LUT path is relative to: the scene file's directory. pub dir: PathBuf, /// What cameras, screens, and broadcasts deliver video as: /// `moq_video::render::output` of the compositor's device. @@ -292,6 +295,14 @@ impl Driver { let _still = listing.and_then(|listing| self.publish(Still::new(listing, bytes))); std::future::pending().await } + Source::Lut { path } => { + let path = self.context.dir.join(path); + let failed = |err: String| Error::Lut(format!("{}: {err}", path.display())); + let text = std::fs::read_to_string(&path).map_err(|err| failed(err.to_string()))?; + let cube = Cube::parse(&text).map_err(failed)?; + self.show(Picture::Lut(cube)); + std::future::pending().await + } Source::Broadcast { path } => self.broadcast(path).await, Source::Clone { .. } => unreachable!("a clone opens no feed"), device => self.device(device, listing).await, @@ -332,7 +343,11 @@ impl Driver { Source::Window { id } => Video::Window(id.clone()), Source::Microphone { id } => return self.listen(Audio::Microphone(id.clone())).await, Source::SystemAudio => return self.listen(Audio::System).await, - Source::Color { .. } | Source::Image { .. } | Source::Broadcast { .. } | Source::Clone { .. } => { + Source::Color { .. } + | Source::Image { .. } + | Source::Broadcast { .. } + | Source::Clone { .. } + | Source::Lut { .. } => { unreachable!("not a device source") } }; From e67336557ddfc45d571157bc3cdf72f9583d6029 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:15:09 -0800 Subject: [PATCH 3/5] feat(studio): the Filters panel edits crops, every filter, and blend modes Co-Authored-By: Claude Opus 5.5 --- rs/moq-scene/src/clip.rs | 21 +- rs/moq-scene/src/collection.rs | 17 +- rs/moq-scene/src/compositor.rs | 6 +- rs/moq-scene/src/effect.rs | 16 +- rs/moq-scene/src/lut.rs | 8 +- rs/moq-studio/src/engine.rs | 9 +- rs/moq-studio/src/ui.rs | 403 +++++++++++++++++++++++++++++---- 7 files changed, 417 insertions(+), 63 deletions(-) diff --git a/rs/moq-scene/src/clip.rs b/rs/moq-scene/src/clip.rs index 6e5933c..7abee57 100644 --- a/rs/moq-scene/src/clip.rs +++ b/rs/moq-scene/src/clip.rs @@ -194,7 +194,13 @@ impl Draft { } if let Some(video) = items.video.get(name) { let filters = rename(&video.filters); - self.set_video(&renamed, Some(Video { filters, ..video.clone() }))?; + self.set_video( + &renamed, + Some(Video { + filters, + ..video.clone() + }), + )?; } } for item in &items.items { @@ -362,10 +368,15 @@ mod tests { mode: MaskMode::Luminance, }; collection.scenes[0].items[0].filters = vec![mask("matte")]; - collection.video.get_mut("cam faded").unwrap().filters.push(Filter::Lut { - source: "grade".into(), - amount: 0.5, - }); + collection + .video + .get_mut("cam faded") + .unwrap() + .filters + .push(Filter::Lut { + source: "grade".into(), + amount: 0.5, + }); collection.validate().unwrap(); let text = Clip::items(&collection, collection.scenes[0].items.clone(), Path::new(DIR)) .to_toml() diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs index 9feb456..aa6e52b 100644 --- a/rs/moq-scene/src/collection.rs +++ b/rs/moq-scene/src/collection.rs @@ -156,7 +156,9 @@ impl Collection { } item.rect.validate(&layer, &item.source)?; let on = format!("`{}` in {layer}", item.source); - item.crop.validate().map_err(|err| Error::Invalid(format!("{on}: {err}")))?; + item.crop + .validate() + .map_err(|err| Error::Invalid(format!("{on}: {err}")))?; self.validate_chain(&item.filters, &on)?; } } @@ -188,7 +190,10 @@ impl Collection { ))); } let on = format!("`{name}`"); - video.crop.validate().map_err(|err| Error::Invalid(format!("{on}: {err}")))?; + video + .crop + .validate() + .map_err(|err| Error::Invalid(format!("{on}: {err}")))?; self.validate_chain(&video.filters, &on)?; } @@ -1217,7 +1222,13 @@ rect = { x = 0, y = 680, width = 1280, height = 40 } collection.scenes[0].items[0].filters = vec![lut("grade"), mask("logo")]; collection.validate().unwrap(); - for filter in [lut("logo"), lut("nobody"), mask("grade"), mask("mic"), mask("camera faded")] { + for filter in [ + lut("logo"), + lut("nobody"), + mask("grade"), + mask("mic"), + mask("camera faded"), + ] { let mut collection = collection.clone(); collection.video.insert( "camera".into(), diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index afef8ce..07ab6fc 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -951,7 +951,11 @@ mod tests { // The share of a Gaussian of deviation 10, centered half a pixel // inside an edge, that falls inside: Φ(0.05). let edge = 0.52; - let cases = [((32, 32), 255.0), ((0, 32), edge * 255.0), ((0, 0), edge * edge * 255.0)]; + let cases = [ + ((32, 32), 255.0), + ((0, 32), edge * 255.0), + ((0, 0), edge * edge * 255.0), + ]; for ((x, y), expected) in cases { let actual = f32::from(at(x, y)); assert!((actual - expected).abs() <= 4.0, "({x}, {y}): {actual} vs {expected}"); diff --git a/rs/moq-scene/src/effect.rs b/rs/moq-scene/src/effect.rs index 3ceb6ab..169aae6 100644 --- a/rs/moq-scene/src/effect.rs +++ b/rs/moq-scene/src/effect.rs @@ -1022,7 +1022,15 @@ impl Effects { } /// `input` through `segment`, blurred by `sigma` canvas pixels over `area`. - fn blur(&mut self, frame: &mut Frame, input: &Input, segment: &Segment, item: Area, area: Area, sigma: f32) -> Input { + fn blur( + &mut self, + frame: &mut Frame, + input: &Input, + segment: &Segment, + item: Area, + area: Area, + sigma: f32, + ) -> Input { let mut current = self.raster(frame, input, segment, item, area); let mut size = self.texels(&area); let texels = |size: (u32, u32)| (size.0 as f32 / area.width(), size.1 as f32 / area.height()); @@ -1179,7 +1187,11 @@ impl Effects { } } -fn extra_bind(device: &wgpu::Device, layout: &wgpu::BindGroupLayout, views: [&wgpu::TextureView; 3]) -> wgpu::BindGroup { +fn extra_bind( + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + views: [&wgpu::TextureView; 3], +) -> wgpu::BindGroup { let entries: Vec = views .iter() .enumerate() diff --git a/rs/moq-scene/src/lut.rs b/rs/moq-scene/src/lut.rs index 32092c0..369a989 100644 --- a/rs/moq-scene/src/lut.rs +++ b/rs/moq-scene/src/lut.rs @@ -28,7 +28,9 @@ impl Cube { for (number, line) in text.lines().enumerate() { let line = line.split('#').next().unwrap_or_default().trim(); let words: Vec<&str> = line.split_whitespace().collect(); - let Some((&first, rest)) = words.split_first() else { continue }; + let Some((&first, rest)) = words.split_first() else { + continue; + }; let floats = |words: &[&str]| { words .iter() @@ -147,7 +149,9 @@ DOMAIN_MAX 1 1 1 let empty = IDENTITY.replace("DOMAIN_MAX 1 1 1", "DOMAIN_MAX 0 1 1"); let unknown = IDENTITY.replace("TITLE", "LUT_FANCY 3\nTITLE"); let nan = IDENTITY.replace("1 0 1", "1 NaN 1"); - for text in [&short, &long, &late, &one, &huge, &pair, &empty, &unknown, &nan, "hello"] { + for text in [ + &short, &long, &late, &one, &huge, &pair, &empty, &unknown, &nan, "hello", + ] { assert!(Cube::parse(text).is_err(), "{text}"); } } diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs index d4d466b..2192c6c 100644 --- a/rs/moq-studio/src/engine.rs +++ b/rs/moq-studio/src/engine.rs @@ -826,12 +826,9 @@ mod tests { color: moq_scene::Rgba([255, 0, 0, 255]), }, ); - collection.scenes[0].items.push(moq_scene::Item { - source: "red".into(), - rect: moq_scene::Rect::fill(&collection.canvas), - visible: true, - filters: Vec::new(), - }); + collection.scenes[0] + .items + .push(moq_scene::Item::new("red", moq_scene::Rect::fill(&collection.canvas))); let gpu = Gpu::new().await.expect("a GPU"); let (autosave, _) = Autosave::open(dir.path().join("show.toml")).unwrap(); diff --git a/rs/moq-studio/src/ui.rs b/rs/moq-studio/src/ui.rs index 7233fcb..f7cadd5 100644 --- a/rs/moq-studio/src/ui.rs +++ b/rs/moq-studio/src/ui.rs @@ -8,7 +8,9 @@ use anyhow::Context; use egui::{Color32, Ui}; use egui_dock::{DockArea, DockState, NodeIndex, Style, TabViewer}; use moq_scene::source::{Feed, Status}; -use moq_scene::{Clip, Collection, Draft, Filter, Item, Layer, Rect, Scene, Source, Video, wgpu}; +use moq_scene::{ + Blend, Clip, Collection, Crop, Draft, Filter, Item, Layer, MaskMode, Rect, Scene, Source, Video, wgpu, +}; use crate::engine::{Engine, State}; use crate::health::{self, Verdict}; @@ -182,21 +184,48 @@ enum Selection { Item(Layer, usize), } +/// What the Filters panel sets on a selection. +#[derive(Clone, Debug, PartialEq)] +struct Look { + crop: Crop, + filters: Vec, + /// An item's blend mode; a source has none. + blend: Option, +} + impl Selection { - /// The filters of what is selected, or `None` if it is gone or has no picture. - fn filters(&self, collection: &Collection) -> Option> { + /// The look of what is selected, or `None` if it is gone or has no picture. + fn look(&self, collection: &Collection) -> Option { match self { Self::Source(name) => { collection.sources.get(name).filter(|source| source.has_video())?; - Some(collection.video.get(name).cloned().unwrap_or_default().filters) + let video = collection.video.get(name).cloned().unwrap_or_default(); + Some(Look { + crop: video.crop, + filters: video.filters, + blend: None, + }) + } + Self::Item(layer, index) => { + let item = collection.items(layer)?.get(*index)?; + Some(Look { + crop: item.crop, + filters: item.filters.clone(), + blend: Some(item.blend), + }) } - Self::Item(layer, index) => Some(collection.items(layer)?.get(*index)?.filters.clone()), } } - fn set_filters(&self, draft: &mut Draft, filters: Vec) -> Result<(), moq_scene::Error> { + /// The filters of what is selected, or `None` if it is gone or has no picture. + fn filters(&self, collection: &Collection) -> Option> { + self.look(collection).map(|look| look.filters) + } + + fn set_look(&self, draft: &mut Draft, look: Look) -> Result<(), moq_scene::Error> { + let Look { crop, filters, blend } = look; match self { - Self::Source(name) => draft.set_video(name, Some(Video { filters })), + Self::Source(name) => draft.set_video(name, Some(Video { crop, filters })), Self::Item(layer, index) => { let item = draft .collection() @@ -205,11 +234,28 @@ impl Selection { .cloned(); let item = item.ok_or_else(|| moq_scene::Error::Conflict(format!("there is no item {index} in {layer}")))?; - draft.set_item(layer, *index, Item { filters, ..item }) + let blend = blend.unwrap_or(item.blend); + draft.set_item( + layer, + *index, + Item { + crop, + filters, + blend, + ..item + }, + ) } } } + fn set_filters(&self, draft: &mut Draft, filters: Vec) -> Result<(), moq_scene::Error> { + let look = self + .look(draft.collection()) + .ok_or_else(|| moq_scene::Error::Conflict("the selection is gone".into()))?; + self.set_look(draft, Look { filters, ..look }) + } + fn describe(&self, collection: &Collection) -> String { match self { Self::Source(name) => format!("source `{name}`"), @@ -889,7 +935,7 @@ impl Panels { self.report(result.map(|name| format!("added `{name}`"))); } - /// The filters of the selected source or item, in order. + /// The crop, filters in order, and blend mode of the selected source or item. fn filters(&mut self, ui: &mut Ui) { let engine = self.engine.clone(); let mut state = engine.state.lock().unwrap(); @@ -899,7 +945,7 @@ impl Panels { return; }; let what = selection.describe(collection); - let Some(filters) = selection.filters(collection) else { + let Some(look) = selection.look(collection) else { ui.label(format!("{what} has no picture to filter.")); return; }; @@ -911,53 +957,107 @@ impl Panels { let source = video.filters.iter().map(describe_filter).collect::>(); ui.weak(format!("after `{}`'s own: {}", item.source, source.join(", "))); } - - let mut next = filters.clone(); + // What a mask can show, a picture of its own, and what a LUT reads. + let pictures: Vec = collection + .sources + .iter() + .filter(|(_, source)| source.has_video() && source.opens()) + .map(|(name, _)| name.clone()) + .collect(); + let luts: Vec = collection + .sources + .iter() + .filter(|(_, source)| matches!(source, Source::Lut { .. })) + .map(|(name, _)| name.clone()) + .collect(); + + let mut next = look.clone(); let mut action: Option<(String, Option)> = None; - let count = next.len(); - let mut moved = None; - let mut remove = None; - for (index, filter) in next.iter_mut().enumerate() { + egui::Grid::new("look").show(ui, |ui| { + ui.label("Crop"); ui.horizontal(|ui| { - match filter { - Filter::Opacity { amount } => { - ui.label("Opacity"); - let response = ui.add(egui::Slider::new(amount, 0.0..=1.0)); - let gesture = self.gestures.of(&response); - if response.changed() { - action = Some((format!("Set opacity on {what}"), gesture)); - } + let crop = &mut next.crop; + for (label, edge) in [ + ("left", &mut crop.left), + ("top", &mut crop.top), + ("right", &mut crop.right), + ("bottom", &mut crop.bottom), + ] { + ui.weak(label); + let response = ui.add(egui::DragValue::new(edge).range(0.0..=0.95).speed(0.002)); + if let Some(gesture) = changed(&response, &mut self.gestures) { + action = Some((format!("Crop {what}"), gesture)); } } - if ui.add_enabled(index > 0, egui::Button::new("Up").small()).clicked() { - moved = Some((index, index - 1)); - } - if ui - .add_enabled(index + 1 < count, egui::Button::new("Down").small()) - .clicked() - { - moved = Some((index, index + 1)); - } - if ui.small_button("Remove").clicked() { - remove = Some(index); + }) + .response + .on_hover_text("The fraction of the picture trimmed from each edge, the rest stretched to fill"); + ui.end_row(); + if let Some(blend) = &mut next.blend { + ui.label("Blend"); + let before = *blend; + egui::ComboBox::from_id_salt("blend") + .selected_text(blend_name(*blend)) + .show_ui(ui, |ui| { + for mode in Blend::ALL { + ui.selectable_value(blend, mode, blend_name(mode)); + } + }); + if *blend != before { + action = Some((format!("Set the blend mode of {what}"), None)); } + ui.end_row(); + } + }); + + let count = next.filters.len(); + let mut moved = None; + let mut remove = None; + for (index, filter) in next.filters.iter_mut().enumerate() { + ui.push_id(index, |ui| { + ui.horizontal_wrapped(|ui| { + if let Some(gesture) = filter_ui(ui, filter, &pictures, &luts, &mut self.gestures) { + action = Some((format!("Set {} on {what}", filter_name(filter)), gesture)); + } + if ui.add_enabled(index > 0, egui::Button::new("Up").small()).clicked() { + moved = Some((index, index - 1)); + } + if ui + .add_enabled(index + 1 < count, egui::Button::new("Down").small()) + .clicked() + { + moved = Some((index, index + 1)); + } + if ui.small_button("Remove").clicked() { + remove = Some(index); + } + }); }); } if let Some((from, to)) = moved { - next.swap(from, to); + next.filters.swap(from, to); action = Some((format!("Reorder filters on {what}"), None)); } if let Some(index) = remove { - next.remove(index); + next.filters.remove(index); action = Some((format!("Remove a filter from {what}"), None)); } ui.separator(); + let filters = look.filters; ui.horizontal(|ui| { - if ui.button("Add opacity").clicked() { - next.push(Filter::Opacity { amount: 1.0 }); - action = Some((format!("Add opacity to {what}"), None)); - } + egui::ComboBox::from_id_salt("add filter") + .selected_text("Add filter") + .show_ui(ui, |ui| { + for filter in templates(&pictures, &luts) { + if ui.selectable_label(false, filter_name(&filter)).clicked() { + action = Some((format!("Add {} to {what}", filter_name(&filter)), None)); + next.filters.push(filter); + } + } + }) + .response + .on_hover_text("A LUT needs a LUT source, and a mask a source with a picture of its own"); if ui.add_enabled(!filters.is_empty(), egui::Button::new("Copy")).clicked() { let clip = Clip::Filters(moq_scene::clip::Filters { filters: filters.clone(), @@ -977,7 +1077,7 @@ impl Panels { }); let Some((label, gesture)) = action else { return }; - let result = edit(&mut state, label, gesture, |draft| selection.set_filters(draft, next)); + let result = edit(&mut state, label, gesture, |draft| selection.set_look(draft, next)); drop(state); self.check(result); } @@ -1264,9 +1364,218 @@ fn linked(ui: &mut Ui, collection: &Collection, layer: &Layer, index: usize, ite } fn describe_filter(filter: &Filter) -> String { + let name = filter_name(filter); + match filter { + Filter::Opacity { amount } + | Filter::Brightness { amount } + | Filter::Contrast { amount } + | Filter::Saturate { amount } + | Filter::Grayscale { amount } + | Filter::Sepia { amount } + | Filter::Invert { amount } => format!("{name} {:.0}%", amount * 100.0), + Filter::HueRotate { degrees } => format!("{name} {degrees:.0}°"), + Filter::Blur { radius } => format!("{name} {radius:.1} px"), + Filter::Lut { source, .. } | Filter::Mask { source, .. } => format!("{name} `{source}`"), + _ => name.to_string(), + } +} + +fn filter_name(filter: &Filter) -> &'static str { + match filter { + Filter::Opacity { .. } => "opacity", + Filter::Brightness { .. } => "brightness", + Filter::Contrast { .. } => "contrast", + Filter::Saturate { .. } => "saturation", + Filter::Grayscale { .. } => "grayscale", + Filter::Sepia { .. } => "sepia", + Filter::Invert { .. } => "invert", + Filter::HueRotate { .. } => "hue rotation", + Filter::Blur { .. } => "blur", + Filter::DropShadow { .. } => "drop shadow", + Filter::ChromaKey { .. } => "chroma key", + Filter::LumaKey { .. } => "luma key", + Filter::Lut { .. } => "LUT", + Filter::Mask { .. } => "mask", + } +} + +fn blend_name(blend: Blend) -> &'static str { + match blend { + Blend::Normal => "normal", + Blend::Multiply => "multiply", + Blend::Screen => "screen", + Blend::Overlay => "overlay", + Blend::Darken => "darken", + Blend::Lighten => "lighten", + Blend::ColorDodge => "color dodge", + Blend::ColorBurn => "color burn", + Blend::HardLight => "hard light", + Blend::SoftLight => "soft light", + Blend::Difference => "difference", + Blend::Exclusion => "exclusion", + Blend::Hue => "hue", + Blend::Saturation => "saturation", + Blend::Color => "color", + Blend::Luminosity => "luminosity", + } +} + +/// A new filter of each kind, as the Add menu offers it: one that leaves the +/// picture as it is where the kind allows. A LUT or mask names the first +/// source it can read, and is left out without one. +fn templates(pictures: &[String], luts: &[String]) -> Vec { + let mut filters = vec![ + Filter::Opacity { amount: 1.0 }, + Filter::Brightness { amount: 1.0 }, + Filter::Contrast { amount: 1.0 }, + Filter::Saturate { amount: 1.0 }, + Filter::Grayscale { amount: 0.0 }, + Filter::Sepia { amount: 0.0 }, + Filter::Invert { amount: 0.0 }, + Filter::HueRotate { degrees: 0.0 }, + Filter::Blur { radius: 4.0 }, + Filter::DropShadow { + x: 4.0, + y: 4.0, + blur: 4.0, + color: moq_scene::Rgba([0, 0, 0, 128]), + }, + Filter::ChromaKey { + color: moq_scene::Rgba([0, 255, 0, 255]), + similarity: 0.4, + smoothness: 0.08, + spill: 0.1, + }, + Filter::LumaKey { + min: 0.0, + max: 1.0, + smoothness: 0.05, + }, + ]; + if let Some(source) = luts.first() { + filters.push(Filter::Lut { + source: source.clone(), + amount: 1.0, + }); + } + if let Some(source) = pictures.first() { + filters.push(Filter::Mask { + source: source.clone(), + mode: MaskMode::Alpha, + }); + } + filters +} + +/// The gesture an edit through `response` belongs to, if it changed anything. +fn changed(response: &egui::Response, gestures: &mut Gestures) -> Option> { + let gesture = gestures.of(response); + response.changed().then_some(gesture) +} + +/// Edit `filter`'s settings; the gesture of the edit, if it made one. +fn filter_ui( + ui: &mut Ui, + filter: &mut Filter, + pictures: &[String], + luts: &[String], + gestures: &mut Gestures, +) -> Option> { + ui.label(filter_name(filter)); + let mut edits = Vec::new(); + let slider = |ui: &mut Ui, value: &mut f32, range| ui.add(egui::Slider::new(value, range)); match filter { - Filter::Opacity { amount } => format!("opacity {:.0}%", amount * 100.0), + Filter::Opacity { amount } + | Filter::Grayscale { amount } + | Filter::Sepia { amount } + | Filter::Invert { amount } => { + edits.push(slider(ui, amount, 0.0..=1.0)); + } + Filter::Brightness { amount } | Filter::Contrast { amount } | Filter::Saturate { amount } => { + edits.push(slider(ui, amount, 0.0..=3.0)); + } + Filter::HueRotate { degrees } => edits.push(ui.add(egui::Slider::new(degrees, -180.0..=180.0).suffix("°"))), + Filter::Blur { radius } => { + edits.push(ui.add(egui::Slider::new(radius, 0.0..=100.0).logarithmic(true).suffix(" px"))); + } + Filter::DropShadow { x, y, blur, color } => { + edits.push(ui.add(egui::DragValue::new(x).prefix("x ").suffix(" px"))); + edits.push(ui.add(egui::DragValue::new(y).prefix("y ").suffix(" px"))); + edits.push(ui.add(egui::Slider::new(blur, 0.0..=50.0).suffix(" px"))); + edits.push(color_ui(ui, color)); + } + Filter::ChromaKey { + color, + similarity, + smoothness, + spill, + } => { + edits.push(color_ui(ui, color)); + for (label, value) in [("similarity", similarity), ("smoothness", smoothness), ("spill", spill)] { + ui.weak(label); + edits.push(slider(ui, value, 0.0..=1.0)); + } + } + Filter::LumaKey { min, max, smoothness } => { + for (label, value) in [("min", &mut *min), ("max", &mut *max), ("smoothness", smoothness)] { + ui.weak(label); + edits.push(slider(ui, value, 0.0..=1.0)); + } + // Dragging one end past the other moves both. + *max = max.max(*min); + } + Filter::Lut { source, amount } => { + edits.push(pick(ui, "lut", source, luts)); + edits.push(slider(ui, amount, 0.0..=1.0)); + } + Filter::Mask { source, mode } => { + edits.push(pick(ui, "mask", source, pictures)); + let before = *mode; + let mut response = egui::ComboBox::from_id_salt("mode") + .selected_text(match mode { + MaskMode::Alpha => "alpha", + MaskMode::Luminance => "luminance", + }) + .show_ui(ui, |ui| { + ui.selectable_value(mode, MaskMode::Alpha, "alpha"); + ui.selectable_value(mode, MaskMode::Luminance, "luminance"); + }) + .response; + if *mode != before { + response.mark_changed(); + } + edits.push(response); + } + } + edits.iter().find_map(|response| changed(response, gestures)) +} + +/// Pick one of `names` for `value`. +fn pick(ui: &mut Ui, id: &str, value: &mut String, names: &[String]) -> egui::Response { + let before = value.clone(); + let mut response = egui::ComboBox::from_id_salt(id) + .selected_text(value.as_str()) + .show_ui(ui, |ui| { + for name in names { + ui.selectable_value(value, name.clone(), name); + } + }) + .response; + if *value != before { + response.mark_changed(); + } + response +} + +/// Edit `color` with straight alpha. +fn color_ui(ui: &mut Ui, color: &mut moq_scene::Rgba) -> egui::Response { + let [r, g, b, a] = color.0; + let mut edited = Color32::from_rgba_unmultiplied(r, g, b, a); + let response = egui::color_picker::color_edit_button_srgba(ui, &mut edited, egui::color_picker::Alpha::OnlyBlend); + if response.changed() { + *color = moq_scene::Rgba(edited.to_srgba_unmultiplied()); } + response } /// Add `source` as `name`, with `video` filters, and place it in front of @@ -1320,6 +1629,7 @@ fn describe(source: &Source) -> String { Source::Color { color } => format!("color: {color}"), Source::Broadcast { path } => format!("broadcast: {path}"), Source::Clone { source } => format!("clone of {source}"), + Source::Lut { path } => format!("LUT: {}", path.display()), } } @@ -1395,10 +1705,11 @@ enum Kind { Window, Microphone, SystemAudio, + Lut, } impl Kind { - const ALL: [Self; 8] = [ + const ALL: [Self; 9] = [ Self::Color, Self::Image, Self::Broadcast, @@ -1407,6 +1718,7 @@ impl Kind { Self::Window, Self::Microphone, Self::SystemAudio, + Self::Lut, ]; fn label(self) -> &'static str { @@ -1419,6 +1731,7 @@ impl Kind { Self::Window => "Window", Self::Microphone => "Microphone", Self::SystemAudio => "System audio", + Self::Lut => "LUT", } } @@ -1427,6 +1740,7 @@ impl Kind { match self { Self::Color | Self::SystemAudio => None, Self::Image => Some("file"), + Self::Lut => Some(".cube file"), Self::Broadcast => Some("path"), Self::Window => Some("window id"), Self::Camera | Self::Display | Self::Microphone => Some("id (empty for the default)"), @@ -1475,6 +1789,7 @@ impl AddSource { color: moq_scene::Rgba(self.color.to_srgba_unmultiplied()), }, Kind::Image => Source::Image { path: value.into() }, + Kind::Lut => Source::Lut { path: value.into() }, Kind::Broadcast => Source::Broadcast { path: value }, Kind::Camera => Source::Camera { id }, Kind::Display => Source::Display { id }, From afe9e8ef7f66d1f33fe33a3aa3fc8606d228637d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:16:02 -0800 Subject: [PATCH 4/5] docs: effects in the README; the quest keeps its ML effects Co-Authored-By: Claude Opus 5.5 --- README.md | 49 +++++++++++++++++++++++++++++++++----- quest/m0/README.md | 2 +- quest/m0/compose/layout.md | 5 +++- quest/m0/effects.md | 36 +++++++++++++++------------- 4 files changed, 67 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 36d3391..1cd8fb6 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Every panel is a tab you can drag, split, or stack. | Items | Place, show, hide, and order what the preview scene or the top layer draws, back to front. Select an item to filter or copy it. | | Sources | Add a source by hand, see whether each and its own track are live and where it is shown, clone one, and remove one. Select a source to filter it. | | Devices | List this machine's cameras, displays, windows, and audio inputs, and add one as a source. | -| Filters | Add, order, and set the filters of the selected source or item, and copy or paste them. | +| Filters | Crop the selected source or item, add, order, and set its filters, set an item's blend mode, and copy or paste filters. | | Audio | Set each audio source's gain and mute, and watch its meter and the program's. | | Output | See the broadcast path, whether the program is encoding, and the encoder's work. | | Health | See how the stream reaches viewers, track by track, and the cause of each problem. | @@ -107,11 +107,23 @@ source = "camera" [video.camera-faded] filters = [{ kind = "opacity", amount = 0.5 }] +[sources.grade] +kind = "lut" +path = "teal-orange.cube" + +[video.camera] +crop = { left = 0.1, right = 0.1 } # trim 10% of the picture off each side +filters = [ + { kind = "chroma_key", color = "#00ff00", similarity = 0.4, smoothness = 0.08, spill = 0.1 }, + { kind = "lut", source = "grade", amount = 0.8 }, +] + [[scenes]] name = "Main" items = [ { source = "camera", rect = { x = 0, y = 0, width = 1920, height = 1080 } }, - { source = "guest", rect = { x = 1280, y = 720, width = 640, height = 360 }, filters = [{ kind = "opacity", amount = 0.8 }] }, + { source = "guest", rect = { x = 1280, y = 720, width = 640, height = 360 }, filters = [{ kind = "drop_shadow", x = 8, y = 8, blur = 12, color = "#00000080" }] }, + { source = "logo", rect = { x = 0, y = 0, width = 1920, height = 1080 }, blend = "multiply", filters = [{ kind = "opacity", amount = 0.3 }] }, ] [[scenes]] @@ -128,10 +140,35 @@ rect = { x = 1600, y = 40, width = 280, height = 120 } Source kinds are `camera`, `display`, and `microphone` (each with an optional `id`, the default device without one), `window` (with an `id`), `system_audio` (macOS), `image` (PNG or JPEG), `color` (`#rrggbb` or `#rrggbbaa`), -`broadcast`, and `clone` (with the `source` it shows). The Devices panel fills -in ids. `[video.]` lists a source's filters, and an item's `filters` -its own; the one filter so far is `opacity`, from 0 to 1, as in CSS. `[[top]]` -items make up the top layer. +`broadcast`, `clone` (with the `source` it shows), and `lut` (a 3D Adobe or +Resolve `.cube` file, which shows nothing itself but grades what a `lut` filter +names it in). The Devices panel fills in ids. `[[top]]` items make up the top +layer. + +`[video.]` sets a source's `crop` and `filters`, wherever it is shown, +and an item's `crop`, `filters`, and `blend` its own. A `crop` trims a fraction +(0 to 1) of the picture off its `left`, `top`, `right`, and `bottom` and +stretches the rest over the item; an item's crop trims what its source's +leaves. Filters apply in order, the source's then the item's, with lengths in +canvas pixels: + +| Filter | Settings | +| --- | --- | +| `opacity`, `grayscale`, `sepia`, `invert` | `amount`, 0 to 1, as in CSS | +| `brightness`, `contrast`, `saturate` | `amount`, 0 or more, 1 leaving the picture as it is, as in CSS | +| `hue_rotate` | `degrees`, as in CSS | +| `blur` | `radius`, the standard deviation; the edges fade out, as in CSS | +| `drop_shadow` | `x`, `y`, `blur` (the standard deviation), and `color`, as in CSS | +| `chroma_key` | `color` keyed out, and `similarity`, `smoothness`, and `spill` from 0 to 1 | +| `luma_key` | `min` and `max` luma kept, and `smoothness`, from 0 to 1 | +| `lut` | the `lut` `source` to grade through, and `amount` from 0 to 1 | +| `mask` | a `source` whose picture, stretched over the item, keeps what it covers: by its alpha, or `mode = "luminance"` | + +An item's `blend` mixes it with what is under it by a CSS `mix-blend-mode`: +`multiply`, `screen`, `overlay`, `darken`, `lighten`, `color_dodge`, +`color_burn`, `hard_light`, `soft_light`, `difference`, `exclusion`, `hue`, +`saturation`, `color`, or `luminosity`. Removing a source removes the filters +that name it. The studio writes the file a second after the last change, so a burst such as a slider drag is one write, and once more on quit. Edits that never pause for a diff --git a/quest/m0/README.md b/quest/m0/README.md index e39b046..90521fa 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -90,7 +90,7 @@ repository. - [Instant replay and DVR](/quest/m0/replay.md) - replays and clips from held groups, and viewer scrubbing through the archive - [HTML/CSS overlays](/quest/m0/overlays.md) - script-free HTML and CSS graphics through Blitz natively and the DOM in the browser - [Animation and transitions](/quest/m0/animation.md) - move, slide, wipe, and stinger transitions, and keyframed item properties -- [Video effects](/quest/m0/effects.md) - a curated effect set with CSS meanings, and ML effects baked into their source +- [ML video effects](/quest/m0/effects.md) - background removal and other ML effects, baked into their source's track - [Snapping and guides](/quest/m0/snapping.md) - preview moves snap to edges, centers, and other items - [Guided setup](/quest/m0/guided-setup.md) - first launch detects devices, measures the uplink, and proposes settings - [Destinations](/quest/m0/destinations.md) - RTMP, RTMPS, and WHIP push with per-destination codecs and shared encoders, on a UI-free egress library a hosted service can reuse diff --git a/quest/m0/compose/layout.md b/quest/m0/compose/layout.md index 252a833..954d96e 100644 --- a/quest/m0/compose/layout.md +++ b/quest/m0/compose/layout.md @@ -25,7 +25,10 @@ catalog (decided 2026-11-02, maintainer; `rs/moq-scene/src/publish.rs`). Clones and filters travel in the layout, not as tracks: a source's track is its capture before any filter, and a clone publishes none, so the layout names the source a clone shows and the filters each source and item applies -(decided 2026-11-02, maintainer). +(decided 2026-11-02, maintainer). That includes crops and blend modes, and the +filters naming another source: a mask names a picture source's track, and a +LUT a `lut` source, whose `.cube` table publishes no track yet, so a viewer +needs one carrying it, as an image's file is. ## Required diff --git a/quest/m0/effects.md b/quest/m0/effects.md index 62e75be..7319da2 100644 --- a/quest/m0/effects.md +++ b/quest/m0/effects.md @@ -1,24 +1,26 @@ -# [M] Video effects +# [M] ML video effects ## Goal -Sources and scene items in `moq-studio` take a curated set of video effects: -color correction, `.cube` LUTs, blur, crop and masks, chroma and luma key, -and blend modes. They render the same in every composition host. ML effects -such as background removal are offered too, applied before the source -publishes. +Sources in `moq-studio` offer ML video effects, starting with background +removal (a person cut out of a camera), applied to the source's picture before +it is drawn or published, so every viewer sees the same cut-out from the +source's own track. ## Plan -Write the effects in WGSL to the meanings the CSS Filter Effects and -Compositing specs define (color matrices, blur, drop-shadow, blend modes), so -an effect means the same thing as the CSS overlays use. The layout track -carries effect names and parameters, never shader code, since a shader from -the network costs every viewer's GPU and can lose the device. An effect the -browser cannot reproduce, like an ML model, is baked into its source's track -per the [composition epic](/quest/m0/compose/README.md), so ML effects -apply to a source, never to one scene item. Prefer an -existing model crate for ML effects over training one. +The rest of the effect set landed as moq-scene `Filter`s that every +composition host renders with the same WGSL and the layout carries by name and +settings. A model cannot meet that parity rule: viewers would each need the +model and a GPU to run it, and inference differs across backends. So an ML +effect is baked into its source's track per the +[composition epic](/quest/m0/compose/README.md): it belongs to a source, never +to one scene item, and sits before the source's crop and filters, as the +encoder in `rs/moq-scene/src/publish.rs` sees the capture. -Effects extend moq-scene's `Filter`, which a source (`[video.]`) or an -item lists, so they undo, copy, and paste as opacity, the first filter, does. +Prefer an existing runtime crate and a small, permissively licensed person +segmentation model (such as MediaPipe's selfie segmentation, Apache-2.0) over +training one. Running it on the studio's own wgpu device keeps the frame on +the GPU and works toward the [web studio](/quest/m0/web.md); a CPU runtime +costs a readback per frame. Measure the per-frame cost at 1080p and 30 fps +before choosing. From 80b08c480761460a9791043e5313b9a1370ae07e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:30:46 -0800 Subject: [PATCH 5/5] fix(scene): refuse a placed LUT and a file source without a file; document crop and luma key limits Co-Authored-By: Claude Opus 5.5 --- README.md | 11 +++++----- rs/moq-scene/src/collection.rs | 37 +++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 78472da..aeb6222 100644 --- a/README.md +++ b/README.md @@ -194,14 +194,15 @@ Source kinds are `camera`, `display`, and `microphone` (each with an optional `id`, the default device without one), `window` (with an `id`), `system_audio` (macOS), `image` (PNG or JPEG), `color` (`#rrggbb` or `#rrggbbaa`), `broadcast`, `clone` (with the `source` it shows), and `lut` (a 3D Adobe or -Resolve `.cube` file, which shows nothing itself but grades what a `lut` filter -names it in). The Devices panel fills in ids. `[[top]]` items make up the top +Resolve `.cube` file, which no item places, but which grades what a `lut` +filter names it in). The Devices panel fills in ids. `[[top]]` items make up the top layer. `[video.]` sets a source's `crop` and `filters`, wherever it is shown, and an item's `crop`, `filters`, and `blend` its own. A `crop` trims a fraction -(0 to 1) of the picture off its `left`, `top`, `right`, and `bottom` and -stretches the rest over the item; an item's crop trims what its source's +(0 to 1) of the picture off its `left`, `top`, `right`, and `bottom`, opposite +sides together trimming less than the whole, and stretches the rest over the +item; an item's crop trims what its source's leaves. Filters apply in order, the source's then the item's, with lengths in canvas pixels: @@ -213,7 +214,7 @@ canvas pixels: | `blur` | `radius`, the standard deviation; the edges fade out, as in CSS | | `drop_shadow` | `x`, `y`, `blur` (the standard deviation), and `color`, as in CSS | | `chroma_key` | `color` keyed out, and `similarity`, `smoothness`, and `spill` from 0 to 1 | -| `luma_key` | `min` and `max` luma kept, and `smoothness`, from 0 to 1 | +| `luma_key` | `min` and `max` luma kept, `min` at most `max`, and `smoothness`, all from 0 to 1 | | `lut` | the `lut` `source` to grade through, and `amount` from 0 to 1 | | `mask` | a `source` whose picture, stretched over the item, keeps what it covers: by its alpha, or `mode = "luminance"` | diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs index aa6e52b..93f5aea 100644 --- a/rs/moq-scene/src/collection.rs +++ b/rs/moq-scene/src/collection.rs @@ -148,11 +148,17 @@ impl Collection { } for (layer, items) in self.layers() { for item in items { - if !self.sources.contains_key(&item.source) { + let Some(source) = self.sources.get(&item.source) else { return Err(Error::Invalid(format!( "{layer} places `{}`, which is not a source", item.source ))); + }; + if matches!(source, Source::Lut { .. }) { + return Err(Error::Invalid(format!( + "{layer} places `{}`, a LUT, which only a filter can name", + item.source + ))); } item.rect.validate(&layer, &item.source)?; let on = format!("`{}` in {layer}", item.source); @@ -166,6 +172,10 @@ impl Collection { if let Some(name) = self.sources.keys().find(|name| name.is_empty()) { return Err(Error::Invalid(format!("a source has an empty name: `{name}`"))); } + let unnamed = |source: &Source| source.path().is_some_and(|path| path.as_os_str().is_empty()); + if let Some((name, _)) = self.sources.iter().find(|(_, source)| unnamed(source)) { + return Err(Error::Invalid(format!("`{name}` names no file"))); + } for (name, source) in &self.sources { let Source::Clone { source } = source else { continue }; match self.sources.get(source) { @@ -837,6 +847,14 @@ impl Source { !matches!(self, Self::Microphone { .. } | Self::SystemAudio | Self::Lut { .. }) } + /// The file an image or LUT source reads. + pub(crate) fn path(&self) -> Option<&std::path::Path> { + match self { + Self::Image { path } | Self::Lut { path } => Some(path), + _ => None, + } + } + /// The file the source reads, to resolve or relativize it. pub(crate) fn path_mut(&mut self) -> Option<&mut PathBuf> { match self { @@ -1162,6 +1180,23 @@ rect = { x = 0, y = 680, width = 1280, height = 40 } let mut collection = example(); collection.top.push(Item::new("nobody", Rect::fill(&collection.canvas))); assert!(matches!(collection.validate(), Err(Error::Invalid(_)))); + + // A LUT is only looked up, and a file source needs a file. + let mut collection = example(); + collection.sources.insert( + "grade".into(), + Source::Lut { + path: "grade.cube".into(), + }, + ); + collection.validate().unwrap(); + collection.top.push(Item::new("grade", Rect::fill(&collection.canvas))); + assert!(matches!(collection.validate(), Err(Error::Invalid(_)))); + for source in [Source::Lut { path: "".into() }, Source::Image { path: "".into() }] { + let mut collection = example(); + collection.sources.insert("blank".into(), source.clone()); + assert!(matches!(collection.validate(), Err(Error::Invalid(_))), "{source:?}"); + } } #[test]