diff --git a/quest/m0/README.md b/quest/m0/README.md index 56677f6..2f80f7b 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -102,7 +102,6 @@ repository. - [Web studio](/quest/m0/web.md) - the full studio in wasm on WebGPU and WebCodecs, served from this repository - [Virtual camera](/quest/m0/vcam/README.md) - a canvas's program exposed as a system camera on Linux, Windows, and macOS - [Recover crashed recordings](/quest/m0/recording-repair.md) - a recording a crash left unfinished is repaired beside the original at launch -- [Premultiplied source sampling](/quest/m0/premultiply.md) - scaled transparent sources draw without edge fringes - [Browser sources without an X display](/quest/m0/cef-headless.md) - CEF runs headless, so Wayland sessions and tests need no X server - [Signing identities are available](/quest/m0/signing.md) - the Apple and Windows code-signing credentials packaging and the macOS camera need - [Packaging](/quest/m0/packaging.md) - signed installers for macOS and Windows and a Flatpak for Linux diff --git a/quest/m0/premultiply.md b/quest/m0/premultiply.md deleted file mode 100644 index f8bd901..0000000 --- a/quest/m0/premultiply.md +++ /dev/null @@ -1,23 +0,0 @@ -# [XS] Premultiplied source sampling - -## Goal - -A scaled source with transparency, such as a PNG logo, draws without a dark -or light fringe at its edges, because the compositor samples it -premultiplied. - -## Plan - -From [moq-dev/studio#17](https://github.com/moq-dev/studio/pull/17), raised -by CodeRabbit on `rs/moq-scene/src/effect.wgsl`. It predates that PR. - -- Source pictures are sampled bilinearly straight (not premultiplied), so a - transparent texel's color bleeds into its neighbours when scaled. - Premultiply before interpolation: upload premultiplied, or premultiply each - tap in the shader, whichever adds no GPU copy. The effect pass's - `premultiplied` flag is not the fix: `fetch()` in `effect.wgsl` interpolates - first and `straight()` divides afterwards, so the flag only describes the - input. -- A compositor test scales a transparent picture with a colored transparent - region and checks that edge texels carry no color from it. -- Measure frame time before and after. diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index 111563f..e45e017 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -426,8 +426,10 @@ impl Compositor { overlay: None, error: None, }); - // A page arrives premultiplied; frames and images are straight. - let premultiplied = matches!(picture, source::Picture::Texture(_)); + // Every picture lands premultiplied, so scaling it mixes no + // transparent texel's color: a page arrives so, moq-video renders + // a frame opaque, a still is premultiplied here, and an overlay's + // paint as it lands. let texture = match picture { source::Picture::Frame(frame) => { let renderer = match layer.renderer.as_mut() { @@ -439,7 +441,10 @@ impl Compositor { .and_then(|renderer| renderer.render(&frame)) .map_err(Error::from) } - source::Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels), + source::Picture::Rgba { size, mut pixels } => { + premultiply(&mut pixels); + upload(&self.device, &self.queue, size, &pixels) + } source::Picture::Texture(texture) => Ok(texture), source::Picture::Lut(cube) => { layer.lut = Some(self.effects.lut(&cube)); @@ -456,7 +461,7 @@ impl Compositor { match overlays { Ok(overlays) => { let overlay = overlays.open(html, *width, *height, now); - layer.picture = Some(self.effects.picture(overlay.texture(), false)); + layer.picture = Some(self.effects.picture(overlay.texture())); layer.overlay = Some(overlay); } Err(err) => layer.error = Some(err.to_string()), @@ -468,7 +473,7 @@ impl Compositor { match texture { Ok(texture) => { layer.error = None; - layer.picture = Some(self.effects.picture(&texture, premultiplied)); + layer.picture = Some(self.effects.picture(&texture)); } Err(err) => { if layer.error.is_none() { @@ -926,6 +931,16 @@ impl Draw<'_> { } } +/// Premultiply tightly packed RGBA in place. +fn premultiply(pixels: &mut [u8]) { + for pixel in pixels.as_chunks_mut::<4>().0 { + let alpha = u16::from(pixel[3]); + for channel in &mut pixel[..3] { + *channel = ((u16::from(*channel) * alpha + 127) / 255) as u8; + } + } +} + /// A texture holding tightly packed RGBA pixels. fn upload( device: &wgpu::Device, @@ -992,7 +1007,8 @@ impl Target { | wgpu::TextureUsages::COPY_SRC, view_formats: &[], }); - let picture = effects.picture(&texture, false); + // A canvas is opaque, which is premultiplied too. + let picture = effects.picture(&texture); Self { texture, picture } } } @@ -2159,6 +2175,95 @@ mod tests { .await; } + /// A scaled picture mixes its texels premultiplied, so the color of a + /// transparent texel bleeds into no edge, of a still, an overlay's + /// straight paint, or a luminance mask, while the edge still fades. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn scaling_bleeds_no_transparent_color() { + local(async { + let dir = tempfile::tempdir().unwrap(); + // Red beside a transparent green, which straight mixing tints the + // edge with. + image::RgbaImage::from_fn(2, 1, |x, _| image::Rgba(if x == 0 { RED } else { [0, 255, 0, 0] })) + .save(dir.path().join("logo.png")) + .unwrap(); + // White, black, then a transparent white, which straight mixing + // lets show through where the mask is black. + image::RgbaImage::from_fn(3, 1, |x, _| { + image::Rgba([[255; 4], BLACK, [255, 255, 255, 0]][x as usize]) + }) + .save(dir.path().join("mask.png")) + .unwrap(); + let mut collection = collection(); + for name in ["logo", "mask"] { + collection.sources.insert( + name.into(), + Source::Image { + path: format!("{name}.png").into(), + }, + ); + } + // White beside transparent, painted straight. + collection.sources.insert( + "overlay".into(), + Source::Overlay { + html: "
" + .into(), + width: 2, + height: 1, + }, + ); + let row = |source: &str, y| { + Item::new( + source, + Rect { + x: 0.0, + y, + width: 8.0, + height: 2.0, + }, + ) + }; + collection.scenes[0].items = vec![ + row("logo", 0.0), + row("overlay", 2.0), + Item { + filters: vec![Filter::Mask { + source: "mask".into(), + mode: MaskMode::Luminance, + }], + ..row("blue", 4.0) + }, + ]; + let program = drawn(collection, dir).await; + + for x in 0..8 { + let logo = pixel(&program, x, 0); + assert_eq!(logo[1..3], [0, 0], "logo at {x}: {logo:?}"); + // The overlay's edge fades as the logo's does. + let overlay = pixel(&program, x, 2); + assert!( + near(overlay, rgb([logo[0]; 3])), + "overlay at {x}: {overlay:?}, logo {logo:?}" + ); + } + assert_eq!(pixel(&program, 0, 0), RED); + let edge = pixel(&program, 3, 0)[0]; + assert!(0 < edge && edge < 255, "the logo's edge fades: {edge}"); + assert_eq!(pixel(&program, 7, 0), BLACK); + + assert_eq!(pixel(&program, 0, 4), BLUE); + for x in 4..8 { + assert_eq!(pixel(&program, x, 4), BLACK, "mask at {x}"); + } + }) + .await; + } + /// A resize draws the next frame on textures of the new size, with items /// where their pixels say, a blend mode and a blur reaching the canvas's /// new width, and a canvas the GPU cannot hold changes nothing. diff --git a/rs/moq-scene/src/effect.rs b/rs/moq-scene/src/effect.rs index 70b9344..94518ee 100644 --- a/rs/moq-scene/src/effect.rs +++ b/rs/moq-scene/src/effect.rs @@ -323,8 +323,6 @@ fn reach(sigma: f32) -> f32 { pub(crate) struct Picture { view: wgpu::TextureView, bind: wgpu::BindGroup, - /// Whether its alpha is premultiplied, as a web page's is. - premultiplied: bool, } /// A LUT source's table on the GPU. @@ -344,7 +342,6 @@ struct Input { area: Area, uv: [f32; 4], valid: [f32; 4], - premultiplied: bool, } /// Per-pixel filters waiting to run when an input is sampled. @@ -662,8 +659,9 @@ impl Effects { } } - /// `texture`, a source's picture or a canvas-sized target, bound to sample. - pub fn picture(&self, texture: &wgpu::Texture, premultiplied: bool) -> Picture { + /// `texture`, a source's picture or a canvas-sized target with + /// premultiplied alpha, bound to sample. + pub fn picture(&self, texture: &wgpu::Texture) -> Picture { let view = texture.create_view(&Default::default()); let bind = self.device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("moq-scene input"), @@ -679,11 +677,7 @@ impl Effects { }, ], }); - Picture { - view, - bind, - premultiplied, - } + Picture { view, bind } } /// Upload a LUT's table. @@ -833,7 +827,6 @@ impl Effects { area, uv: crop, valid: crop, - premultiplied: item.picture.premultiplied, }; let stages: Vec = item.filters.iter().map(|filter| Stage::of(filter)).collect(); @@ -870,7 +863,7 @@ impl Effects { let picture = library.picture(source)?; let op = Op { kind: [Op::MASK, 0, 0, 0], - params: [mode, f32::from(u8::from(picture.premultiplied)), 0.0, 0.0], + params: [mode, 0.0, 0.0, 0.0], ..Default::default() }; (op, None, Some(picture.view)) @@ -951,7 +944,6 @@ impl Effects { area, uv: UNIT, valid: UNIT, - premultiplied: false, }, segment: Segment { ops: ops.into_iter().collect(), @@ -1022,7 +1014,7 @@ impl Effects { input: input.area.map(input.uv, quad), valid: input.valid, item: item.map(UNIT, quad), - mode: [segment.ops.len() as u32, input.premultiplied.into(), 0, 0], + mode: [segment.ops.len() as u32, 0, 0, 0], ops, ..Default::default() } @@ -1048,7 +1040,6 @@ impl Effects { area, uv: UNIT, valid: UNIT, - premultiplied: true, } } @@ -1095,7 +1086,6 @@ impl Effects { rect: FULL, input: UNIT, valid: UNIT, - mode: [0, 1, 0, 0], blur, ..Default::default() }; @@ -1132,7 +1122,6 @@ impl Effects { input: picture.area.map(UNIT, &area), valid: UNIT, aux: shape.area.map(UNIT, &area.shift(-offset.0, -offset.1)), - mode: [0, 1, 0, 0], color: color.0.map(|channel| f32::from(channel) / 255.0), ..Default::default() }; @@ -1151,7 +1140,6 @@ impl Effects { area, uv: UNIT, valid: UNIT, - premultiplied: true, } } @@ -1189,7 +1177,7 @@ impl Effects { }); Raster { size, - picture: self.picture(&texture, true), + picture: self.picture(&texture), } } }; diff --git a/rs/moq-scene/src/effect.wgsl b/rs/moq-scene/src/effect.wgsl index 40d503a..3afcab6 100644 --- a/rs/moq-scene/src/effect.wgsl +++ b/rs/moq-scene/src/effect.wgsl @@ -1,7 +1,8 @@ // Every draw the compositor makes: a quad sampling one picture, through a list // of per-pixel filters, a blur, or a drop shadow. Colors are gamma-encoded -// sRGB, as CSS filters see them. Rasters between passes hold premultiplied -// alpha, so a blur weighs a transparent pixel as nothing. +// sRGB, as CSS filters see them. Every picture and every raster between passes +// holds premultiplied alpha, so neither scaling nor a blur mixes in a +// transparent pixel's color. const OPS: u32 = 8u; @@ -19,8 +20,7 @@ const LUMA: vec3 = vec3(0.2126, 0.7152, 0.0722); struct Op { kind: vec4, // MATRIX: unused. CHROMA_KEY: key Cb, key Cr, similarity, smoothness. - // LUMA_KEY: min, max, smoothness. LUT: amount, size. MASK: mode, and 1 if - // the mask is premultiplied. + // LUMA_KEY: min, max, smoothness. LUT: amount, size. MASK: mode. params: vec4, // MATRIX: applied to straight RGBA. LUT: the domain's max in column 0. matrix: mat4x4, @@ -39,7 +39,7 @@ struct Draw { item: vec4, // The aux texture's coordinates at the quad's corners. aux: vec4, - // Ops in x, whether the input is premultiplied in y, the blend mode in z. + // Ops in x, the blend mode in z. mode: vec4, // A blur's step between taps in xy, its standard deviation in z, and its // taps each side in w, all in texels. @@ -79,7 +79,8 @@ fn within(uv: vec2, low: vec2, high: vec2) -> bool { return all(uv >= low) && all(uv <= high); } -// The input at `uv`, transparent outside its picture, as CSS's edge mode none. +// The input at `uv`, premultiplied, and transparent outside its picture, as +// CSS's edge mode none. fn fetch(uv: vec2) -> vec4 { if !within(uv, draw.valid.xy, draw.valid.zw) { return vec4(0.0); @@ -90,9 +91,6 @@ fn fetch(uv: vec2) -> vec4 { // The input at `uv`, with straight alpha. fn straight(uv: vec2) -> vec4 { let color = fetch(uv); - if draw.mode.y == 0u { - return color; - } if color.a <= 0.0 { return vec4(0.0); } @@ -138,14 +136,14 @@ fn lookup(rgb: vec3, op: Op) -> vec3 { } // The mask's coverage at `item`, stretched over the item and empty outside it. +// Luminance is of the premultiplied color, so a transparent texel covers none. fn coverage(item: vec2, op: Op) -> f32 { if !within(item, vec2(0.0), vec2(1.0)) { return 0.0; } let color = textureSampleLevel(mask, bilinear, item, 0.0); if op.params.x > 0.5 { - let rgb = select(color.rgb * color.a, color.rgb, op.params.y > 0.5); - return dot(rgb, LUMA); + return dot(color.rgb, LUMA); } return color.a; } diff --git a/rs/moq-scene/src/overlay.rs b/rs/moq-scene/src/overlay.rs index f8e95fd..7d66ceb 100644 --- a/rs/moq-scene/src/overlay.rs +++ b/rs/moq-scene/src/overlay.rs @@ -1,6 +1,6 @@ //! Overlays: titles, lower thirds, and other graphics written as HTML and CSS, -//! laid out by Blitz and painted by Vello straight into a texture on the -//! compositor's device, on the program clock. +//! laid out by Blitz and painted by Vello into a texture on the compositor's +//! device, on the program clock. //! //! An overlay is untrusted, so it loads and runs nothing: Blitz has no script //! engine, its document has no network or navigation provider, and [`check`] @@ -22,6 +22,24 @@ use style::shared_lock::{SharedRwLock, SharedRwLockReadGuard, ToCssWithGuard}; use style::stylesheets::{AllowImportRules, CssRule, CssRuleType, Origin, StylesheetContents, UrlExtraData}; use crate::Error; +use crate::compositor::FORMAT; + +/// Copies an overlay's straight paint into the texture the compositor samples, +/// premultiplied, so scaling it mixes no transparent texel's color. +const PREMULTIPLY: &str = r#" +@group(0) @binding(0) var paint: texture_2d; + +// One triangle covering the whole target. +@vertex fn vs_main(@builtin(vertex_index) index: u32) -> @builtin(position) vec4 { + let corner = vec2(f32((index << 1u) & 2u), f32(index & 2u)); + return vec4(corner * 2.0 - 1.0, 0.0, 1.0); +} + +@fragment fn fs_main(@builtin(position) position: vec4) -> @location(0) vec4 { + let color = textureLoad(paint, vec2(position.xy), 0); + return vec4(color.rgb * color.a, color.a); +} +"#; /// The elements an overlay may hold: text and the boxes around it. Anything /// that loads a resource (`img`, `link`, `iframe`), runs code (`script`), @@ -540,6 +558,9 @@ enum Request { Open { id: u64, html: String, + /// Where Vello paints, straight. + paint: wgpu::TextureView, + /// Where the paint lands premultiplied, for the compositor to sample. view: wgpu::TextureView, start: Instant, }, @@ -575,27 +596,33 @@ impl Overlays { /// Lay out `html` at `width` by `height`, its CSS timeline starting at /// `start` on the program clock. The HTML has passed [`check`]. pub fn open(&mut self, html: &str, width: u32, height: u32, start: Instant) -> Overlay { - let texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("moq-scene overlay"), - size: wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - // Vello writes gamma-encoded straight alpha, what the compositor reads. - format: crate::FORMAT, - usage: wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::TEXTURE_BINDING, - view_formats: &[], - }); + let texture = |label, usage| { + self.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + // Gamma-encoded, as the compositor reads. + format: FORMAT, + usage: usage | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) + }; + // Vello writes straight alpha, which one pass per paint premultiplies. + let paint = texture("moq-scene overlay paint", wgpu::TextureUsages::STORAGE_BINDING); + let texture = texture("moq-scene overlay", wgpu::TextureUsages::RENDER_ATTACHMENT); let id = self.next; self.next += 1; // A thread that stopped fails the next paint, which says so. let _ = self.requests.send(Request::Open { id, html: html.to_string(), + paint: paint.create_view(&Default::default()), view: texture.create_view(&Default::default()), start, }); @@ -640,7 +667,7 @@ impl Overlay { self.id } - /// The texture the overlay paints into, straight alpha. + /// The texture each paint lands in, premultiplied. pub fn texture(&self) -> &wgpu::Texture { &self.texture } @@ -664,7 +691,13 @@ fn run( let mut painter: Option> = None; for request in requests { match request { - Request::Open { id, html, view, start } => { + Request::Open { + id, + html, + paint, + view, + start, + } => { let size = view.texture().size(); let mut document = document(&html, size.width, size.height); // Stylo starts an animation at the time of the resolve that @@ -673,7 +706,9 @@ fn run( document.resolve(0.0); let document = Document { document, + paint, view, + premultiply: None, start, painted: false, }; @@ -705,11 +740,13 @@ fn run( } } -/// Vello's compute pipelines on the compositor's device, compiled once and -/// shared by every overlay. +/// Vello's compute pipelines on the compositor's device, and the pass that +/// premultiplies what they paint, compiled once and shared by every overlay. struct Painter { renderer: vello::Renderer, scene: vello::Scene, + premultiply: wgpu::RenderPipeline, + layout: wgpu::BindGroupLayout, } impl Painter { @@ -724,17 +761,107 @@ impl Painter { ..Default::default() }; let renderer = vello::Renderer::new(device, options).map_err(|err| err.to_string())?; + + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene premultiply"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("moq-scene premultiply"), + bind_group_layouts: &[Some(&layout)], + immediate_size: 0, + }); + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("moq-scene premultiply"), + source: wgpu::ShaderSource::Wgsl(PREMULTIPLY.into()), + }); + let premultiply = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("moq-scene premultiply"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: Default::default(), + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); Ok(Self { renderer, scene: vello::Scene::new(), + premultiply, + layout, }) } + + /// Copy `paint` into `view`, premultiplied, after the work already queued. + fn premultiply( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + bind: &wgpu::BindGroup, + view: &wgpu::TextureView, + ) { + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("moq-scene premultiply"), + }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("moq-scene premultiply"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(&self.premultiply); + pass.set_bind_group(0, bind, &[]); + pass.draw(0..3, 0..1); + } + queue.submit([encoder.finish()]); + } } /// An overlay's document on the overlay thread, and where it paints. struct Document { document: BaseDocument, + /// Where Vello paints, straight. + paint: wgpu::TextureView, + /// Where the paint lands premultiplied. view: wgpu::TextureView, + /// `paint` bound for [`Painter::premultiply`], from the first paint on. + premultiply: Option, /// Where its CSS timeline starts on the program clock. start: Instant, /// Whether the texture holds a paint yet. @@ -757,7 +884,7 @@ impl Document { self.document .resolve(now.saturating_duration_since(self.start).as_secs_f64()); - let size = self.view.texture().size(); + let size = self.paint.texture().size(); painter.scene.reset(); let mut scene = anyrender_vello::VelloScenePainter::new(&mut painter.scene); blitz_paint::paint_scene(&mut scene, &mut self.document, 1.0, size.width, size.height, 0, 0); @@ -769,8 +896,19 @@ impl Document { }; painter .renderer - .render_to_texture(device, queue, &painter.scene, &self.view, ¶ms) + .render_to_texture(device, queue, &painter.scene, &self.paint, ¶ms) .map_err(|err| err.to_string())?; + let bind = self.premultiply.get_or_insert_with(|| { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene premultiply"), + layout: &painter.layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&self.paint), + }], + }) + }); + painter.premultiply(device, queue, bind, &self.view); self.painted = true; Ok(()) }