fix(scene): sample transparent pictures premultiplied

quest/m0/README.md

@@ -102,7 +102,6 @@ repository.
 - [Web studio](/quest/m0/web.md) - the full studio in wasm on WebGPU and WebCodecs, served from this repository
 - [Virtual camera](/quest/m0/vcam/README.md) - a canvas's program exposed as a system camera on Linux, Windows, and macOS
 - [Recover crashed recordings](/quest/m0/recording-repair.md) - a recording a crash left unfinished is repaired beside the original at launch
-- [Premultiplied source sampling](/quest/m0/premultiply.md) - scaled transparent sources draw without edge fringes
 - [Browser sources without an X display](/quest/m0/cef-headless.md) - CEF runs headless, so Wayland sessions and tests need no X server
 - [Signing identities are available](/quest/m0/signing.md) - the Apple and Windows code-signing credentials packaging and the macOS camera need
 - [Packaging](/quest/m0/packaging.md) - signed installers for macOS and Windows and a Flatpak for Linux

rs/moq-scene/src/compositor.rs

@@ -426,8 +426,10 @@ impl Compositor {
 				overlay: None,
 				error: None,
 			});
-			// A page arrives premultiplied; frames and images are straight.
-			let premultiplied = matches!(picture, source::Picture::Texture(_));
+			// Every picture lands premultiplied, so scaling it mixes no
+			// transparent texel's color: a page arrives so, moq-video renders
+			// a frame opaque, a still is premultiplied here, and an overlay's
+			// paint as it lands.
 			let texture = match picture {
 				source::Picture::Frame(frame) => {
 					let renderer = match layer.renderer.as_mut() {
@@ -439,7 +441,10 @@ impl Compositor {
 						.and_then(|renderer| renderer.render(&frame))
 						.map_err(Error::from)
 				}
-				source::Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels),
+				source::Picture::Rgba { size, mut pixels } => {
+					premultiply(&mut pixels);
+					upload(&self.device, &self.queue, size, &pixels)
+				}
 				source::Picture::Texture(texture) => Ok(texture),
 				source::Picture::Lut(cube) => {
 					layer.lut = Some(self.effects.lut(&cube));
@@ -456,7 +461,7 @@ impl Compositor {
 					match overlays {
 						Ok(overlays) => {
 							let overlay = overlays.open(html, *width, *height, now);
-							layer.picture = Some(self.effects.picture(overlay.texture(), false));
+							layer.picture = Some(self.effects.picture(overlay.texture()));
 							layer.overlay = Some(overlay);
 						}
 						Err(err) => layer.error = Some(err.to_string()),
@@ -468,7 +473,7 @@ impl Compositor {
 			match texture {
 				Ok(texture) => {
 					layer.error = None;
-					layer.picture = Some(self.effects.picture(&texture, premultiplied));
+					layer.picture = Some(self.effects.picture(&texture));
 				}
 				Err(err) => {
 					if layer.error.is_none() {
@@ -926,6 +931,16 @@ impl Draw<'_> {
 	}
 }
 
+/// Premultiply tightly packed RGBA in place.
+fn premultiply(pixels: &mut [u8]) {
+	for pixel in pixels.as_chunks_mut::<4>().0 {
+		let alpha = u16::from(pixel[3]);
+		for channel in &mut pixel[..3] {
+			*channel = ((u16::from(*channel) * alpha + 127) / 255) as u8;
+		}
+	}
+}
+
 /// A texture holding tightly packed RGBA pixels.
 fn upload(
 	device: &wgpu::Device,
@@ -992,7 +1007,8 @@ impl Target {
 				| wgpu::TextureUsages::COPY_SRC,
 			view_formats: &[],
 		});
-		let picture = effects.picture(&texture, false);
+		// A canvas is opaque, which is premultiplied too.
+		let picture = effects.picture(&texture);
 		Self { texture, picture }
 	}
 }
@@ -2159,6 +2175,95 @@ mod tests {
 		.await;
 	}
 
+	/// A scaled picture mixes its texels premultiplied, so the color of a
+	/// transparent texel bleeds into no edge, of a still, an overlay's
+	/// straight paint, or a luminance mask, while the edge still fades.
+	#[tokio::test]
+	#[ignore = "needs a Vulkan device; run by just rs render-sw"]
+	async fn scaling_bleeds_no_transparent_color() {
+		local(async {
+			let dir = tempfile::tempdir().unwrap();
+			// Red beside a transparent green, which straight mixing tints the
+			// edge with.
+			image::RgbaImage::from_fn(2, 1, |x, _| image::Rgba(if x == 0 { RED } else { [0, 255, 0, 0] }))
+				.save(dir.path().join("logo.png"))
+				.unwrap();
+			// White, black, then a transparent white, which straight mixing
+			// lets show through where the mask is black.
+			image::RgbaImage::from_fn(3, 1, |x, _| {
+				image::Rgba([[255; 4], BLACK, [255, 255, 255, 0]][x as usize])
+			})
+			.save(dir.path().join("mask.png"))
+			.unwrap();
+			let mut collection = collection();
+			for name in ["logo", "mask"] {
+				collection.sources.insert(
+					name.into(),
+					Source::Image {
+						path: format!("{name}.png").into(),
+					},
+				);
+			}
+			// White beside transparent, painted straight.
+			collection.sources.insert(
+				"overlay".into(),
+				Source::Overlay {
+					html: "<style>
+						body { margin: 0 }
+						div { width: 1px; height: 1px; background: white }
+					</style><div></div>"
+						.into(),
+					width: 2,
+					height: 1,
+				},
+			);
+			let row = |source: &str, y| {
+				Item::new(
+					source,
+					Rect {
+						x: 0.0,
+						y,
+						width: 8.0,
+						height: 2.0,
+					},
+				)
+			};
+			collection.scenes[0].items = vec![
+				row("logo", 0.0),
+				row("overlay", 2.0),
+				Item {
+					filters: vec![Filter::Mask {
+						source: "mask".into(),
+						mode: MaskMode::Luminance,
+					}],
+					..row("blue", 4.0)
+				},
+			];
+			let program = drawn(collection, dir).await;
+
+			for x in 0..8 {
+				let logo = pixel(&program, x, 0);
+				assert_eq!(logo[1..3], [0, 0], "logo at {x}: {logo:?}");
+				// The overlay's edge fades as the logo's does.
+				let overlay = pixel(&program, x, 2);
+				assert!(
+					near(overlay, rgb([logo[0]; 3])),
+					"overlay at {x}: {overlay:?}, logo {logo:?}"
+				);
+			}
+			assert_eq!(pixel(&program, 0, 0), RED);
+			let edge = pixel(&program, 3, 0)[0];
+			assert!(0 < edge && edge < 255, "the logo's edge fades: {edge}");
+			assert_eq!(pixel(&program, 7, 0), BLACK);
+
+			assert_eq!(pixel(&program, 0, 4), BLUE);
+			for x in 4..8 {
+				assert_eq!(pixel(&program, x, 4), BLACK, "mask at {x}");
+			}
+		})
+		.await;
+	}
+
 	/// A resize draws the next frame on textures of the new size, with items
 	/// where their pixels say, a blend mode and a blur reaching the canvas's
 	/// new width, and a canvas the GPU cannot hold changes nothing.

rs/moq-scene/src/effect.rs

@@ -323,8 +323,6 @@ fn reach(sigma: f32) -> f32 {
 pub(crate) struct Picture {
 	view: wgpu::TextureView,
 	bind: wgpu::BindGroup,
-	/// Whether its alpha is premultiplied, as a web page's is.
-	premultiplied: bool,
 }
 
 /// A LUT source's table on the GPU.
@@ -344,7 +342,6 @@ struct Input {
 	area: Area,
 	uv: [f32; 4],
 	valid: [f32; 4],
-	premultiplied: bool,
 }
 
 /// Per-pixel filters waiting to run when an input is sampled.
@@ -662,8 +659,9 @@ impl Effects {
 		}
 	}
 
-	/// `texture`, a source's picture or a canvas-sized target, bound to sample.
-	pub fn picture(&self, texture: &wgpu::Texture, premultiplied: bool) -> Picture {
+	/// `texture`, a source's picture or a canvas-sized target with
+	/// premultiplied alpha, bound to sample.
+	pub fn picture(&self, texture: &wgpu::Texture) -> Picture {
 		let view = texture.create_view(&Default::default());
 		let bind = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
 			label: Some("moq-scene input"),
@@ -679,11 +677,7 @@ impl Effects {
 				},
 			],
 		});
-		Picture {
-			view,
-			bind,
-			premultiplied,
-		}
+		Picture { view, bind }
 	}
 
 	/// Upload a LUT's table.
@@ -833,7 +827,6 @@ impl Effects {
 			area,
 			uv: crop,
 			valid: crop,
-			premultiplied: item.picture.premultiplied,
 		};
 
 		let stages: Vec<Stage> = item.filters.iter().map(|filter| Stage::of(filter)).collect();
@@ -870,7 +863,7 @@ impl Effects {
 							let picture = library.picture(source)?;
 							let op = Op {
 								kind: [Op::MASK, 0, 0, 0],
-								params: [mode, f32::from(u8::from(picture.premultiplied)), 0.0, 0.0],
+								params: [mode, 0.0, 0.0, 0.0],
 								..Default::default()
 							};
 							(op, None, Some(picture.view))
@@ -951,7 +944,6 @@ impl Effects {
 				area,
 				uv: UNIT,
 				valid: UNIT,
-				premultiplied: false,
 			},
 			segment: Segment {
 				ops: ops.into_iter().collect(),
@@ -1022,7 +1014,7 @@ impl Effects {
 			input: input.area.map(input.uv, quad),
 			valid: input.valid,
 			item: item.map(UNIT, quad),
-			mode: [segment.ops.len() as u32, input.premultiplied.into(), 0, 0],
+			mode: [segment.ops.len() as u32, 0, 0, 0],
 			ops,
 			..Default::default()
 		}
@@ -1048,7 +1040,6 @@ impl Effects {
 			area,
 			uv: UNIT,
 			valid: UNIT,
-			premultiplied: true,
 		}
 	}
 
@@ -1095,7 +1086,6 @@ impl Effects {
 			rect: FULL,
 			input: UNIT,
 			valid: UNIT,
-			mode: [0, 1, 0, 0],
 			blur,
 			..Default::default()
 		};
@@ -1132,7 +1122,6 @@ impl Effects {
 			input: picture.area.map(UNIT, &area),
 			valid: UNIT,
 			aux: shape.area.map(UNIT, &area.shift(-offset.0, -offset.1)),
-			mode: [0, 1, 0, 0],
 			color: color.0.map(|channel| f32::from(channel) / 255.0),
 			..Default::default()
 		};
@@ -1151,7 +1140,6 @@ impl Effects {
 			area,
 			uv: UNIT,
 			valid: UNIT,
-			premultiplied: true,
 		}
 	}
 
@@ -1189,7 +1177,7 @@ impl Effects {
 				});
 				Raster {
 					size,
-					picture: self.picture(&texture, true),
+					picture: self.picture(&texture),
 				}
 			}
 		};

rs/moq-scene/src/effect.wgsl

@@ -1,7 +1,8 @@
 // Every draw the compositor makes: a quad sampling one picture, through a list
 // of per-pixel filters, a blur, or a drop shadow. Colors are gamma-encoded
-// sRGB, as CSS filters see them. Rasters between passes hold premultiplied
-// alpha, so a blur weighs a transparent pixel as nothing.
+// sRGB, as CSS filters see them. Every picture and every raster between passes
+// holds premultiplied alpha, so neither scaling nor a blur mixes in a
+// transparent pixel's color.
 
 const OPS: u32 = 8u;
 
@@ -19,8 +20,7 @@ const LUMA: vec3<f32> = vec3<f32>(0.2126, 0.7152, 0.0722);
 struct Op {
 	kind: vec4<u32>,
 	// MATRIX: unused. CHROMA_KEY: key Cb, key Cr, similarity, smoothness.
-	// LUMA_KEY: min, max, smoothness. LUT: amount, size. MASK: mode, and 1 if
-	// the mask is premultiplied.
+	// LUMA_KEY: min, max, smoothness. LUT: amount, size. MASK: mode.
 	params: vec4<f32>,
 	// MATRIX: applied to straight RGBA. LUT: the domain's max in column 0.
 	matrix: mat4x4<f32>,
@@ -39,7 +39,7 @@ struct Draw {
 	item: vec4<f32>,
 	// The aux texture's coordinates at the quad's corners.
 	aux: vec4<f32>,
-	// Ops in x, whether the input is premultiplied in y, the blend mode in z.
+	// Ops in x, the blend mode in z.
 	mode: vec4<u32>,
 	// A blur's step between taps in xy, its standard deviation in z, and its
 	// taps each side in w, all in texels.
@@ -79,7 +79,8 @@ fn within(uv: vec2<f32>, low: vec2<f32>, high: vec2<f32>) -> bool {
 	return all(uv >= low) && all(uv <= high);
 }
 
-// The input at `uv`, transparent outside its picture, as CSS's edge mode none.
+// The input at `uv`, premultiplied, and transparent outside its picture, as
+// CSS's edge mode none.
 fn fetch(uv: vec2<f32>) -> vec4<f32> {
 	if !within(uv, draw.valid.xy, draw.valid.zw) {
 		return vec4<f32>(0.0);
@@ -90,9 +91,6 @@ fn fetch(uv: vec2<f32>) -> vec4<f32> {
 // The input at `uv`, with straight alpha.
 fn straight(uv: vec2<f32>) -> vec4<f32> {
 	let color = fetch(uv);
-	if draw.mode.y == 0u {
-		return color;
-	}
 	if color.a <= 0.0 {
 		return vec4<f32>(0.0);
 	}
@@ -138,14 +136,14 @@ fn lookup(rgb: vec3<f32>, op: Op) -> vec3<f32> {
 }
 
 // The mask's coverage at `item`, stretched over the item and empty outside it.
+// Luminance is of the premultiplied color, so a transparent texel covers none.
 fn coverage(item: vec2<f32>, op: Op) -> f32 {
 	if !within(item, vec2<f32>(0.0), vec2<f32>(1.0)) {
 		return 0.0;
 	}
 	let color = textureSampleLevel(mask, bilinear, item, 0.0);
 	if op.params.x > 0.5 {
-		let rgb = select(color.rgb * color.a, color.rgb, op.params.y > 0.5);
-		return dot(rgb, LUMA);
+		return dot(color.rgb, LUMA);
 	}
 	return color.a;
 }