From 5527deaefe0ea9854c2630fd6d019eaac4a7b7e3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 08:41:11 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m0/browser-popups.md Quest-Claim: 960434867f3e1bc0 From 8e1d48962aba42fd954807659331339f1a1ffc0f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 08:47:39 -0800 Subject: [PATCH 2/5] feat(scene): browser source popups draw over the page and take input CEF paints a popup widget, such as a `'s, are not drawn yet. +**Reload** in Sources restarts a page, failed or not. A page's dropdown +lists, such as a `` dropdowns, draw on the -canvas and take input like the rest of the page, on both the GPU import and -the CPU fallback paths. - -## Plan - -CEF paints popup widgets separately (`PET_POPUP`) with their own rectangle, -and the helper drops them today; `README.md` states the limitation, and this -quest removes that note. `window.open` and `target="_blank"` stay cancelled: -those are new browsers, not widgets. Raised on -[#8](https://github.com/moq-dev/studio/pull/8). - -## Related - -- [Browser sources](/quest/m0/browser-source.md) - the macOS and Windows half diff --git a/rs/moq-scene/src/browser/host.rs b/rs/moq-scene/src/browser/host.rs index 4d85c8e..9c20ec7 100644 --- a/rs/moq-scene/src/browser/host.rs +++ b/rs/moq-scene/src/browser/host.rs @@ -13,7 +13,7 @@ use std::sync::{Arc, Mutex, mpsc}; use cef::*; use super::outbox::Commands; -use super::protocol::{Channel, Command, Event, Format, Id, Open, Plane, Start, Texture}; +use super::protocol::{Area, Channel, Command, Event, Format, Id, Layer, Open, Plane, Start, Texture}; use super::{Button, Input, Modifiers}; /// Run one of CEF's subprocesses (renderer, GPU, utility, zygote). @@ -495,6 +495,25 @@ wrap_render_handler! { } } + // The studio draws a popup widget over the page, and CEF hands input + // over it to it, so the studio sends input as for the page alone. + fn on_popup_show(&self, _browser: Option<&mut Browser>, show: ::std::os::raw::c_int) { + if show == 0 { + self.page.shared.send(&Event::Popup(self.page.id, None), &[]); + } + } + + fn on_popup_size(&self, _browser: Option<&mut Browser>, rect: Option<&Rect>) { + let Some(rect) = rect else { return }; + let area = Area { + x: rect.x, + y: rect.y, + width: rect.width.max(0) as u32, + height: rect.height.max(0) as u32, + }; + self.page.shared.send(&Event::Popup(self.page.id, Some(area)), &[]); + } + fn on_paint( &self, _browser: Option<&mut Browser>, @@ -504,7 +523,8 @@ wrap_render_handler! { width: ::std::os::raw::c_int, height: ::std::os::raw::c_int, ) { - if type_ != PaintElementType::VIEW || buffer.is_null() || width <= 0 || height <= 0 { + let Some(layer) = layer(type_) else { return }; + if buffer.is_null() || width <= 0 || height <= 0 { return; } let len = width as usize * height as usize * 4; @@ -513,6 +533,7 @@ wrap_render_handler! { let data = unsafe { std::slice::from_raw_parts(buffer, len) }.to_vec(); let event = Event::Pixels { page: self.page.id, + layer, width: width as u32, height: height as u32, data, @@ -527,10 +548,9 @@ wrap_render_handler! { _dirty_rects: Option<&[Rect]>, info: Option<&AcceleratedPaintInfo>, ) { - let Some(info) = info else { return }; - if type_ != PaintElementType::VIEW { + let (Some(info), Some(layer)) = (info, layer(type_)) else { return; - } + }; let format = match info.format { ColorType::RGBA_8888 => Format::Rgba, ColorType::BGRA_8888 => Format::Bgra, @@ -558,11 +578,22 @@ wrap_render_handler! { // SAFETY: CEF owns these descriptors and keeps them open for this // call, which outlasts the send that duplicates them. let fds: Vec<_> = planes.iter().map(|plane| unsafe { BorrowedFd::borrow_raw(plane.fd) }).collect(); - self.page.shared.paint(&Event::Texture(self.page.id, texture), &fds); + self.page.shared.paint(&Event::Texture(self.page.id, layer, texture), &fds); } } } +/// The layer CEF painted, if it is one the studio shows. +fn layer(type_: PaintElementType) -> Option { + if type_ == PaintElementType::VIEW { + Some(Layer::View) + } else if type_ == PaintElementType::POPUP { + Some(Layer::Popup) + } else { + None + } +} + wrap_audio_handler! { struct HelperAudio { page: Arc, @@ -680,7 +711,7 @@ wrap_life_span_handler! { } impl LifeSpanHandler { - // A source shows one page, and nothing would show a popup, so + // A source shows one page, and nothing would show another window, so // `window.open` and `target="_blank"` open nothing. That also keeps // the studio's pages the only browsers, whose closing ends the helper. fn on_before_popup( diff --git a/rs/moq-scene/src/browser/layers.rs b/rs/moq-scene/src/browser/layers.rs new file mode 100644 index 0000000..c0f2162 --- /dev/null +++ b/rs/moq-scene/src/browser/layers.rs @@ -0,0 +1,169 @@ +//! What a page shows: its view and, while one is open, the popup CEF paints +//! apart from it, such as a ``'s list, arrive the same way apart +//! from the page, and the studio draws them over it. A page's sound joins the +//! program bus at the bus's rate. //! //! Linux only for now, and only with the `cef` feature: elsewhere a browser //! source reports that it is unsupported. @@ -25,6 +27,8 @@ use serde::{Deserialize, Serialize}; use crate::wgpu; +#[cfg(target_os = "linux")] +mod layers; #[cfg(target_os = "linux")] mod outbox; #[cfg(target_os = "linux")] @@ -316,7 +320,7 @@ pub struct Modifiers { } /// The texture a page's pixels are uploaded to: premultiplied BGRA, as CEF -/// paints. +/// paints, and a copy source for drawing a popup over the page. #[cfg_attr( not(target_os = "linux"), expect(dead_code, reason = "browser sources are Linux-only") @@ -349,7 +353,7 @@ fn upload( sample_count: 1, dimension: wgpu::TextureDimension::D2, format: wgpu::TextureFormat::Bgra8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::COPY_SRC, view_formats: &[], }); queue.write_texture( diff --git a/rs/moq-scene/src/browser/protocol.rs b/rs/moq-scene/src/browser/protocol.rs index fec4d76..426da8c 100644 --- a/rs/moq-scene/src/browser/protocol.rs +++ b/rs/moq-scene/src/browser/protocol.rs @@ -78,18 +78,21 @@ pub(crate) struct Open { /// What the helper tells the studio. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub(crate) enum Event { - /// A page painted into a shared texture, whose plane descriptors ride - /// with this message. Valid until the studio answers + /// A page painted a layer into a shared texture, whose plane descriptors + /// ride with this message. Valid until the studio answers /// [`Command::Painted`]. - Texture(Id, Texture), - /// A page painted into memory: `width * height` premultiplied BGRA - /// pixels, tightly packed. + Texture(Id, Layer, Texture), + /// A page painted `layer` into memory: `width * height` premultiplied + /// BGRA pixels, tightly packed. Pixels { page: Id, + layer: Layer, width: u32, height: u32, data: Vec, }, + /// A page's popup opened or moved to this area of its view, or closed. + Popup(Id, Option), /// Interleaved samples at the [`Open`] rate and channel count. Audio(Id, Vec), /// A page cannot be shown, and why. @@ -98,6 +101,27 @@ pub(crate) enum Event { Fatal(String), } +/// Which of a page's pictures a paint is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) enum Layer { + /// The page itself. + View, + /// Its open popup, such as a ``'s list, which CEF paints apart from the page, draws over the +/// page, and a click on it picks an option like a click on the page would. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs CEF and a Vulkan device; run by just render-sw"] +async fn a_dropdown_draws_and_takes_clicks() { + local(async { + // A select across the top, whose list of lime options covers the + // center once open; picking one turns the page blue. + let options = "".repeat(12); + let html = format!( + "\ + " + ); + let mut studio = Studio::new(&[("page", page(&html))]).await; + studio.until_center([255, 0, 0, 255], &[]).await; + + let page = studio.sources.get("page").unwrap().page().unwrap().clone(); + let click = |x, y| { + for down in [true, false] { + page.send(Input::Button { + x, + y, + button: Button::Left, + down, + clicks: 1, + modifiers: Modifiers::default(), + }); + } + }; + page.send(Input::Focus(true)); + click(160, 10); + studio.until_center(GREEN, &[]).await; + click(160, 120); + studio.until_center(BLUE, &[]).await; + }) + .await; +} + /// A page that closes itself fails its own source and leaves the canvas, /// while the page beside it in the same helper keeps drawing. #[tokio::test(flavor = "multi_thread")] From 66974b00b3652a7cf1c1873bd4dc7d9186609e04 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 08:57:12 -0800 Subject: [PATCH 3/5] test(scene): name the popup clip test's copy type for clippy Co-Authored-By: Claude Opus 5.5 --- rs/moq-scene/src/browser/layers.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rs/moq-scene/src/browser/layers.rs b/rs/moq-scene/src/browser/layers.rs index c0f2162..93ee1dc 100644 --- a/rs/moq-scene/src/browser/layers.rs +++ b/rs/moq-scene/src/browser/layers.rs @@ -124,8 +124,10 @@ mod tests { Area { x, y, width, height } } - /// Copies as `((from x, from y), (to x, to y), (width, height))`. - fn clipped(area: Area, popup: (u32, u32), view: (u32, u32)) -> Option<((u32, u32), (u32, u32), (u32, u32))> { + /// A copy as `((from x, from y), (to x, to y), (width, height))`. + type Copied = ((u32, u32), (u32, u32), (u32, u32)); + + fn clipped(area: Area, popup: (u32, u32), view: (u32, u32)) -> Option { clip(area, extent(popup.0, popup.1), extent(view.0, view.1)) .map(|(from, to, size)| ((from.x, from.y), (to.x, to.y), (size.width, size.height))) } From eaf0655e367d41ea9a042a36df0b02b407724c4f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 09:21:18 -0800 Subject: [PATCH 4/5] fix(scene): blend a browser popup over its page rather than copy it A popup's first paints can be transparent, which a copy turned into a hole through the page to the scene behind it. Draw it premultiplied over the page instead, reading it with textureLoad so its channel order need not match. Co-Authored-By: Claude Opus 5.5 --- justfile | 2 +- rs/moq-scene/src/browser/layers.rs | 380 +++++++++++++++++++++++++---- rs/moq-scene/src/browser/run.rs | 42 ++-- 3 files changed, 352 insertions(+), 72 deletions(-) diff --git a/justfile b/justfile index 61f640f..ef0a266 100644 --- a/justfile +++ b/justfile @@ -54,7 +54,7 @@ render-sw: loader=$(nix build --inputs-from . --no-link --print-out-paths nixpkgs#vulkan-loader) icd=("$mesa"/share/vulkan/icd.d/lvp_icd.*.json) export LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" - cargo nextest run --locked -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/)' + cargo nextest run --locked -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/) | test(/^browser::layers::tests::/)' cargo nextest run --locked -p moq-studio --features openh264 --run-ignored only -E 'test(/^engine::tests::the_program_/) | test(=setup::tests::times_every_canvas)' # CEF's browser process opens an X display even to paint offscreen. xvfb=$(nix build --inputs-from . --no-link --print-out-paths nixpkgs#xvfb-run) diff --git a/rs/moq-scene/src/browser/layers.rs b/rs/moq-scene/src/browser/layers.rs index 93ee1dc..21a8c12 100644 --- a/rs/moq-scene/src/browser/layers.rs +++ b/rs/moq-scene/src/browser/layers.rs @@ -3,6 +3,197 @@ use super::protocol::{Area, Layer}; use crate::wgpu; +use crate::wgpu::util::DeviceExt; + +/// The formats a page arrives in: CEF's shared textures, and the uploads. +const FORMATS: [wgpu::TextureFormat; 2] = [wgpu::TextureFormat::Rgba8Unorm, wgpu::TextureFormat::Bgra8Unorm]; + +/// Each fragment of the popup's area reads the popup's pixel `offset` from +/// it, whatever order the popup's channels are in. +const SHADER: &str = " +struct Place { offset: vec2 } +@group(0) @binding(0) var popup: texture_2d; +@group(0) @binding(1) var place: Place; + +@vertex +fn vs_main(@builtin(vertex_index) index: u32) -> @builtin(position) vec4 { + let corner = vec2(f32((index << 1u) & 2u), f32(index & 2u)); + return vec4(corner * 2.0 - 1.0, 0.0, 1.0); +} + +@fragment +fn fs_main(@builtin(position) position: vec4) -> @location(0) vec4 { + return textureLoad(popup, vec2(position.xy) + place.offset, 0); +} +"; + +/// The studio's device, which pages' pictures are copied to, and the pass that +/// draws a popup over its page there. +pub(super) struct Gpu { + pub device: wgpu::Device, + pub queue: wgpu::Queue, + layout: wgpu::BindGroupLayout, + /// One per [`FORMATS`], in order. + pipelines: Vec, +} + +impl Gpu { + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Self { + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene popup"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("moq-scene popup"), + bind_group_layouts: &[Some(&layout)], + immediate_size: 0, + }); + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("moq-scene popup"), + source: wgpu::ShaderSource::Wgsl(SHADER.into()), + }); + let pipelines = FORMATS + .iter() + .map(|&format| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("moq-scene popup"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: Default::default(), + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + // Both arrive premultiplied, so a popup's translucent + // or not yet painted pixels show the page under them. + targets: &[Some(wgpu::ColorTargetState { + format, + blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }) + }) + .collect(); + Self { + device: device.clone(), + queue: queue.clone(), + layout, + pipelines, + } + } + + /// A copy of `view` with `popup` drawn over it at `area`, or `view` itself + /// when none of the popup lands on it. + fn over(&self, view: &wgpu::Texture, popup: &wgpu::Texture, area: Area) -> Result { + let Some((from, to, size)) = clip(area, popup.size(), view.size()) else { + return Ok(view.clone()); + }; + let pipeline = FORMATS + .iter() + .position(|format| *format == view.format()) + .map(|at| &self.pipelines[at]) + .ok_or_else(|| format!("cannot draw a popup over a {:?} page", view.format()))?; + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene page with popup"), + size: view.size(), + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: view.format(), + usage: wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_DST + | wgpu::TextureUsages::RENDER_ATTACHMENT, + view_formats: &[], + }); + let offset = [from.x as i32 - to.x as i32, from.y as i32 - to.y as i32]; + let place = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("moq-scene popup"), + contents: bytemuck::cast_slice(&offset), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene popup"), + layout: &self.layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&popup.create_view(&Default::default())), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: place.as_entire_binding(), + }, + ], + }); + + let mut encoder = self.device.create_command_encoder(&Default::default()); + encoder.copy_texture_to_texture(view.as_image_copy(), texture.as_image_copy(), view.size()); + { + let target = texture.create_view(&Default::default()); + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("moq-scene popup"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &target, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(pipeline); + pass.set_bind_group(0, &bind, &[]); + // One triangle over the viewport, which is the part that lands. + pass.set_viewport( + to.x as f32, + to.y as f32, + size.width as f32, + size.height as f32, + 0.0, + 1.0, + ); + pass.draw(0..3, 0..1); + } + self.queue.submit([encoder.finish()]); + Ok(texture) + } +} /// A page's newest pictures. #[derive(Default)] @@ -36,52 +227,18 @@ impl Layers { /// What the page shows, once its view has painted: the view itself, or a /// copy of it with the popup over it, so the view stays clean for when the /// popup changes or closes. - pub fn picture(&self, device: &wgpu::Device, queue: &wgpu::Queue) -> Result, String> { + pub fn picture(&self, gpu: &Gpu) -> Result, String> { let Some(view) = &self.view else { return Ok(None) }; - let Some((area, Some(popup))) = &self.popup else { - return Ok(Some(view.clone())); - }; - let Some((from, to, size)) = clip(*area, popup.size(), view.size()) else { - return Ok(Some(view.clone())); - }; - if popup.format() != view.format() { - return Err(format!( - "the page painted its popup as {:?} over a {:?} page", - popup.format(), - view.format() - )); + match &self.popup { + Some((area, Some(popup))) => gpu.over(view, popup, *area).map(Some), + _ => Ok(Some(view.clone())), } - let texture = device.create_texture(&wgpu::TextureDescriptor { - label: Some("moq-scene page with popup"), - size: view.size(), - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: view.format(), - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - let mut encoder = device.create_command_encoder(&Default::default()); - encoder.copy_texture_to_texture(view.as_image_copy(), texture.as_image_copy(), view.size()); - encoder.copy_texture_to_texture( - wgpu::TexelCopyTextureInfo { - origin: from, - ..popup.as_image_copy() - }, - wgpu::TexelCopyTextureInfo { - origin: to, - ..texture.as_image_copy() - }, - size, - ); - queue.submit([encoder.finish()]); - Ok(Some(texture)) } } /// Where a `popup`-sized picture placed at `area` lands on a `view`-sized -/// one: the corner copied from, the corner copied to, and how much, or `None` -/// if none of it does. The area comes from the page, so it may reach past the +/// one: the corner read from, the corner drawn to, and how much, or `None` if +/// none of it does. The area comes from the page, so it may reach past the /// view, and the picture may not match it. fn clip( area: Area, @@ -133,10 +290,10 @@ mod tests { } /// A popup reaching past any edge of the view, or a picture that does not - /// match its area, copies only what lands on the view, from the matching - /// part of the picture, and one entirely off the view copies nothing. + /// match its area, draws only what lands on the view, from the matching + /// part of the picture, and one entirely off the view draws nothing. #[test] - fn a_popup_copies_only_what_lands_on_the_view() { + fn a_popup_draws_only_what_lands_on_the_view() { let view = (320, 240); assert_eq!( clipped(area(10, 20, 100, 50), (100, 50), view), @@ -168,4 +325,141 @@ mod tests { assert_eq!(clipped(area(-10, 0, 10, 10), (10, 10), view), None); assert_eq!(clipped(area(0, 0, 0, 10), (10, 10), view), None); } + + const RED: [u8; 4] = [255, 0, 0, 255]; + const BLUE: [u8; 4] = [0, 0, 255, 255]; + + /// A 4x4 texture of `format` holding `pixel(x, y)`, in RGBA. + fn texture(gpu: &Gpu, format: wgpu::TextureFormat, pixel: impl Fn(u32, u32) -> [u8; 4]) -> wgpu::Texture { + let size = extent(4, 4); + let texture = gpu.device.create_texture(&wgpu::TextureDescriptor { + label: None, + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let data: Vec = (0..4) + .flat_map(|y| (0..4).map(move |x| (x, y))) + .flat_map(|(x, y)| { + let [r, g, b, a] = pixel(x, y); + match format { + wgpu::TextureFormat::Bgra8Unorm => [b, g, r, a], + _ => [r, g, b, a], + } + }) + .collect(); + gpu.queue.write_texture( + texture.as_image_copy(), + &data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(16), + rows_per_image: None, + }, + size, + ); + texture + } + + /// A 4x4 RGBA texture's pixels, row by row. + fn read(gpu: &Gpu, texture: &wgpu::Texture) -> Vec<[u8; 4]> { + let row = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; + let buffer = gpu.device.create_buffer(&wgpu::BufferDescriptor { + label: None, + size: u64::from(row * 4), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let mut encoder = gpu.device.create_command_encoder(&Default::default()); + encoder.copy_texture_to_buffer( + texture.as_image_copy(), + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(row), + rows_per_image: None, + }, + }, + texture.size(), + ); + gpu.queue.submit([encoder.finish()]); + buffer.map_async(wgpu::MapMode::Read, .., |result| result.unwrap()); + gpu.device.poll(wgpu::PollType::wait_indefinitely()).unwrap(); + let mapped = buffer.slice(..).get_mapped_range().unwrap(); + mapped + .chunks_exact(row as usize) + .flat_map(|row| { + row[..16] + .chunks_exact(4) + .map(|pixel| pixel.try_into().unwrap()) + .collect::>() + }) + .collect() + } + + /// A popup hanging off the view's left edge draws its visible part where + /// the page placed it, premultiplied over the page: a transparent pixel + /// shows the page, a half-transparent one mixes with it. Its channel order + /// need not match the page's, and the page's own picture is untouched. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just render-sw"] + async fn a_popup_blends_over_the_page_where_placed() { + let adapter = wgpu::Instance::default() + .request_adapter(&wgpu::RequestAdapterOptions::default()) + .await + .expect("a GPU adapter"); + let (device, queue) = adapter + .request_device(&wgpu::DeviceDescriptor::default()) + .await + .expect("a GPU device"); + let gpu = Gpu::new(&device, &queue); + + let view = texture(&gpu, wgpu::TextureFormat::Rgba8Unorm, |_, _| RED); + // Column 2 transparent; column 3 half-transparent green on top, blue + // below. + let popup = texture(&gpu, wgpu::TextureFormat::Bgra8Unorm, |x, y| match (x, y) { + (0..=2, _) => [0; 4], + (_, 0) => [0, 128, 0, 128], + _ => BLUE, + }); + let mut layers = Layers::default(); + layers.paint(Layer::View, view.clone()); + layers.place(Some(area(-2, 1, 4, 4))); + layers.paint(Layer::Popup, popup); + + let shown = layers.picture(&gpu).unwrap().unwrap(); + let pixels = read(&gpu, &shown); + let at = |x: usize, y: usize| pixels[y * 4 + x]; + assert_eq!(at(0, 1), RED, "transparent shows the page"); + let mixed = at(1, 1); + assert!( + mixed[0].abs_diff(127) <= 1 && mixed[1].abs_diff(128) <= 1 && mixed[2] == 0 && mixed[3] == 255, + "half green over red: {mixed:?}" + ); + assert_eq!(at(1, 2), BLUE); + assert_eq!(at(1, 3), BLUE); + for (x, y) in [(0, 0), (1, 0), (2, 1), (3, 3), (0, 3)] { + assert_eq!( + at(x, y), + RED, + "({x}, {y}) is off the popup, or under its transparent column" + ); + } + assert!( + read(&gpu, &view).iter().all(|pixel| *pixel == RED), + "the page stays clean" + ); + + layers.place(None); + assert_eq!( + layers.picture(&gpu).unwrap().unwrap(), + view, + "a closed popup shows the page" + ); + } } diff --git a/rs/moq-scene/src/browser/run.rs b/rs/moq-scene/src/browser/run.rs index ef8e3d4..d5a1f38 100644 --- a/rs/moq-scene/src/browser/run.rs +++ b/rs/moq-scene/src/browser/run.rs @@ -14,7 +14,7 @@ use std::thread::JoinHandle; use moq_audio::bus::Channel as Sound; -use super::layers::Layers; +use super::layers::{Gpu, Layers}; use super::outbox::Commands; use super::protocol::{Area, Channel, Command, Event, Id, Layer, Open, Start}; use super::{ARG, Helper, Page, Paint}; @@ -101,9 +101,6 @@ pub(crate) struct Instance { reader: Mutex>>, } -/// The studio's device, which pages' pictures are copied to. -type Gpu<'a> = (&'a wgpu::Device, &'a wgpu::Queue); - /// Where each open page's events go. #[derive(Default)] struct Routes { @@ -143,7 +140,7 @@ impl Routes { /// Take `texture`, painted the way `paint` says, as page `id`'s newest /// `layer`, and show the page. - fn paint(&mut self, gpu: Gpu<'_>, id: Id, layer: Layer, texture: wgpu::Texture, paint: Paint) { + fn paint(&mut self, gpu: &Gpu, id: Id, layer: Layer, texture: wgpu::Texture, paint: Paint) { let Some(route) = self.pages.get_mut(&id) else { return }; if layer == Layer::View { route.page.state.lock().unwrap().paint = Some(paint); @@ -153,16 +150,16 @@ impl Routes { } /// Page `id`'s popup opened or moved to `area`, or closed. - fn place(&mut self, gpu: Gpu<'_>, id: Id, area: Option) { + fn place(&mut self, gpu: &Gpu, id: Id, area: Option) { let Some(route) = self.pages.get_mut(&id) else { return }; route.layers.place(area); self.show(gpu, id); } /// Show page `id`'s newest pictures, once it has painted. - fn show(&mut self, (device, queue): Gpu<'_>, id: Id) { + fn show(&mut self, gpu: &Gpu, id: Id) { let Some(route) = self.pages.get_mut(&id) else { return }; - let texture = match route.layers.picture(device, queue) { + let texture = match route.layers.picture(gpu) { Ok(Some(texture)) => texture, Ok(None) => return, Err(reason) => return self.end(id, reason), @@ -225,11 +222,11 @@ impl Instance { let reader = { // Not the helper, whose hold on this instance would keep it running // once every other owner is gone. - let (device, queue) = (helper.device.clone(), helper.queue.clone()); + let gpu = Gpu::new(&helper.device, &helper.queue); let (reading, routes) = (commands.clone(), routes.clone()); let spawned = std::thread::Builder::new() .name("browser-events".into()) - .spawn(move || read(&device, &queue, process, &events, &reading, &routes)); + .spawn(move || read(&gpu, process, &events, &reading, &routes)); // The writer ends with it; the helper, dropped with the closure, is // killed. spawned.map_err(|err| { @@ -390,15 +387,8 @@ fn spawn(helper: &Helper) -> std::io::Result<(Process, Channel)> { /// Deliver the helper's events until it exits or breaks the protocol, then /// stop it and every page still open, saying why. -fn read( - device: &wgpu::Device, - queue: &wgpu::Queue, - mut process: Process, - events: &Channel, - commands: &Commands, - routes: &Mutex, -) { - let reason = match deliver(device, queue, events, commands, routes) { +fn read(gpu: &Gpu, mut process: Process, events: &Channel, commands: &Commands, routes: &Mutex) { + let reason = match deliver(gpu, events, commands, routes) { // Gone, or exiting: wait for it, so the next helper finds the profile // free. Ok(()) => match process.exited() { @@ -421,15 +411,11 @@ fn read( /// Route the helper's events to their pages until it closes the socket, or /// fail with why it cannot go on. -fn deliver( - device: &wgpu::Device, - queue: &wgpu::Queue, - events: &Channel, - commands: &Commands, - routes: &Mutex, -) -> Result<(), String> { +fn deliver(gpu: &Gpu, events: &Channel, commands: &Commands, routes: &Mutex) -> Result<(), String> { use std::io::ErrorKind::*; + let Gpu { device, queue, .. } = gpu; + loop { let (event, fds) = match events.recv::() { Ok(Some(received)) => received, @@ -457,7 +443,7 @@ fn deliver( Paint::Cpu, ), Event::Popup(id, area) => { - routes.lock().unwrap().place((device, queue), id, area); + routes.lock().unwrap().place(gpu, id, area); continue; } Event::Audio(id, samples) => { @@ -478,7 +464,7 @@ fn deliver( commands.painted(); let mut routes = routes.lock().unwrap(); match texture { - Ok(texture) => routes.paint((device, queue), id, layer, texture, paint), + Ok(texture) => routes.paint(gpu, id, layer, texture, paint), Err(reason) => routes.end(id, reason), } } From 035cf8643f5f10037b4c5de9d390000a4d40cb25 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 09:33:12 -0800 Subject: [PATCH 5/5] test(scene): read back the composed page in the popup blend test Co-Authored-By: Claude Opus 5.5 --- rs/moq-scene/src/browser/layers.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/rs/moq-scene/src/browser/layers.rs b/rs/moq-scene/src/browser/layers.rs index 21a8c12..1203cf7 100644 --- a/rs/moq-scene/src/browser/layers.rs +++ b/rs/moq-scene/src/browser/layers.rs @@ -131,7 +131,9 @@ impl Gpu { sample_count: 1, dimension: wgpu::TextureDimension::D2, format: view.format(), + // A copy source like every other picture of a page. usage: wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::RENDER_ATTACHMENT, view_formats: &[], @@ -393,12 +395,7 @@ mod tests { let mapped = buffer.slice(..).get_mapped_range().unwrap(); mapped .chunks_exact(row as usize) - .flat_map(|row| { - row[..16] - .chunks_exact(4) - .map(|pixel| pixel.try_into().unwrap()) - .collect::>() - }) + .flat_map(|row| row[..16].as_chunks::<4>().0.to_vec()) .collect() }