diff --git a/README.md b/README.md index b3ad1ef..3584705 100644 --- a/README.md +++ b/README.md @@ -164,9 +164,10 @@ a restart of the studio. One studio at a time can use it: a second one's browser sources fail saying so. Each page renders in a process of its own, so a page that crashes fails only its own source, with the reason, while the stream carries on. A crash of the browser itself fails every browser source. -**Reload** in Sources restarts a page, failed or not. A page opens no other -windows (`window.open` and new-tab links do nothing), and its dropdown lists, -such as a ``'s, draw over it and take clicks under +**Interact**, but it opens no other windows (`window.open` and new-tab links +do nothing). An overlay is a title, lower third, or other graphic written as HTML and CSS, drawn by the studio itself (through [Blitz](https://github.com/DioxusLabs/blitz)) diff --git a/justfile b/justfile index 61f640f..ef0a266 100644 --- a/justfile +++ b/justfile @@ -54,7 +54,7 @@ render-sw: loader=$(nix build --inputs-from . --no-link --print-out-paths nixpkgs#vulkan-loader) icd=("$mesa"/share/vulkan/icd.d/lvp_icd.*.json) export LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" - cargo nextest run --locked -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/)' + cargo nextest run --locked -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/) | test(/^browser::layers::tests::/)' cargo nextest run --locked -p moq-studio --features openh264 --run-ignored only -E 'test(/^engine::tests::the_program_/) | test(=setup::tests::times_every_canvas)' # CEF's browser process opens an X display even to paint offscreen. xvfb=$(nix build --inputs-from . --no-link --print-out-paths nixpkgs#xvfb-run) diff --git a/quest/m0/README.md b/quest/m0/README.md index 0c85959..24aca04 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -103,7 +103,6 @@ repository. - [Audio mixes](/quest/m0/mixes.md) - per-output mixes, meters, monitoring, and sync offsets - [Audio filters](/quest/m0/audio-filters.md) - per-source noise suppression, gate, compressor, limiter, and plugins, configured from the studio - [Browser sources on macOS and Windows](/quest/m0/browser-source.md) - the out-of-process CEF pages Linux has, with shared GPU textures on the other desktops -- [Browser source popups](/quest/m0/browser-popups.md) - dropdowns and other in-page popups draw and take input - [Published independently](/quest/m0/publish-independently.md) - a source opts out of the program and publishes its own track for services that place it themselves - [Web studio](/quest/m0/web.md) - the full studio in wasm on WebGPU and WebCodecs, served from this repository - [Virtual camera](/quest/m0/vcam/README.md) - a canvas's program exposed as a system camera on Linux, Windows, and macOS diff --git a/quest/m0/browser-popups.md b/quest/m0/browser-popups.md deleted file mode 100644 index 34a2907..0000000 --- a/quest/m0/browser-popups.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] Browser source popups - -## Goal - -In-page popups of a browser source, such as ``'s list, drawn over it. + +use super::protocol::{Area, Layer}; +use crate::wgpu; +use crate::wgpu::util::DeviceExt; + +/// The formats a page arrives in: CEF's shared textures, and the uploads. +const FORMATS: [wgpu::TextureFormat; 2] = [wgpu::TextureFormat::Rgba8Unorm, wgpu::TextureFormat::Bgra8Unorm]; + +/// Each fragment of the popup's area reads the popup's pixel `offset` from +/// it, whatever order the popup's channels are in. +const SHADER: &str = " +struct Place { offset: vec2 } +@group(0) @binding(0) var popup: texture_2d; +@group(0) @binding(1) var place: Place; + +@vertex +fn vs_main(@builtin(vertex_index) index: u32) -> @builtin(position) vec4 { + let corner = vec2(f32((index << 1u) & 2u), f32(index & 2u)); + return vec4(corner * 2.0 - 1.0, 0.0, 1.0); +} + +@fragment +fn fs_main(@builtin(position) position: vec4) -> @location(0) vec4 { + return textureLoad(popup, vec2(position.xy) + place.offset, 0); +} +"; + +/// The studio's device, which pages' pictures are copied to, and the pass that +/// draws a popup over its page there. +pub(super) struct Gpu { + pub device: wgpu::Device, + pub queue: wgpu::Queue, + layout: wgpu::BindGroupLayout, + /// One per [`FORMATS`], in order. + pipelines: Vec, +} + +impl Gpu { + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Self { + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene popup"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("moq-scene popup"), + bind_group_layouts: &[Some(&layout)], + immediate_size: 0, + }); + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("moq-scene popup"), + source: wgpu::ShaderSource::Wgsl(SHADER.into()), + }); + let pipelines = FORMATS + .iter() + .map(|&format| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("moq-scene popup"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: Default::default(), + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + // Both arrive premultiplied, so a popup's translucent + // or not yet painted pixels show the page under them. + targets: &[Some(wgpu::ColorTargetState { + format, + blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }) + }) + .collect(); + Self { + device: device.clone(), + queue: queue.clone(), + layout, + pipelines, + } + } + + /// A copy of `view` with `popup` drawn over it at `area`, or `view` itself + /// when none of the popup lands on it. + fn over(&self, view: &wgpu::Texture, popup: &wgpu::Texture, area: Area) -> Result { + let Some((from, to, size)) = clip(area, popup.size(), view.size()) else { + return Ok(view.clone()); + }; + let pipeline = FORMATS + .iter() + .position(|format| *format == view.format()) + .map(|at| &self.pipelines[at]) + .ok_or_else(|| format!("cannot draw a popup over a {:?} page", view.format()))?; + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene page with popup"), + size: view.size(), + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: view.format(), + // A copy source like every other picture of a page. + usage: wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::COPY_DST + | wgpu::TextureUsages::RENDER_ATTACHMENT, + view_formats: &[], + }); + let offset = [from.x as i32 - to.x as i32, from.y as i32 - to.y as i32]; + let place = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("moq-scene popup"), + contents: bytemuck::cast_slice(&offset), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene popup"), + layout: &self.layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&popup.create_view(&Default::default())), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: place.as_entire_binding(), + }, + ], + }); + + let mut encoder = self.device.create_command_encoder(&Default::default()); + encoder.copy_texture_to_texture(view.as_image_copy(), texture.as_image_copy(), view.size()); + { + let target = texture.create_view(&Default::default()); + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("moq-scene popup"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &target, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(pipeline); + pass.set_bind_group(0, &bind, &[]); + // One triangle over the viewport, which is the part that lands. + pass.set_viewport( + to.x as f32, + to.y as f32, + size.width as f32, + size.height as f32, + 0.0, + 1.0, + ); + pass.draw(0..3, 0..1); + } + self.queue.submit([encoder.finish()]); + Ok(texture) + } +} + +/// A page's newest pictures. +#[derive(Default)] +pub(super) struct Layers { + view: Option, + /// The open popup's area of the view, and its picture once painted. + popup: Option<(Area, Option)>, +} + +impl Layers { + /// Take `texture` as the newest picture of `layer`. A popup painted while + /// none is open is dropped. + pub fn paint(&mut self, layer: Layer, texture: wgpu::Texture) { + match layer { + Layer::View => self.view = Some(texture), + Layer::Popup => { + if let Some((_, picture)) = &mut self.popup { + *picture = Some(texture); + } + } + } + } + + /// The popup opened or moved to `area`, or closed with `None`. One that + /// moved keeps its picture until it paints again. + pub fn place(&mut self, area: Option) { + let picture = self.popup.take().and_then(|(_, picture)| picture); + self.popup = area.map(|area| (area, picture)); + } + + /// What the page shows, once its view has painted: the view itself, or a + /// copy of it with the popup over it, so the view stays clean for when the + /// popup changes or closes. + pub fn picture(&self, gpu: &Gpu) -> Result, String> { + let Some(view) = &self.view else { return Ok(None) }; + match &self.popup { + Some((area, Some(popup))) => gpu.over(view, popup, *area).map(Some), + _ => Ok(Some(view.clone())), + } + } +} + +/// Where a `popup`-sized picture placed at `area` lands on a `view`-sized +/// one: the corner read from, the corner drawn to, and how much, or `None` if +/// none of it does. The area comes from the page, so it may reach past the +/// view, and the picture may not match it. +fn clip( + area: Area, + popup: wgpu::Extent3d, + view: wgpu::Extent3d, +) -> Option<(wgpu::Origin3d, wgpu::Origin3d, wgpu::Extent3d)> { + let (x, y) = (i64::from(area.x), i64::from(area.y)); + let (left, top) = (x.max(0), y.max(0)); + let right = (x + i64::from(area.width.min(popup.width))).min(view.width.into()); + let bottom = (y + i64::from(area.height.min(popup.height))).min(view.height.into()); + if left >= right || top >= bottom { + return None; + } + let origin = |x: i64, y: i64| wgpu::Origin3d { + x: x as u32, + y: y as u32, + z: 0, + }; + let size = wgpu::Extent3d { + width: (right - left) as u32, + height: (bottom - top) as u32, + depth_or_array_layers: 1, + }; + Some((origin(left - x, top - y), origin(left, top), size)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn extent(width: u32, height: u32) -> wgpu::Extent3d { + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + } + } + + fn area(x: i32, y: i32, width: u32, height: u32) -> Area { + Area { x, y, width, height } + } + + /// A copy as `((from x, from y), (to x, to y), (width, height))`. + type Copied = ((u32, u32), (u32, u32), (u32, u32)); + + fn clipped(area: Area, popup: (u32, u32), view: (u32, u32)) -> Option { + clip(area, extent(popup.0, popup.1), extent(view.0, view.1)) + .map(|(from, to, size)| ((from.x, from.y), (to.x, to.y), (size.width, size.height))) + } + + /// A popup reaching past any edge of the view, or a picture that does not + /// match its area, draws only what lands on the view, from the matching + /// part of the picture, and one entirely off the view draws nothing. + #[test] + fn a_popup_draws_only_what_lands_on_the_view() { + let view = (320, 240); + assert_eq!( + clipped(area(10, 20, 100, 50), (100, 50), view), + Some(((0, 0), (10, 20), (100, 50))) + ); + assert_eq!( + clipped(area(300, 200, 100, 100), (100, 100), view), + Some(((0, 0), (300, 200), (20, 40))) + ); + assert_eq!( + clipped(area(-30, -40, 100, 100), (100, 100), view), + Some(((30, 40), (0, 0), (70, 60))) + ); + assert_eq!( + clipped(area(i32::MIN, i32::MIN, u32::MAX, u32::MAX), (5000, 5000), view), + None, + "far off the view, however large" + ); + assert_eq!( + clipped(area(-10, 0, u32::MAX, u32::MAX), (4096, 4096), view), + Some(((10, 0), (0, 0), (320, 240))) + ); + assert_eq!( + clipped(area(10, 10, 100, 100), (40, 300), view), + Some(((0, 0), (10, 10), (40, 100))), + "the smaller of the picture and its area" + ); + assert_eq!(clipped(area(320, 0, 10, 10), (10, 10), view), None); + assert_eq!(clipped(area(-10, 0, 10, 10), (10, 10), view), None); + assert_eq!(clipped(area(0, 0, 0, 10), (10, 10), view), None); + } + + const RED: [u8; 4] = [255, 0, 0, 255]; + const BLUE: [u8; 4] = [0, 0, 255, 255]; + + /// A 4x4 texture of `format` holding `pixel(x, y)`, in RGBA. + fn texture(gpu: &Gpu, format: wgpu::TextureFormat, pixel: impl Fn(u32, u32) -> [u8; 4]) -> wgpu::Texture { + let size = extent(4, 4); + let texture = gpu.device.create_texture(&wgpu::TextureDescriptor { + label: None, + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let data: Vec = (0..4) + .flat_map(|y| (0..4).map(move |x| (x, y))) + .flat_map(|(x, y)| { + let [r, g, b, a] = pixel(x, y); + match format { + wgpu::TextureFormat::Bgra8Unorm => [b, g, r, a], + _ => [r, g, b, a], + } + }) + .collect(); + gpu.queue.write_texture( + texture.as_image_copy(), + &data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(16), + rows_per_image: None, + }, + size, + ); + texture + } + + /// A 4x4 RGBA texture's pixels, row by row. + fn read(gpu: &Gpu, texture: &wgpu::Texture) -> Vec<[u8; 4]> { + let row = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; + let buffer = gpu.device.create_buffer(&wgpu::BufferDescriptor { + label: None, + size: u64::from(row * 4), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let mut encoder = gpu.device.create_command_encoder(&Default::default()); + encoder.copy_texture_to_buffer( + texture.as_image_copy(), + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(row), + rows_per_image: None, + }, + }, + texture.size(), + ); + gpu.queue.submit([encoder.finish()]); + buffer.map_async(wgpu::MapMode::Read, .., |result| result.unwrap()); + gpu.device.poll(wgpu::PollType::wait_indefinitely()).unwrap(); + let mapped = buffer.slice(..).get_mapped_range().unwrap(); + mapped + .chunks_exact(row as usize) + .flat_map(|row| row[..16].as_chunks::<4>().0.to_vec()) + .collect() + } + + /// A popup hanging off the view's left edge draws its visible part where + /// the page placed it, premultiplied over the page: a transparent pixel + /// shows the page, a half-transparent one mixes with it. Its channel order + /// need not match the page's, and the page's own picture is untouched. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just render-sw"] + async fn a_popup_blends_over_the_page_where_placed() { + let adapter = wgpu::Instance::default() + .request_adapter(&wgpu::RequestAdapterOptions::default()) + .await + .expect("a GPU adapter"); + let (device, queue) = adapter + .request_device(&wgpu::DeviceDescriptor::default()) + .await + .expect("a GPU device"); + let gpu = Gpu::new(&device, &queue); + + let view = texture(&gpu, wgpu::TextureFormat::Rgba8Unorm, |_, _| RED); + // Column 2 transparent; column 3 half-transparent green on top, blue + // below. + let popup = texture(&gpu, wgpu::TextureFormat::Bgra8Unorm, |x, y| match (x, y) { + (0..=2, _) => [0; 4], + (_, 0) => [0, 128, 0, 128], + _ => BLUE, + }); + let mut layers = Layers::default(); + layers.paint(Layer::View, view.clone()); + layers.place(Some(area(-2, 1, 4, 4))); + layers.paint(Layer::Popup, popup); + + let shown = layers.picture(&gpu).unwrap().unwrap(); + let pixels = read(&gpu, &shown); + let at = |x: usize, y: usize| pixels[y * 4 + x]; + assert_eq!(at(0, 1), RED, "transparent shows the page"); + let mixed = at(1, 1); + assert!( + mixed[0].abs_diff(127) <= 1 && mixed[1].abs_diff(128) <= 1 && mixed[2] == 0 && mixed[3] == 255, + "half green over red: {mixed:?}" + ); + assert_eq!(at(1, 2), BLUE); + assert_eq!(at(1, 3), BLUE); + for (x, y) in [(0, 0), (1, 0), (2, 1), (3, 3), (0, 3)] { + assert_eq!( + at(x, y), + RED, + "({x}, {y}) is off the popup, or under its transparent column" + ); + } + assert!( + read(&gpu, &view).iter().all(|pixel| *pixel == RED), + "the page stays clean" + ); + + layers.place(None); + assert_eq!( + layers.picture(&gpu).unwrap().unwrap(), + view, + "a closed popup shows the page" + ); + } +} diff --git a/rs/moq-scene/src/browser/mod.rs b/rs/moq-scene/src/browser/mod.rs index b9b4cf4..15d2f7c 100644 --- a/rs/moq-scene/src/browser/mod.rs +++ b/rs/moq-scene/src/browser/mod.rs @@ -12,7 +12,9 @@ //! become one; CEF's own subprocesses start the same way. The helper paints //! into a shared GPU texture the studio copies on its device, or into memory //! the studio uploads when either side cannot share textures ([`Paint`] says -//! which). A page's sound joins the program bus at the bus's rate. +//! which). Its popups, such as a ``'s list, which CEF paints apart + /// from the page, drawn over it at the area the last [`Event::Popup`] + /// gave. + Popup, +} + +/// A rectangle of a page's view, in pixels from its top-left corner. It may +/// reach past the view. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Area { + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, +} + /// A shared texture's layout, CEF's `cef_accelerated_paint_info_t` minus the /// descriptors. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -246,7 +270,7 @@ mod tests { let file = tempfile::tempfile().unwrap(); helper.send(&Event::Audio(1, vec![0.5; 3]), &[]).unwrap(); helper - .send(&Event::Texture(1, texture.clone()), &[file.as_fd()]) + .send(&Event::Texture(1, Layer::View, texture.clone()), &[file.as_fd()]) .unwrap(); helper.send(&Event::Failed(1, "gone".into()), &[]).unwrap(); drop(helper); @@ -256,7 +280,7 @@ mod tests { assert!(fds.is_empty()); let (event, fds) = studio.recv::().unwrap().unwrap(); - assert_eq!(event, Event::Texture(1, texture)); + assert_eq!(event, Event::Texture(1, Layer::View, texture)); assert_eq!(fds.len(), 1); let ours = rustix::fs::fstat(&fds[0]).unwrap(); let theirs = rustix::fs::fstat(&file).unwrap(); @@ -280,6 +304,7 @@ mod tests { let data: Vec = (0..1920 * 1080 * 4).map(|i| i as u8).collect(); let sent = Event::Pixels { page: 1, + layer: Layer::View, width: 1920, height: 1080, data, diff --git a/rs/moq-scene/src/browser/run.rs b/rs/moq-scene/src/browser/run.rs index 2a1fd4c..d5a1f38 100644 --- a/rs/moq-scene/src/browser/run.rs +++ b/rs/moq-scene/src/browser/run.rs @@ -14,8 +14,9 @@ use std::thread::JoinHandle; use moq_audio::bus::Channel as Sound; +use super::layers::{Gpu, Layers}; use super::outbox::Commands; -use super::protocol::{Channel, Command, Event, Id, Open, Start}; +use super::protocol::{Area, Channel, Command, Event, Id, Layer, Open, Start}; use super::{ARG, Helper, Page, Paint}; use crate::Error; use crate::source::{Mailbox, Picture, Status}; @@ -47,6 +48,7 @@ pub(crate) async fn run( picture: picture.clone(), sound, status: status.clone(), + layers: Layers::default(), live: false, end: kio::Producer::new(None), }; @@ -113,6 +115,7 @@ struct Route { picture: Mailbox, sound: Option, status: kio::Producer, + layers: Layers, /// Whether a picture has arrived. live: bool, /// Why the page stopped, once it has. @@ -135,10 +138,32 @@ impl Routes { } } - /// Show `texture`, painted the way `paint` says, on page `id`. - fn show(&mut self, id: Id, texture: wgpu::Texture, paint: Paint) { + /// Take `texture`, painted the way `paint` says, as page `id`'s newest + /// `layer`, and show the page. + fn paint(&mut self, gpu: &Gpu, id: Id, layer: Layer, texture: wgpu::Texture, paint: Paint) { let Some(route) = self.pages.get_mut(&id) else { return }; - route.page.state.lock().unwrap().paint = Some(paint); + if layer == Layer::View { + route.page.state.lock().unwrap().paint = Some(paint); + } + route.layers.paint(layer, texture); + self.show(gpu, id); + } + + /// Page `id`'s popup opened or moved to `area`, or closed. + fn place(&mut self, gpu: &Gpu, id: Id, area: Option) { + let Some(route) = self.pages.get_mut(&id) else { return }; + route.layers.place(area); + self.show(gpu, id); + } + + /// Show page `id`'s newest pictures, once it has painted. + fn show(&mut self, gpu: &Gpu, id: Id) { + let Some(route) = self.pages.get_mut(&id) else { return }; + let texture = match route.layers.picture(gpu) { + Ok(Some(texture)) => texture, + Ok(None) => return, + Err(reason) => return self.end(id, reason), + }; *route.picture.lock().unwrap() = Some(Picture::Texture(texture)); if !std::mem::replace(&mut route.live, true) && let Ok(mut status) = route.status.write() @@ -197,11 +222,11 @@ impl Instance { let reader = { // Not the helper, whose hold on this instance would keep it running // once every other owner is gone. - let (device, queue) = (helper.device.clone(), helper.queue.clone()); + let gpu = Gpu::new(&helper.device, &helper.queue); let (reading, routes) = (commands.clone(), routes.clone()); let spawned = std::thread::Builder::new() .name("browser-events".into()) - .spawn(move || read(&device, &queue, process, &events, &reading, &routes)); + .spawn(move || read(&gpu, process, &events, &reading, &routes)); // The writer ends with it; the helper, dropped with the closure, is // killed. spawned.map_err(|err| { @@ -362,15 +387,8 @@ fn spawn(helper: &Helper) -> std::io::Result<(Process, Channel)> { /// Deliver the helper's events until it exits or breaks the protocol, then /// stop it and every page still open, saying why. -fn read( - device: &wgpu::Device, - queue: &wgpu::Queue, - mut process: Process, - events: &Channel, - commands: &Commands, - routes: &Mutex, -) { - let reason = match deliver(device, queue, events, commands, routes) { +fn read(gpu: &Gpu, mut process: Process, events: &Channel, commands: &Commands, routes: &Mutex) { + let reason = match deliver(gpu, events, commands, routes) { // Gone, or exiting: wait for it, so the next helper finds the profile // free. Ok(()) => match process.exited() { @@ -393,15 +411,11 @@ fn read( /// Route the helper's events to their pages until it closes the socket, or /// fail with why it cannot go on. -fn deliver( - device: &wgpu::Device, - queue: &wgpu::Queue, - events: &Channel, - commands: &Commands, - routes: &Mutex, -) -> Result<(), String> { +fn deliver(gpu: &Gpu, events: &Channel, commands: &Commands, routes: &Mutex) -> Result<(), String> { use std::io::ErrorKind::*; + let Gpu { device, queue, .. } = gpu; + loop { let (event, fds) = match events.recv::() { Ok(Some(received)) => received, @@ -409,14 +423,29 @@ fn deliver( Err(err) if matches!(err.kind(), BrokenPipe | ConnectionReset | UnexpectedEof) => return Ok(()), Err(err) => return Err(format!("the browser process's socket: {err}")), }; - let (id, texture, paint) = match event { - Event::Texture(id, texture) => (id, super::import::texture(device, queue, &texture, fds), Paint::Gpu), + let (id, layer, texture, paint) = match event { + Event::Texture(id, layer, texture) => ( + id, + layer, + super::import::texture(device, queue, &texture, fds), + Paint::Gpu, + ), Event::Pixels { page, + layer, width, height, data, - } => (page, super::upload(device, queue, width, height, &data), Paint::Cpu), + } => ( + page, + layer, + super::upload(device, queue, width, height, &data), + Paint::Cpu, + ), + Event::Popup(id, area) => { + routes.lock().unwrap().place(gpu, id, area); + continue; + } Event::Audio(id, samples) => { routes.lock().unwrap().play(id, &samples); continue; @@ -435,7 +464,7 @@ fn deliver( commands.painted(); let mut routes = routes.lock().unwrap(); match texture { - Ok(texture) => routes.show(id, texture, paint), + Ok(texture) => routes.paint(gpu, id, layer, texture, paint), Err(reason) => routes.end(id, reason), } } diff --git a/rs/moq-studio/tests/browser.rs b/rs/moq-studio/tests/browser.rs index 661e037..0b8c898 100644 --- a/rs/moq-studio/tests/browser.rs +++ b/rs/moq-studio/tests/browser.rs @@ -269,6 +269,45 @@ async fn a_click_reaches_the_page() { .await; } +/// A `\ + {options}" + ); + let mut studio = Studio::new(&[("page", page(&html))]).await; + studio.until_center([255, 0, 0, 255], &[]).await; + + let page = studio.sources.get("page").unwrap().page().unwrap().clone(); + let click = |x, y| { + for down in [true, false] { + page.send(Input::Button { + x, + y, + button: Button::Left, + down, + clicks: 1, + modifiers: Modifiers::default(), + }); + } + }; + page.send(Input::Focus(true)); + click(160, 10); + studio.until_center(GREEN, &[]).await; + click(160, 120); + studio.until_center(BLUE, &[]).await; + }) + .await; +} + /// A page that closes itself fails its own source and leaves the canvas, /// while the page beside it in the same helper keeps drawing. #[tokio::test(flavor = "multi_thread")]