From d96efd127219666964df4a386e2e563216608b41 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Fri, 21 Aug 2026 15:42:41 -0400 Subject: [PATCH 1/9] refactor(dgw): parse growing WebM with TagDecoder The original streamer treated incomplete tags as UnexpectedEOF and rewound the file. ebml-iterable 0.7 keeps those tags pending, so live recordings can append without seek/skip. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 1 + crates/video-streamer/Cargo.toml | 1 + crates/video-streamer/src/bench_support.rs | 18 +- crates/video-streamer/src/streamer/iter.rs | 498 ++++++++++----------- crates/video-streamer/src/streamer/mod.rs | 23 +- 5 files changed, 257 insertions(+), 284 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d56712658..9d86b3553 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8614,6 +8614,7 @@ version = "0.0.0" dependencies = [ "anyhow", "axum 0.8.9", + "bytes 1.12.1", "cadeau", "criterion", "ebml-iterable", diff --git a/crates/video-streamer/Cargo.toml b/crates/video-streamer/Cargo.toml index fb03371d1..857b3aecc 100644 --- a/crates/video-streamer/Cargo.toml +++ b/crates/video-streamer/Cargo.toml @@ -13,6 +13,7 @@ bench = ["perf-diagnostics"] [dependencies] anyhow = "1.0" +bytes = "1" futures-util = { version = "0.3", features = ["sink"] } tokio = { version = "1.52", features = [ "io-util", diff --git a/crates/video-streamer/src/bench_support.rs b/crates/video-streamer/src/bench_support.rs index 1ee6ff4c6..b370072b5 100644 --- a/crates/video-streamer/src/bench_support.rs +++ b/crates/video-streamer/src/bench_support.rs @@ -3,7 +3,6 @@ use std::path::Path; use std::time::{Duration, Instant}; use anyhow::Context as _; -use webm_iterable::WebmIterator; use webm_iterable::errors::TagIteratorError; use webm_iterable::matroska_spec::{Block, Master, MatroskaSpec, SimpleBlock}; @@ -71,21 +70,20 @@ where { let started_at = Instant::now(); - let mut webm_itr = WebmPositionedIterator::new(WebmIterator::new( - input_stream, - &[MatroskaSpec::BlockGroup(Master::Start)], - )); + let mut webm_itr = WebmPositionedIterator::new(input_stream, cadeau::xmf::vpx::VpxCodec::VP8); let mut headers = vec![]; - while let Some(tag) = webm_itr.next() { - let tag = tag?; - if matches!(tag, MatroskaSpec::Cluster(Master::Start)) { - break; + loop { + match webm_itr.next() { + Some(Ok(MatroskaSpec::Cluster(Master::Start))) => break, + Some(Ok(tag)) => headers.push(tag), + Some(Err(error)) => return Err(error.into()), + None => anyhow::bail!("recording ended before the first cluster"), } - headers.push(tag); } let encode_writer_config = EncodeWriterConfig::try_from((headers.as_slice(), &config))?; + webm_itr.set_codec(encode_writer_config.codec); let mut sink = CountingWriter::default(); let mut header_writer = HeaderWriter::new(&mut sink); diff --git a/crates/video-streamer/src/streamer/iter.rs b/crates/video-streamer/src/streamer/iter.rs index 60832a728..e8e36dd6b 100644 --- a/crates/video-streamer/src/streamer/iter.rs +++ b/crates/video-streamer/src/streamer/iter.rs @@ -1,15 +1,17 @@ -use std::io::Seek; +use std::io::{Read, Seek, SeekFrom}; -use anyhow::Context; +use bytes::BytesMut; use cadeau::xmf::vpx::VpxCodec; +use ebml_iterable::TagDecoder; use thiserror::Error; -use webm_iterable::WebmIterator; use webm_iterable::errors::TagIteratorError; use webm_iterable::matroska_spec::{Block, Master, MatroskaSpec, SimpleBlock}; use super::block_tag::is_vpx_key_frame; use crate::reopenable::Reopenable; +const INPUT_CHUNK_SIZE: usize = 8 * 1024; + #[derive(Debug, Clone, Copy)] pub(crate) enum LastKeyFrameInfo { NotMet { @@ -23,26 +25,16 @@ pub(crate) enum LastKeyFrameInfo { }, } -pub(crate) struct WebmPositionedIterator { - inner: Option>, - // The absolute position of the last tag emitted +pub(crate) struct WebmPositionedIterator { + reader: R, + decoder: TagDecoder, + input: BytesMut, + // Absolute file offset of the last tag emitted. previous_emitted_tag_postion: usize, - // The absolute position of the last cluster start tag emitted - last_cluster_position: Option, - - // The absolute position of the last block group/simple block that is a keyframe + // Absolute file offset of the last block group/simple block that is a keyframe. last_key_frame_info: LastKeyFrameInfo, - // The absolute position of the last tag emitted before rollback + // Absolute file offset where the current decoder's position 0 maps. rollback_record: Option, - - // When rollback at BlockGroup Full, then the Cluster(Master::end) will not be emitted - // So we need to keep track of weather we hit the cluster start and rolled back - // if so, we need to emit the cluster end tag manually - rolled_back_between_cluster: bool, - - should_emit_cache: Option, - - // VPX codec type for codec-aware keyframe detection. codec: VpxCodec, } @@ -50,8 +42,6 @@ pub(crate) struct WebmPositionedIterator { pub(crate) enum IteratorError { #[error("Inner Iterator Error: {0}")] InnerError(#[from] TagIteratorError), - #[error("Position Correction Error: {before_correct_position}")] - PositionCorrectionError { before_correct_position: u64 }, #[error("Value Expected: {0}")] ValueExpected(&'static str), #[error("IO Error: {0}")] @@ -62,201 +52,65 @@ pub(crate) enum IteratorError { impl WebmPositionedIterator where - R: std::io::Read + Seek + Reopenable, + R: Read + Seek + Reopenable, { - pub(crate) fn new(mut inner: WebmIterator, codec: VpxCodec, cluster_start_position: usize) -> Self { - inner.emit_master_end_when_eof(false); + pub(crate) fn new(reader: R, codec: VpxCodec) -> Self { Self { - inner: Some(inner), - previous_emitted_tag_postion: cluster_start_position, - last_cluster_position: Some(cluster_start_position), + reader, + decoder: new_decoder(), + input: BytesMut::new(), + previous_emitted_tag_postion: 0, rollback_record: None, - rolled_back_between_cluster: false, - should_emit_cache: None, last_key_frame_info: LastKeyFrameInfo::NotMet { cluster_timestamp: None, - cluster_start_position: Some(cluster_start_position), + cluster_start_position: None, }, codec, } } - pub(crate) fn next(&mut self) -> Option> { - let Some(inner) = self.inner.as_mut() else { - return Some(Err(IteratorError::ValueExpected("inner tag writer"))); - }; - - let result = inner.next(); - - if result.is_none() { - let record = self.rollback_record.unwrap_or(0); - if record + inner.last_emitted_tag_offset() > self.previous_emitted_tag_postion { - self.previous_emitted_tag_postion = record + inner.last_emitted_tag_offset(); - } - return None; - } - - if let Some(Ok(tag)) = &result { - let record = self.rollback_record.unwrap_or(0); - // The last emitted tag is relative, i.e when rollback, the last_emitted_tag_offset() will be reset to 0 - if record + inner.last_emitted_tag_offset() >= self.previous_emitted_tag_postion { - self.previous_emitted_tag_postion = record + inner.last_emitted_tag_offset(); - } - - if matches!(tag, MatroskaSpec::BlockGroup(Master::Full(_))) { - // we check if the tag is BlockGroup Full, - // If so, we need to correct for the last tag position - // because the full element offset will skip the header - - if let Err(e) = - self.correct_for_blockgroup_header() - .map_err(|_| IteratorError::PositionCorrectionError { - before_correct_position: self.previous_emitted_tag_postion as u64, - }) - { - return Some(Err(e)); - } - } - - if let MatroskaSpec::Timestamp(time) = tag { - match self.last_key_frame_info { - LastKeyFrameInfo::NotMet { - cluster_timestamp: ref mut potential_cluster_timestamp, - .. - } => { - potential_cluster_timestamp.replace(*time); - } - LastKeyFrameInfo::Met { - ref mut cluster_timestamp, - .. - } => { - *cluster_timestamp = *time; - } - } - - return result.map(|result| result.map_err(|err| err.into())); - } - - match self.is_key_frame(tag) { - Err(e) => { - return Some(Err(e)); - } - Ok(false) => {} - Ok(true) => { - perf_trace!( - last_tag_position = self.previous_emitted_tag_postion, - last_key_frame_info = ?self.last_key_frame_info, - "Key Frame Found" - ); - match self.last_key_frame_info { - LastKeyFrameInfo::NotMet { - cluster_timestamp, - cluster_start_position, - } => { - let Some(cluster_timestamp) = cluster_timestamp else { - return Some(Err(IteratorError::ValueExpected("cluster_timestamp"))); - }; - - let Some(cluster_start_position) = cluster_start_position else { - return Some(Err(IteratorError::ValueExpected("cluster_start_position"))); - }; - - self.last_key_frame_info = LastKeyFrameInfo::Met { - position: self.previous_emitted_tag_postion, - cluster_timestamp, - cluster_start_position, - } - } - LastKeyFrameInfo::Met { ref mut position, .. } => { - *position = self.previous_emitted_tag_postion; - } - } - } - }; - - if let Some(Ok(MatroskaSpec::Cluster(Master::Start))) = &result { - self.last_cluster_position = Some(self.previous_emitted_tag_postion); + pub(crate) fn set_codec(&mut self, codec: VpxCodec) { + self.codec = codec; + } - match self.last_key_frame_info { - LastKeyFrameInfo::NotMet { - ref mut cluster_start_position, - .. - } => { - cluster_start_position.replace(self.previous_emitted_tag_postion); - } - LastKeyFrameInfo::Met { - ref mut cluster_start_position, - .. - } => { - *cluster_start_position = self.previous_emitted_tag_postion; - } - }; - - if self.rolled_back_between_cluster { - self.should_emit_cache = Some(MatroskaSpec::Cluster(Master::Start)); - self.rolled_back_between_cluster = false; - return Some(Ok(MatroskaSpec::Cluster(Master::End))); - } else { - return result.map(|result| result.map_err(|err| err.into())); + pub(crate) fn next(&mut self) -> Option> { + loop { + match self.decoder.decode(&mut self.input) { + Ok(Some(positioned)) => { + return Some(self.observe_tag(positioned.tag, positioned.offset)); } + Ok(None) => match self.fill_input() { + Ok(0) => return None, + Ok(_) => continue, + Err(error) => return Some(Err(error.into())), + }, + Err(error) => return Some(Err(error.into())), } } - - result.map(|result| result.map_err(|err| err.into())) } - pub(crate) fn rollback_to_last_successful_tag(&mut self) -> anyhow::Result<()> { - perf_debug!( - last_tag_position = self.previous_emitted_tag_postion, - "Rolling back to last successful tag" - ); - let inner = self.inner.take().context("no inner iterator")?; - let mut file = inner.into_inner(); - file.reopen()?; - file.seek(std::io::SeekFrom::Start(self.previous_emitted_tag_postion as u64))?; - self.new_inner(file); - self.rollback_record = Some(self.previous_emitted_tag_postion); - - if self - .last_cluster_position - .map(|last_cluster_position| last_cluster_position != self.previous_emitted_tag_postion) - .unwrap_or(false) - { - self.rolled_back_between_cluster = true; - } - - Ok(()) - } - - pub(crate) fn skip(&mut self, number: u32) -> anyhow::Result<()> { - for _ in 0..number { - let _ = self.next().context("failed to skip tag")??; - } - + pub(crate) fn refresh_from_disk(&mut self) -> anyhow::Result<()> { + self.reader.reopen()?; + let absolute_read_head = self.rollback_record.unwrap_or(0) + self.decoder.position() + self.input.len(); + self.reader.seek(SeekFrom::Start(absolute_read_head.try_into()?))?; Ok(()) } pub(crate) fn rollback_to_last_key_frame(&mut self) -> Result { let LastKeyFrameInfo::Met { position: last_key_frame_position, - cluster_start_position, .. } = self.last_key_frame_info else { return Ok(self.last_key_frame_info); }; - let inner = self - .inner - .take() - .ok_or(IteratorError::ValueExpected("inner tag writer"))?; - let mut file = inner.into_inner(); - file.reopen()?; - file.seek(std::io::SeekFrom::Start(last_key_frame_position as u64))?; + self.reader.reopen()?; + self.reader.seek(SeekFrom::Start(last_key_frame_position as u64))?; + self.decoder = new_decoder(); + self.input.clear(); self.rollback_record = Some(last_key_frame_position); self.previous_emitted_tag_postion = last_key_frame_position; - self.new_inner(file); - self.last_cluster_position = Some(cluster_start_position); Ok(self.last_key_frame_info) } @@ -264,34 +118,91 @@ where self.previous_emitted_tag_postion } - // The BlockGroup element binary layout is like this - // a0 [VInt for content length] [content] - // We search for a0 [VInt for content length] from 16 bytes backward from current position - fn correct_for_blockgroup_header(&mut self) -> anyhow::Result<()> { - let file = self.inner.as_mut().context("inner is none")?.get_mut(); - let current_position = file.stream_position()?; - file.seek(std::io::SeekFrom::Start(self.previous_emitted_tag_postion.try_into()?))?; - let mut lookback_range = [0u8; 16]; - file.seek_relative(-16)?; - file.read_exact(&mut lookback_range)?; - - let mut found = false; - for i in (1..lookback_range.len()).rev() { - let slice = &lookback_range[i..]; - if slice[0] == 0xa0 && read_vint(&slice[1..]).is_ok_and(|opt| opt.is_some()) { - let trace_back_offset = 16 - i; - self.previous_emitted_tag_postion -= trace_back_offset; - found = true; - break; + fn fill_input(&mut self) -> std::io::Result { + let mut buf = [0u8; INPUT_CHUNK_SIZE]; + let read = self.reader.read(&mut buf)?; + if read > 0 { + self.input.extend_from_slice(&buf[..read]); + } + Ok(read) + } + + fn observe_tag(&mut self, tag: MatroskaSpec, relative_offset: usize) -> Result { + let record = self.rollback_record.unwrap_or(0); + let absolute_offset = record + relative_offset; + if absolute_offset >= self.previous_emitted_tag_postion { + self.previous_emitted_tag_postion = absolute_offset; + } + + if let MatroskaSpec::Timestamp(time) = tag { + match self.last_key_frame_info { + LastKeyFrameInfo::NotMet { + cluster_timestamp: ref mut potential_cluster_timestamp, + .. + } => { + potential_cluster_timestamp.replace(time); + } + LastKeyFrameInfo::Met { + ref mut cluster_timestamp, + .. + } => { + *cluster_timestamp = time; + } + } + return Ok(tag); + } + + match self.is_key_frame(&tag) { + Err(error) => return Err(error), + Ok(false) => {} + Ok(true) => { + perf_trace!( + last_tag_position = self.previous_emitted_tag_postion, + last_key_frame_info = ?self.last_key_frame_info, + "Key Frame Found" + ); + match self.last_key_frame_info { + LastKeyFrameInfo::NotMet { + cluster_timestamp, + cluster_start_position, + } => { + let Some(cluster_timestamp) = cluster_timestamp else { + return Err(IteratorError::ValueExpected("cluster_timestamp")); + }; + let Some(cluster_start_position) = cluster_start_position else { + return Err(IteratorError::ValueExpected("cluster_start_position")); + }; + self.last_key_frame_info = LastKeyFrameInfo::Met { + position: self.previous_emitted_tag_postion, + cluster_timestamp, + cluster_start_position, + }; + } + LastKeyFrameInfo::Met { ref mut position, .. } => { + *position = self.previous_emitted_tag_postion; + } + } } } - file.seek(std::io::SeekFrom::Start(current_position))?; - if !found { - anyhow::bail!("no EBML Element of BlockGroup Found"); + if matches!(tag, MatroskaSpec::Cluster(Master::Start)) { + match self.last_key_frame_info { + LastKeyFrameInfo::NotMet { + ref mut cluster_start_position, + .. + } => { + cluster_start_position.replace(self.previous_emitted_tag_postion); + } + LastKeyFrameInfo::Met { + ref mut cluster_start_position, + .. + } => { + *cluster_start_position = self.previous_emitted_tag_postion; + } + } } - Ok(()) + Ok(tag) } fn is_key_frame(&self, tag: &MatroskaSpec) -> Result { @@ -322,71 +233,136 @@ where _ => Ok(false), } } +} - fn new_inner(&mut self, reader: R) { - let mut inner = WebmIterator::new(reader, &[MatroskaSpec::BlockGroup(Master::Start)]); - // Disable automatic Master::End or Master::Start tag emission at EOF. - // - // Scenario 1 - EOF within a Cluster: - // - When we hit EOF between Cluster(Master::Start) and expected Cluster(Master::End) - // - By default, iterator emits Cluster(Master::End) automatically - // - This causes last_emitted_tag_offset() to jump back to the Cluster(Master::Start) position - // - Our position tracking becomes incorrect as it's smaller than BlockGroup/SimpleBlock we read - // - // Scenario 2 - EOF when reading from middle of a Cluster: - // - When we start reading from middle of a Cluster (after rollback/seek) - // - At EOF, iterator assumes we need a matching Start tag. - // - It emits a Cluster(Master::Start) with offset 0 - // - This resets last_emitted_tag_offset() to 0, breaking our position tracking - inner.emit_master_end_when_eof(false); - self.inner = Some(inner); - } +fn new_decoder() -> TagDecoder { + TagDecoder::new(&[MatroskaSpec::BlockGroup(Master::Start)]) } -pub(crate) fn read_vint(buffer: &[u8]) -> anyhow::Result> { - if buffer.is_empty() { - return Ok(None); - } +#[cfg(test)] +mod tests { + use std::io::{self, SeekFrom}; - if buffer[0] == 0 { - anyhow::bail!("VInt first byte cannot be 0"); - } + use webm_iterable::{WebmWriter, WriteOptions}; - let length = 8 - buffer[0].ilog2() as usize; + use super::*; - if length > buffer.len() { - // Not enough data in the buffer to read out the vint value - return Ok(None); + struct GrowingFile { + data: Vec, + pos: usize, + visible: usize, } - let mut value = u64::from(buffer[0]); - value -= 1 << (8 - length); + impl Read for GrowingFile { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.pos >= self.visible { + return Ok(0); + } + let read = (self.visible - self.pos).min(buf.len()); + buf[..read].copy_from_slice(&self.data[self.pos..self.pos + read]); + self.pos += read; + Ok(read) + } + } - for item in buffer.iter().take(length).skip(1) { - value <<= 8; - value += u64::from(*item); + impl Seek for GrowingFile { + fn seek(&mut self, from: SeekFrom) -> io::Result { + let next = match from { + SeekFrom::Start(offset) => i64::try_from(offset).map_err(io::Error::other)?, + SeekFrom::Current(offset) => i64::try_from(self.pos) + .map_err(io::Error::other)? + .checked_add(offset) + .ok_or_else(|| io::Error::other("seek overflow"))?, + SeekFrom::End(offset) => i64::try_from(self.visible) + .map_err(io::Error::other)? + .checked_add(offset) + .ok_or_else(|| io::Error::other("seek overflow"))?, + }; + if next < 0 { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "negative seek")); + } + self.pos = usize::try_from(next).map_err(io::Error::other)?; + Ok(self.pos as u64) + } } - Ok(Some((value, length))) -} + impl Reopenable for GrowingFile { + fn reopen(&mut self) -> io::Result<()> { + Ok(()) + } + } -#[cfg(test)] -mod tests { - use super::*; + fn sample_webm() -> Vec { + let mut dest = Vec::new(); + let mut writer = WebmWriter::new(&mut dest); + writer + .write(&MatroskaSpec::Ebml(Master::Start)) + .expect("write ebml start"); + writer.write(&MatroskaSpec::EbmlVersion(1)).expect("write ebml version"); + writer.write(&MatroskaSpec::Ebml(Master::End)).expect("write ebml end"); + writer + .write_advanced( + &MatroskaSpec::Segment(Master::Start), + WriteOptions::is_unknown_sized_element(), + ) + .expect("write segment start"); + writer + .write(&MatroskaSpec::Cluster(Master::Start)) + .expect("write cluster start"); + writer.write(&MatroskaSpec::Timestamp(0)).expect("write timestamp"); + writer.flush().expect("flush webm writer"); + dest + } #[test] - fn test_read_vint() { - let test_cases = vec![ - (vec![0x46, 0xa0, 0x00], Some((1696, 2))), // Single-byte VINT - (vec![0x46, 0xa0], Some((1696, 2))), // Single-byte VINT - ]; - - for (input, expected) in test_cases { - let result = read_vint(&input); - let Ok(result) = result else { - panic!("Failed to read vint"); - }; - assert_eq!(result, expected); + fn incomplete_input_waits_then_resumes_after_refresh() { + let data = sample_webm(); + assert!( + data.len() > 16, + "fixture too small ({} bytes): {:02x?}", + data.len(), + data + ); + + let mut complete = WebmPositionedIterator::new( + GrowingFile { + data: data.clone(), + pos: 0, + visible: data.len(), + }, + VpxCodec::VP8, + ); + let complete_tags = std::iter::from_fn(|| complete.next()) + .map(|tag| tag.expect("complete fixture should parse")) + .collect::>(); + assert!( + complete_tags + .iter() + .any(|tag| matches!(tag, MatroskaSpec::Cluster(Master::Start))), + "complete fixture tags: {complete_tags:?}" + ); + + let mut iter = WebmPositionedIterator::new( + GrowingFile { + data: data.clone(), + pos: 0, + visible: 4, + }, + VpxCodec::VP8, + ); + + assert!(iter.next().is_none()); + + iter.reader.visible = data.len(); + iter.refresh_from_disk().expect("refresh growing file"); + + let mut saw_cluster = false; + while let Some(tag) = iter.next() { + if matches!(tag.expect("valid tag"), MatroskaSpec::Cluster(Master::Start)) { + saw_cluster = true; + break; + } } + assert!(saw_cluster); } } diff --git a/crates/video-streamer/src/streamer/mod.rs b/crates/video-streamer/src/streamer/mod.rs index 7d4f13eb7..03a0420f5 100644 --- a/crates/video-streamer/src/streamer/mod.rs +++ b/crates/video-streamer/src/streamer/mod.rs @@ -9,7 +9,6 @@ use tag_writers::{EncodeWriterConfig, HeaderWriter, WriterResult}; use tokio::sync::{Mutex, Notify, watch}; use tokio_util::codec::Framed; use tracing::Instrument; -use webm_iterable::WebmIterator; use webm_iterable::errors::{TagIteratorError, TagWriterError}; use webm_iterable::matroska_spec::{Master, MatroskaSpec}; @@ -34,21 +33,20 @@ pub fn webm_stream( config: StreamingConfig, when_new_chunk_appended: impl Fn() -> tokio::sync::oneshot::Receiver<()>, ) -> anyhow::Result<()> { - let mut raw_itr = WebmIterator::new(input_stream, &[MatroskaSpec::BlockGroup(Master::Start)]); + let mut webm_itr = WebmPositionedIterator::new(input_stream, cadeau::xmf::vpx::VpxCodec::VP8); let mut headers = vec![]; - // we extract all the headers before the first cluster - for tag in raw_itr.by_ref() { - let tag = tag?; - if matches!(tag, MatroskaSpec::Cluster(Master::Start)) { - break; + // Extract all headers before the first cluster. + loop { + match webm_itr.next() { + Some(Ok(MatroskaSpec::Cluster(Master::Start))) => break, + Some(Ok(tag)) => headers.push(tag), + Some(Err(error)) => return Err(error.into()), + None => anyhow::bail!("recording ended before the first cluster"), } - - headers.push(tag); } let encode_writer_config = EncodeWriterConfig::try_from((headers.as_slice(), &config))?; - let cluster_start_position = raw_itr.last_emitted_tag_offset(); - let mut webm_itr = WebmPositionedIterator::new(raw_itr, encode_writer_config.codec, cluster_start_position); + webm_itr.set_codec(encode_writer_config.codec); // we run to the last cluster, skipping everything that has been played while let Some(tag) = webm_itr.next() { @@ -146,8 +144,7 @@ pub fn webm_stream( // the source unconsumed guarantees every clone will detect any pending shutdown. match when_eof(&when_new_chunk_appended, shutdown_rx.clone()) { WhenEofControlFlow::Continue => { - webm_itr.rollback_to_last_successful_tag()?; - webm_itr.skip(1)?; + webm_itr.refresh_from_disk()?; } WhenEofControlFlow::Break => { break Ok(()); From bf4e8cdc07c6df7e05ff21b276584ed50a75fba8 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Fri, 21 Aug 2026 17:11:29 -0400 Subject: [PATCH 2/9] refactor(dgw): drop leftover TagDecoder error retries TagDecoder reports incomplete input as None, not UnexpectedEOF. Retrying InvalidTagData just stalled on real corruption. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/video-streamer/src/bench_support.rs | 4 +-- crates/video-streamer/src/streamer/iter.rs | 42 +++------------------- crates/video-streamer/src/streamer/mod.rs | 26 +++++--------- 3 files changed, 14 insertions(+), 58 deletions(-) diff --git a/crates/video-streamer/src/bench_support.rs b/crates/video-streamer/src/bench_support.rs index b370072b5..35e82da70 100644 --- a/crates/video-streamer/src/bench_support.rs +++ b/crates/video-streamer/src/bench_support.rs @@ -3,12 +3,11 @@ use std::path::Path; use std::time::{Duration, Instant}; use anyhow::Context as _; -use webm_iterable::errors::TagIteratorError; use webm_iterable::matroska_spec::{Block, Master, MatroskaSpec, SimpleBlock}; use crate::StreamingConfig; use crate::reopenable::Reopenable; -use crate::streamer::iter::{IteratorError, WebmPositionedIterator}; +use crate::streamer::iter::WebmPositionedIterator; use crate::streamer::tag_writers::{EncodeWriterConfig, HeaderWriter, WriterResult}; #[derive(Debug, Clone)] @@ -154,7 +153,6 @@ where WriterResult::Continue => {} } } - Some(Err(IteratorError::InnerError(TagIteratorError::UnexpectedEOF { .. }))) => break, Some(Err(e)) => return Err(e).context("webm iterator error"), None => break, } diff --git a/crates/video-streamer/src/streamer/iter.rs b/crates/video-streamer/src/streamer/iter.rs index e8e36dd6b..26be940c5 100644 --- a/crates/video-streamer/src/streamer/iter.rs +++ b/crates/video-streamer/src/streamer/iter.rs @@ -14,26 +14,17 @@ const INPUT_CHUNK_SIZE: usize = 8 * 1024; #[derive(Debug, Clone, Copy)] pub(crate) enum LastKeyFrameInfo { - NotMet { - cluster_start_position: Option, - cluster_timestamp: Option, - }, - Met { - position: usize, - cluster_timestamp: u64, - cluster_start_position: usize, - }, + NotMet { cluster_timestamp: Option }, + Met { position: usize, cluster_timestamp: u64 }, } pub(crate) struct WebmPositionedIterator { reader: R, decoder: TagDecoder, input: BytesMut, - // Absolute file offset of the last tag emitted. previous_emitted_tag_postion: usize, - // Absolute file offset of the last block group/simple block that is a keyframe. last_key_frame_info: LastKeyFrameInfo, - // Absolute file offset where the current decoder's position 0 maps. + // File offset of decoder position 0 after the last seek. rollback_record: Option, codec: VpxCodec, } @@ -63,7 +54,6 @@ where rollback_record: None, last_key_frame_info: LastKeyFrameInfo::NotMet { cluster_timestamp: None, - cluster_start_position: None, }, codec, } @@ -162,20 +152,13 @@ where "Key Frame Found" ); match self.last_key_frame_info { - LastKeyFrameInfo::NotMet { - cluster_timestamp, - cluster_start_position, - } => { + LastKeyFrameInfo::NotMet { cluster_timestamp } => { let Some(cluster_timestamp) = cluster_timestamp else { return Err(IteratorError::ValueExpected("cluster_timestamp")); }; - let Some(cluster_start_position) = cluster_start_position else { - return Err(IteratorError::ValueExpected("cluster_start_position")); - }; self.last_key_frame_info = LastKeyFrameInfo::Met { position: self.previous_emitted_tag_postion, cluster_timestamp, - cluster_start_position, }; } LastKeyFrameInfo::Met { ref mut position, .. } => { @@ -185,23 +168,6 @@ where } } - if matches!(tag, MatroskaSpec::Cluster(Master::Start)) { - match self.last_key_frame_info { - LastKeyFrameInfo::NotMet { - ref mut cluster_start_position, - .. - } => { - cluster_start_position.replace(self.previous_emitted_tag_postion); - } - LastKeyFrameInfo::Met { - ref mut cluster_start_position, - .. - } => { - *cluster_start_position = self.previous_emitted_tag_postion; - } - } - } - Ok(tag) } diff --git a/crates/video-streamer/src/streamer/mod.rs b/crates/video-streamer/src/streamer/mod.rs index 03a0420f5..2c0769fae 100644 --- a/crates/video-streamer/src/streamer/mod.rs +++ b/crates/video-streamer/src/streamer/mod.rs @@ -1,7 +1,6 @@ use std::sync::Arc; use channel_writer::{ChannelWriter, ChannelWriterError, ChannelWriterReceiver}; -use ebml_iterable::error::CorruptedFileError; use futures_util::SinkExt; use iter::{IteratorError, WebmPositionedIterator}; use protocol::{ProtocolCodeC, UserFriendlyError}; @@ -9,7 +8,7 @@ use tag_writers::{EncodeWriterConfig, HeaderWriter, WriterResult}; use tokio::sync::{Mutex, Notify, watch}; use tokio_util::codec::Framed; use tracing::Instrument; -use webm_iterable::errors::{TagIteratorError, TagWriterError}; +use webm_iterable::errors::TagWriterError; use webm_iterable::matroska_spec::{Master, MatroskaSpec}; pub(crate) mod block_tag; @@ -48,12 +47,8 @@ pub fn webm_stream( let encode_writer_config = EncodeWriterConfig::try_from((headers.as_slice(), &config))?; webm_itr.set_codec(encode_writer_config.codec); - // we run to the last cluster, skipping everything that has been played - while let Some(tag) = webm_itr.next() { - if let Err(IteratorError::InnerError(TagIteratorError::UnexpectedEOF { .. })) = tag { - break; - } - } + // Skip already-recorded tags. Incomplete trailing input is `None`. + while let Some(Ok(_)) = webm_itr.next() {} let cut_block_position = webm_itr.previous_emitted_tag_postion(); @@ -124,18 +119,15 @@ pub fn webm_stream( let result = loop { match webm_itr.next() { - Some(Err(IteratorError::InnerError(TagIteratorError::ReadError { source }))) => { + Some(Err(IteratorError::IOError(source))) => { return Err(source.into()); } - Some(Err(IteratorError::InnerError(TagIteratorError::UnexpectedEOF { .. }))) - // Sometimes the file is not corrupted, it's just that specific tag is still on the fly - | Some(Err(IteratorError::InnerError(TagIteratorError::CorruptedFileData( - CorruptedFileError::InvalidTagData { .. }, - )))) - | None => { - perf_trace!("End of file reached or invalid tag data hit, retrying"); + None => { + perf_trace!("End of file reached, retrying"); if retry_count >= MAX_RETRY_COUNT { - anyhow::bail!("reached max retry count, the webm iterator cannot proceed with the current streaming file"); + anyhow::bail!( + "reached max retry count, the webm iterator cannot proceed with the current streaming file" + ); } retry_count += 1; From 899a8b685e5f1c554d4ea4c0fb27ee83398f71eb Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Fri, 21 Aug 2026 17:15:49 -0400 Subject: [PATCH 3/9] fix(dgw): fail the live skip when TagDecoder errors A parse error stays at the same offset. Swallowing it while draining already-recorded tags hid corruption. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/video-streamer/src/streamer/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/video-streamer/src/streamer/mod.rs b/crates/video-streamer/src/streamer/mod.rs index 2c0769fae..899d84a67 100644 --- a/crates/video-streamer/src/streamer/mod.rs +++ b/crates/video-streamer/src/streamer/mod.rs @@ -48,7 +48,13 @@ pub fn webm_stream( webm_itr.set_codec(encode_writer_config.codec); // Skip already-recorded tags. Incomplete trailing input is `None`. - while let Some(Ok(_)) = webm_itr.next() {} + loop { + match webm_itr.next() { + Some(Ok(_)) => {} + Some(Err(error)) => return Err(error.into()), + None => break, + } + } let cut_block_position = webm_itr.previous_emitted_tag_postion(); From 3eed5fa2d98b9bbe8be422608b228e4e3ca0ed01 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Fri, 21 Aug 2026 16:22:05 -0400 Subject: [PATCH 4/9] feat(dgw): stream reconnecting WebM sessions Gateway WebM shadowing now consumes a recording event stream so clips can reconnect and change size. The Start/Pull codes stay valid for existing clients. Push disconnect marks the session ended so viewers receive StreamEnded. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/video-streamer/README.md | 16 +- crates/video-streamer/src/decoder.rs | 55 ++ crates/video-streamer/src/lib.rs | 6 + crates/video-streamer/src/normalizer.rs | 639 ++++++++++++++++++ crates/video-streamer/src/protocol.rs | 440 ++++++++++++ crates/video-streamer/src/session.rs | 55 ++ .../video-streamer/src/streamer/block_tag.rs | 16 +- .../src/streamer/signal_writer.rs | 8 +- devolutions-gateway/src/api/jrec.rs | 21 +- devolutions-gateway/src/recording.rs | 227 ++++++- devolutions-gateway/src/streaming.rs | 244 +++++-- devolutions-gateway/src/ws.rs | 45 ++ 12 files changed, 1680 insertions(+), 92 deletions(-) create mode 100644 crates/video-streamer/src/decoder.rs create mode 100644 crates/video-streamer/src/normalizer.rs create mode 100644 crates/video-streamer/src/protocol.rs create mode 100644 crates/video-streamer/src/session.rs diff --git a/crates/video-streamer/README.md b/crates/video-streamer/README.md index 1151304f2..31826d8d4 100644 --- a/crates/video-streamer/README.md +++ b/crates/video-streamer/README.md @@ -1,8 +1,18 @@ # video-streamer -This crate takes an unseekable WebM recording (typically from Chrome CaptureStream) and rewrites it into a “fresh” WebM stream that can start playing immediately. -It does this by parsing the incoming WebM, finding the correct cut point, and re-encoding frames. -The output stream begins with a keyframe and valid headers. +This crate takes an unseekable WebM recording and rewrites it into a stream that can start playing immediately. + +`webm_stream` still serves one growing file over the original Start/Pull protocol. +`stream_session` accepts a multi-clip recording event stream, reconnects across clips, and emits independent VP8 WebM segments over the same Start/Pull codes. + +The input event grammar is: + +```text +(ClipStarted Bytes* CaughtUp Bytes* ClipEnded)* SessionEnded +``` + +Pulls that arrive while a response is pending are queued. +`Stream ended` (type code 3) ends the session. ## Prerequisites diff --git a/crates/video-streamer/src/decoder.rs b/crates/video-streamer/src/decoder.rs new file mode 100644 index 000000000..b8fce099e --- /dev/null +++ b/crates/video-streamer/src/decoder.rs @@ -0,0 +1,55 @@ +use anyhow::Context as _; +use cadeau::xmf::vpx::{VpxCodec, VpxDecoder, VpxImage}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct Dimensions { + pub width: u32, + pub height: u32, +} + +pub(crate) struct DecodedFrame<'decoder> { + pub image: VpxImage<'decoder>, + pub dimensions: Dimensions, +} + +pub(crate) struct InputDecoder { + codec: VpxCodec, + threads: u32, + decoder: Option, +} + +impl InputDecoder { + pub(crate) fn new(codec: VpxCodec, threads: u32) -> Self { + Self { + codec, + threads, + decoder: None, + } + } + + pub(crate) fn decode<'decoder>(&'decoder mut self, data: &[u8]) -> anyhow::Result> { + if self.decoder.is_none() { + self.decoder = Some( + VpxDecoder::builder() + .threads(self.threads) + .width(0) + .height(0) + .codec(self.codec) + .build()?, + ); + } + + let decoder = self.decoder.as_mut().context("input decoder is missing")?; + decoder.decode(data)?; + let image = decoder.next_frame()?; + let dimensions = Dimensions { + width: image.width(), + height: image.height(), + }; + anyhow::ensure!( + dimensions.width > 0 && dimensions.height > 0, + "decoder returned invalid frame dimensions" + ); + Ok(DecodedFrame { image, dimensions }) + } +} diff --git a/crates/video-streamer/src/lib.rs b/crates/video-streamer/src/lib.rs index e568689a0..04e480466 100644 --- a/crates/video-streamer/src/lib.rs +++ b/crates/video-streamer/src/lib.rs @@ -25,7 +25,11 @@ macro_rules! perf_debug { pub mod config; pub mod debug; +mod decoder; +mod normalizer; +mod protocol; pub mod reopenable; +mod session; pub(crate) mod streamer; #[macro_use] @@ -39,6 +43,8 @@ pub use streamer::reopenable_file::ReOpenableFile; pub use streamer::signal_writer::SignalWriter; #[rustfmt::skip] pub use streamer::webm_stream; +#[rustfmt::skip] +pub use session::{RecordingEvent, SessionConfig, StartAt, stream_session}; #[cfg(feature = "bench")] pub mod bench_support; diff --git a/crates/video-streamer/src/normalizer.rs b/crates/video-streamer/src/normalizer.rs new file mode 100644 index 000000000..1630bde5c --- /dev/null +++ b/crates/video-streamer/src/normalizer.rs @@ -0,0 +1,639 @@ +use std::io::{self, Write}; +use std::pin::Pin; +use std::task::{Context as TaskContext, Poll}; + +use anyhow::Context; +use bytes::{Bytes, BytesMut}; +use cadeau::xmf::vpx::{VpxCodec, VpxEncoder, VpxEncoderPreset, VpxImage}; +use ebml_iterable::TagDecoder; +use futures_util::{Stream, StreamExt}; +use tokio::sync::mpsc; +use webm_iterable::matroska_spec::{Master, MatroskaSpec, SimpleBlock}; +use webm_iterable::{WebmWriter, WriteOptions}; + +use crate::decoder::{Dimensions, InputDecoder}; +use crate::session::{RecordingEvent, SessionConfig, StartAt}; +use crate::streamer::block_tag::{VideoBlock, is_vpx_key_frame}; + +const OUTPUT_CHANNEL_CAPACITY: usize = 1; +const INPUT_CHANNEL_CAPACITY: usize = 1; +const INPUT_CHUNK_SIZE: usize = 64 * 1024; +const MAX_BUFFERED_TAG_BYTES: usize = 64 * 1024 * 1024; +const MAX_PENDING_GOP_BYTES: usize = 64 * 1024 * 1024; +const OUTPUT_BITRATE: u32 = 256 * 1024; +const VPX_EFLAG_FORCE_KF: u32 = 0x0000_0001; +const WEBM_TIMESTAMP_SCALE_NS: u64 = 1_000_000; +const MAX_WEBM_BLOCK_TIMESTAMP: u64 = 32_767; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct SegmentInfo { + pub sequence: u64, + pub width: u32, + pub height: u32, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum SegmentEvent { + Begin(SegmentInfo), + Data(Bytes), + End, +} + +pub(crate) struct NormalizedSession { + receiver: mpsc::Receiver>, + supervisor: Option>, +} + +impl Stream for NormalizedSession { + type Item = anyhow::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + self.receiver.poll_recv(cx) + } +} + +impl NormalizedSession { + pub(crate) async fn shutdown(mut self) -> anyhow::Result<()> { + self.receiver.close(); + let supervisor = self.supervisor.take().context("normalizer supervisor is missing")?; + supervisor.await.context("normalizer supervisor failed") + } +} + +impl Drop for NormalizedSession { + fn drop(&mut self) { + if let Some(supervisor) = self.supervisor.take() { + supervisor.abort(); + } + } +} + +pub(crate) fn normalize(source: S, config: SessionConfig) -> NormalizedSession +where + S: Stream> + Send + 'static, +{ + let (output_sender, output_receiver) = mpsc::channel(OUTPUT_CHANNEL_CAPACITY); + let (input_sender, input_receiver) = mpsc::channel(INPUT_CHANNEL_CAPACITY); + + let supervisor = tokio::spawn(async move { + let worker_sender = output_sender.clone(); + let mut worker = tokio::task::spawn_blocking(move || normalize_events(input_receiver, worker_sender, config)); + let mut forward = Box::pin(async move { + tokio::pin!(source); + while let Some(event) = source.next().await { + if input_sender.send(event).await.is_err() { + break; + } + } + }); + + tokio::select! { + result = &mut worker => publish_worker_result(result, &output_sender).await, + () = output_sender.closed() => { + drop(forward); + let _ = worker.await; + } + () = &mut forward => { + drop(forward); + publish_worker_result(worker.await, &output_sender).await; + } + }; + }); + + NormalizedSession { + receiver: output_receiver, + supervisor: Some(supervisor), + } +} + +async fn publish_worker_result( + result: Result, tokio::task::JoinError>, + sender: &mpsc::Sender>, +) { + let error = match result { + Ok(Ok(())) => return, + Ok(Err(error)) => error.context("session normalization failed"), + Err(error) => anyhow::Error::new(error).context("normalizer worker failed"), + }; + let _ = sender.send(Err(error)).await; +} + +fn normalize_events( + mut receiver: mpsc::Receiver>, + sender: mpsc::Sender>, + config: SessionConfig, +) -> anyhow::Result<()> { + let mut phase = SessionPhase::AwaitClip; + let mut next_segment_sequence = 0; + + while let Some(event) = receiver.blocking_recv() { + match event.context("recording source failed")? { + RecordingEvent::ClipStarted { sequence, start_at } => { + anyhow::ensure!( + matches!(phase, SessionPhase::AwaitClip), + "clip {sequence} started before the previous clip ended" + ); + phase = SessionPhase::InClip(Box::new(ClipNormalizer::new( + sequence, + start_at, + sender.clone(), + config, + next_segment_sequence, + ))); + } + RecordingEvent::Bytes(bytes) => { + let SessionPhase::InClip(clip) = &mut phase else { + anyhow::bail!("recording bytes arrived outside a clip"); + }; + clip.push(&bytes)?; + } + RecordingEvent::CaughtUp => { + let SessionPhase::InClip(clip) = &mut phase else { + anyhow::bail!("caught-up arrived outside a clip"); + }; + clip.caught_up()?; + } + RecordingEvent::ClipEnded => { + let SessionPhase::InClip(current) = std::mem::replace(&mut phase, SessionPhase::AwaitClip) else { + anyhow::bail!("clip end arrived outside a clip"); + }; + next_segment_sequence = (*current).finish()?; + } + RecordingEvent::SessionEnded => { + anyhow::ensure!( + matches!(phase, SessionPhase::AwaitClip), + "session ended before the active clip ended" + ); + phase = SessionPhase::Ended; + break; + } + } + } + + anyhow::ensure!( + matches!(phase, SessionPhase::Ended), + "recording source ended before the session end event" + ); + Ok(()) +} + +enum SessionPhase { + AwaitClip, + InClip(Box), + Ended, +} + +#[derive(Clone, Copy)] +struct SourceVideo { + track: u64, + codec: VpxCodec, +} + +struct PendingFrame { + data: Vec, + timestamp: u64, + codec: VpxCodec, + key_frame: bool, +} + +enum ClipPhase { + History(HistoryPolicy), + Live, +} + +enum HistoryPolicy { + EmitAll, + KeepLatestGop(PendingGop), +} + +#[derive(Default)] +struct PendingGop { + frames: Vec, + bytes: usize, +} + +impl PendingGop { + fn push(&mut self, frame: PendingFrame) -> anyhow::Result<()> { + if frame.key_frame { + self.frames.clear(); + self.bytes = 0; + } else if self.frames.is_empty() { + return Ok(()); + } + + let bytes = self + .bytes + .checked_add(frame.data.len()) + .context("pending GOP size overflow")?; + anyhow::ensure!(bytes <= MAX_PENDING_GOP_BYTES, "pending GOP exceeds the resource limit"); + self.frames.push(frame); + self.bytes = bytes; + Ok(()) + } +} + +struct ClipNormalizer { + clip_sequence: u64, + decoder: TagDecoder, + input: BytesMut, + source_video: Option, + cluster_timestamp: Option, + timestamp_scale_ns: u64, + phase: ClipPhase, + input_decoder: Option, + output_segment: Option, + next_segment_sequence: u64, + sender: mpsc::Sender>, + config: SessionConfig, +} + +impl ClipNormalizer { + fn new( + clip_sequence: u64, + start_at: StartAt, + sender: mpsc::Sender>, + config: SessionConfig, + next_segment_sequence: u64, + ) -> Self { + let targets = [ + MatroskaSpec::TrackEntry(Master::Start), + MatroskaSpec::BlockGroup(Master::Start), + ]; + let mut decoder = TagDecoder::new(&targets); + decoder.set_max_allowable_tag_size(Some(MAX_BUFFERED_TAG_BYTES)); + let phase = match start_at { + StartAt::Beginning => ClipPhase::History(HistoryPolicy::EmitAll), + StartAt::LiveEdge => ClipPhase::History(HistoryPolicy::KeepLatestGop(PendingGop::default())), + }; + Self { + clip_sequence, + decoder, + input: BytesMut::new(), + source_video: None, + cluster_timestamp: None, + timestamp_scale_ns: WEBM_TIMESTAMP_SCALE_NS, + phase, + input_decoder: None, + output_segment: None, + next_segment_sequence, + sender, + config, + } + } + + fn push(&mut self, bytes: &[u8]) -> anyhow::Result<()> { + for chunk in bytes.chunks(INPUT_CHUNK_SIZE) { + self.input.extend_from_slice(chunk); + while let Some(positioned) = self.decoder.decode(&mut self.input)? { + self.handle_tag(positioned.tag)?; + } + } + Ok(()) + } + + fn caught_up(&mut self) -> anyhow::Result<()> { + let history = match std::mem::replace(&mut self.phase, ClipPhase::Live) { + ClipPhase::History(history) => history, + ClipPhase::Live => anyhow::bail!("clip {} sent caught-up twice", self.clip_sequence), + }; + if let HistoryPolicy::KeepLatestGop(pending) = history { + for frame in pending.frames { + self.process_frame(frame)?; + } + } + Ok(()) + } + + fn finish(mut self) -> anyhow::Result { + anyhow::ensure!( + matches!(self.phase, ClipPhase::Live), + "clip {} ended before caught-up", + self.clip_sequence + ); + loop { + match self.decoder.decode_eof(&mut self.input)? { + Some(positioned) => self.handle_tag(positioned.tag)?, + None if self.decoder.is_finished() => break, + None => continue, + } + } + + if let Some(segment) = self.output_segment.take() { + segment.finish()?; + } + Ok(self.next_segment_sequence) + } + + fn handle_tag(&mut self, tag: MatroskaSpec) -> anyhow::Result<()> { + match tag { + MatroskaSpec::TrackEntry(Master::Full(children)) => { + if let Some(video) = parse_video_track(&children)? { + anyhow::ensure!(self.source_video.is_none(), "multiple video tracks are not supported"); + self.source_video = Some(video); + } + } + MatroskaSpec::TimestampScale(value) => self.timestamp_scale_ns = value, + MatroskaSpec::Cluster(Master::Start) => self.cluster_timestamp = None, + MatroskaSpec::Timestamp(value) => self.cluster_timestamp = Some(value), + tag @ (MatroskaSpec::SimpleBlock(_) | MatroskaSpec::BlockGroup(Master::Full(_))) => { + self.handle_block(tag)?; + } + _ => {} + } + Ok(()) + } + + fn handle_block(&mut self, tag: MatroskaSpec) -> anyhow::Result<()> { + let video = self + .source_video + .context("video track header not found before video data")?; + let block = VideoBlock::new(tag, self.cluster_timestamp, video.codec)?; + if block.track != video.track { + return Ok(()); + } + + let data = block.get_frame()?; + let key_frame = is_vpx_key_frame(&data, video.codec); + let timestamp = scale_timestamp(block.absolute_timestamp()?, self.timestamp_scale_ns)?; + let frame = PendingFrame { + data, + timestamp, + codec: video.codec, + key_frame, + }; + + match &mut self.phase { + ClipPhase::History(HistoryPolicy::KeepLatestGop(pending)) => pending.push(frame), + ClipPhase::History(HistoryPolicy::EmitAll) | ClipPhase::Live => self.process_frame(frame), + } + } + + fn process_frame(&mut self, frame: PendingFrame) -> anyhow::Result<()> { + let input_decoder = self + .input_decoder + .get_or_insert_with(|| InputDecoder::new(frame.codec, self.config.encoder_threads)); + let decoded = input_decoder.decode(&frame.data)?; + let dimensions = decoded.dimensions; + let size_changed = self + .output_segment + .as_ref() + .is_some_and(|current| current.dimensions != dimensions); + if size_changed { + self.output_segment + .take() + .context("missing active output segment")? + .finish()?; + } + + let new_segment = if self.output_segment.is_none() { + Some(SegmentInfo { + sequence: self.next_segment_sequence, + width: dimensions.width, + height: dimensions.height, + }) + } else { + None + }; + + if let Some(info) = new_segment { + self.output_segment = Some(OutputSegment::new(self.sender.clone(), info, self.config)?); + self.next_segment_sequence = self + .next_segment_sequence + .checked_add(1) + .context("segment sequence overflow")?; + } + self.output_segment + .as_mut() + .context("output segment is missing")? + .encode(&decoded.image, frame.timestamp)?; + Ok(()) + } +} + +fn parse_video_track(children: &[MatroskaSpec]) -> anyhow::Result> { + let is_video = children + .iter() + .find_map(|tag| match tag { + MatroskaSpec::TrackType(value) => Some(*value == 1), + _ => None, + }) + .unwrap_or(false); + + if !is_video { + return Ok(None); + } + + let track = children + .iter() + .find_map(|tag| match tag { + MatroskaSpec::TrackNumber(value) => Some(*value), + _ => None, + }) + .context("video track number is missing")?; + let codec_id = children + .iter() + .find_map(|tag| match tag { + MatroskaSpec::CodecID(value) => Some(value.as_str()), + _ => None, + }) + .context("video codec ID is missing")?; + let codec = match codec_id { + "V_VP8" | "vp8" => VpxCodec::VP8, + "V_VP9" | "vp9" => VpxCodec::VP9, + _ => anyhow::bail!("unsupported video codec: {codec_id}"), + }; + + Ok(Some(SourceVideo { track, codec })) +} + +fn scale_timestamp(value: u64, timestamp_scale_ns: u64) -> anyhow::Result { + let nanoseconds = u128::from(value) + .checked_mul(u128::from(timestamp_scale_ns)) + .context("video timestamp overflow")?; + u64::try_from(nanoseconds / u128::from(WEBM_TIMESTAMP_SCALE_NS)).context("video timestamp is too large") +} + +struct OutputSegment { + info: SegmentInfo, + dimensions: Dimensions, + origin_timestamp: Option, + previous_timestamp: Option, + cluster_timestamp: Option, + encoder: VpxEncoder, + writer: WebmWriter, +} + +impl OutputSegment { + fn new( + sender: mpsc::Sender>, + info: SegmentInfo, + config: SessionConfig, + ) -> anyhow::Result { + send_event(&sender, SegmentEvent::Begin(info))?; + + let encoder = VpxEncoder::builder() + .timebase_num(1) + .timebase_den(1000) + .codec(VpxCodec::VP8) + .width(info.width) + .height(info.height) + .threads(config.encoder_threads) + .bitrate(OUTPUT_BITRATE) + .preset(VpxEncoderPreset::BestPerformance) + .build()?; + let mut writer = WebmWriter::new(EventWriter { sender }); + write_header(&mut writer, info.width, info.height)?; + + Ok(Self { + info, + dimensions: Dimensions { + width: info.width, + height: info.height, + }, + origin_timestamp: None, + previous_timestamp: None, + cluster_timestamp: None, + encoder, + writer, + }) + } + + fn encode(&mut self, image: &VpxImage<'_>, timestamp: u64) -> anyhow::Result<()> { + let origin = *self.origin_timestamp.get_or_insert(timestamp); + let relative_timestamp = timestamp.saturating_sub(origin); + let duration = self + .previous_timestamp + .map_or(30, |previous| timestamp.saturating_sub(previous).max(1)); + self.previous_timestamp = Some(timestamp); + + let cluster_timestamp_expired = self.cluster_timestamp.is_some_and(|cluster_timestamp| { + relative_timestamp.saturating_sub(cluster_timestamp) > MAX_WEBM_BLOCK_TIMESTAMP + }); + let flags = if relative_timestamp == 0 || cluster_timestamp_expired { + VPX_EFLAG_FORCE_KF + } else { + 0 + }; + self.encoder.encode_frame( + image, + i64::try_from(relative_timestamp).context("relative timestamp is too large")?, + usize::try_from(duration).unwrap_or(usize::MAX), + flags, + )?; + self.write_encoded_frames() + } + + fn write_encoded_frames(&mut self) -> anyhow::Result<()> { + let frames = self + .encoder + .packet_iterator() + .filter_map(|packet| packet.frame()) + .map(|frame| { + let timestamp = u64::try_from(frame.pts()).context("encoder returned a negative timestamp")?; + let data = frame.buffer().context("encoder returned a frame without data")?; + Ok((timestamp, data)) + }) + .collect::>>()?; + + for (timestamp, data) in frames { + let is_key_frame = is_vpx_key_frame(&data, VpxCodec::VP8); + anyhow::ensure!( + self.cluster_timestamp.is_some() || is_key_frame, + "output segment does not begin with a key frame" + ); + if self.cluster_timestamp.is_none() || is_key_frame { + if self.cluster_timestamp.is_some() { + self.writer.write(&MatroskaSpec::Cluster(Master::End))?; + } + self.writer.write_advanced( + &MatroskaSpec::Cluster(Master::Start), + WriteOptions::is_unknown_sized_element(), + )?; + self.writer.write(&MatroskaSpec::Timestamp(timestamp))?; + self.cluster_timestamp = Some(timestamp); + } + + let cluster_timestamp = self.cluster_timestamp.context("output cluster timestamp is missing")?; + let block_timestamp = timestamp + .checked_sub(cluster_timestamp) + .context("output frame timestamp precedes its cluster")?; + let block_timestamp = + i16::try_from(block_timestamp).context("output cluster exceeds block timestamp range")?; + let block = SimpleBlock::new_uncheked(&data, 1, block_timestamp, false, None, false, is_key_frame); + self.writer.write(&MatroskaSpec::from(block))?; + } + + Ok(()) + } + + fn finish(mut self) -> anyhow::Result<()> { + self.encoder.flush()?; + self.write_encoded_frames()?; + if self.cluster_timestamp.is_some() { + self.writer.write(&MatroskaSpec::Cluster(Master::End))?; + } + let event_writer = self.writer.into_inner()?; + send_event(&event_writer.sender, SegmentEvent::End) + .with_context(|| format!("failed to finish segment {}", self.info.sequence)) + } +} + +fn write_header(writer: &mut WebmWriter, width: u32, height: u32) -> anyhow::Result<()> { + writer.write(&MatroskaSpec::Ebml(Master::Full(vec![ + MatroskaSpec::EbmlVersion(1), + MatroskaSpec::EbmlReadVersion(1), + MatroskaSpec::EbmlMaxIdLength(4), + MatroskaSpec::EbmlMaxSizeLength(8), + MatroskaSpec::DocType("webm".to_owned()), + MatroskaSpec::DocTypeVersion(4), + MatroskaSpec::DocTypeReadVersion(2), + ])))?; + writer.write_advanced( + &MatroskaSpec::Segment(Master::Start), + WriteOptions::is_unknown_sized_element(), + )?; + writer.write(&MatroskaSpec::Info(Master::Full(vec![ + MatroskaSpec::TimestampScale(WEBM_TIMESTAMP_SCALE_NS), + MatroskaSpec::MuxingApp("Devolutions Gateway".to_owned()), + MatroskaSpec::WritingApp("Devolutions Gateway".to_owned()), + ])))?; + writer.write(&MatroskaSpec::Tracks(Master::Full(vec![MatroskaSpec::TrackEntry( + Master::Full(vec![ + MatroskaSpec::TrackNumber(1), + MatroskaSpec::TrackUID(1), + MatroskaSpec::TrackType(1), + MatroskaSpec::FlagEnabled(1), + MatroskaSpec::FlagDefault(1), + MatroskaSpec::FlagLacing(0), + MatroskaSpec::CodecID("V_VP8".to_owned()), + MatroskaSpec::Video(Master::Full(vec![ + MatroskaSpec::PixelWidth(u64::from(width)), + MatroskaSpec::PixelHeight(u64::from(height)), + ])), + ]), + )])))?; + Ok(()) +} + +fn send_event(sender: &mpsc::Sender>, event: SegmentEvent) -> anyhow::Result<()> { + sender + .blocking_send(Ok(event)) + .map_err(|_| anyhow::anyhow!("segment event receiver closed")) +} + +struct EventWriter { + sender: mpsc::Sender>, +} + +impl Write for EventWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.sender + .blocking_send(Ok(SegmentEvent::Data(Bytes::copy_from_slice(buffer)))) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "segment event receiver closed"))?; + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/crates/video-streamer/src/protocol.rs b/crates/video-streamer/src/protocol.rs new file mode 100644 index 000000000..ea375a241 --- /dev/null +++ b/crates/video-streamer/src/protocol.rs @@ -0,0 +1,440 @@ +use std::collections::VecDeque; +use std::error::Error; +use std::pin::Pin; + +use anyhow::Context as _; +use bytes::{BufMut as _, Bytes, BytesMut}; +use futures_util::{Sink, SinkExt as _, Stream, StreamExt as _}; + +use crate::normalizer::{SegmentEvent, SegmentInfo}; + +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum ServerMessage { + Chunk(Bytes), + SegmentStarted(SegmentInfo), + Error(UserFriendlyError), + StreamEnded, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ClientMessage { + Start, + Pull, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum UserFriendlyError { + UnexpectedError, +} + +impl UserFriendlyError { + fn as_str(&self) -> &'static str { + match self { + Self::UnexpectedError => "UnexpectedError", + } + } +} + +pub(crate) async fn stream_segments(mut transport: T, segments: S) -> anyhow::Result<()> +where + T: Stream> + Sink + Unpin, + S: Stream>, + E: Error + Send + Sync + 'static, +{ + tokio::pin!(segments); + let mut expected = ClientMessage::Start; + let mut segment_state = SegmentState::AwaitingBegin; + let mut queued_pulls = VecDeque::new(); + + loop { + let message = if let Some(message) = queued_pulls.pop_front() { + Ok(message) + } else { + let Some(message) = transport.next().await else { + return Ok(()); + }; + let message = message + .map_err(anyhow::Error::new) + .context("read client stream message")?; + decode_client_message(&message) + }; + let message = match message { + Ok(message) if message == expected => message, + Ok(message) => { + debug!( + expected = ?expected, + got = ?message, + "Rejected client request in wrong state" + ); + let _ = + send_server_message(&mut transport, ServerMessage::Error(UserFriendlyError::UnexpectedError)).await; + anyhow::bail!("invalid client stream state"); + } + Err(error) => { + debug!(error = %error, "Rejected undecodable client request"); + let _ = + send_server_message(&mut transport, ServerMessage::Error(UserFriendlyError::UnexpectedError)).await; + anyhow::bail!("invalid client stream state"); + } + }; + debug!( + request = ?message, + segment_state = ?segment_state, + queued_pulls = queued_pulls.len(), + "Serving client request" + ); + + let response = match wait_for_response(&mut transport, segments.as_mut(), &mut segment_state, &mut queued_pulls) + .await + { + Ok(Some(response)) => { + debug!( + response = ?response_kind(&response), + queued_pulls = queued_pulls.len(), + "Sending server response" + ); + response + } + Ok(None) => return Ok(()), + Err(error) => { + debug!(error = format!("{error:#}"), "Request failed while waiting"); + let _ = + send_server_message(&mut transport, ServerMessage::Error(UserFriendlyError::UnexpectedError)).await; + return Err(error); + } + }; + + let ended = response == ServerMessage::StreamEnded; + send_server_message(&mut transport, response).await?; + if ended { + return Ok(()); + } + + expected = match message { + ClientMessage::Start | ClientMessage::Pull => ClientMessage::Pull, + }; + } +} + +async fn send_server_message(transport: &mut T, message: ServerMessage) -> anyhow::Result<()> +where + T: Sink + Unpin, + E: Error + Send + Sync + 'static, +{ + transport + .send(encode_server_message(message)) + .await + .map_err(anyhow::Error::new) + .context("write server stream message") +} + +fn decode_client_message(message: &[u8]) -> anyhow::Result { + match message { + [0] => Ok(ClientMessage::Start), + [1] => Ok(ClientMessage::Pull), + _ => anyhow::bail!("invalid client message"), + } +} + +fn response_kind(message: &ServerMessage) -> &'static str { + match message { + ServerMessage::Chunk(_) => "chunk", + ServerMessage::SegmentStarted(_) => "segment-started", + ServerMessage::Error(_) => "error", + ServerMessage::StreamEnded => "stream-ended", + } +} + +fn encode_server_message(message: ServerMessage) -> Bytes { + let mut encoded = BytesMut::new(); + match message { + ServerMessage::Chunk(chunk) => { + encoded.reserve(1 + chunk.len()); + encoded.put_u8(0); + encoded.put(chunk); + } + ServerMessage::SegmentStarted(info) => { + encoded.put_u8(1); + let json = format!( + "{{\"codec\":\"vp8\",\"sequence\":{},\"width\":{},\"height\":{}}}", + info.sequence, info.width, info.height + ); + encoded.put(json.as_bytes()); + } + ServerMessage::Error(error) => { + encoded.put_u8(2); + let json = format!("{{\"error\":\"{}\"}}", error.as_str()); + encoded.put(json.as_bytes()); + } + ServerMessage::StreamEnded => encoded.put_u8(3), + } + encoded.freeze() +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SegmentState { + AwaitingBegin, + Streaming, +} + +async fn wait_for_response( + transport: &mut T, + mut segments: Pin<&mut S>, + state: &mut SegmentState, + queued_pulls: &mut VecDeque, +) -> anyhow::Result> +where + T: Stream> + Unpin, + S: Stream>, + E: Error + Send + Sync + 'static, +{ + loop { + tokio::select! { + biased; + response = next_segment_message(segments.as_mut(), state) => { + return response; + } + message = transport.next() => match message { + None => return Ok(None), + Some(Ok(message)) => { + let message = decode_client_message(&message).context("decode pipelined client request")?; + if message == ClientMessage::Pull { + debug!( + queued_pulls = queued_pulls.len() + 1, + "Queued pipelined Pull while waiting for response" + ); + queued_pulls.push_back(message); + } else { + debug!( + incoming = ?message, + "Overlapping non-Pull request while waiting for response" + ); + anyhow::bail!("client sent another request before receiving a response"); + } + } + Some(Err(error)) => { + return Err(anyhow::Error::new(error).context("read client stream message")); + } + }, + } + } +} + +async fn next_segment_message( + mut segments: Pin<&mut S>, + state: &mut SegmentState, +) -> anyhow::Result> +where + S: Stream>, +{ + loop { + let Some(event) = segments.as_mut().next().await else { + anyhow::ensure!( + *state == SegmentState::AwaitingBegin, + "segment stream ended inside a segment" + ); + return Ok(Some(ServerMessage::StreamEnded)); + }; + + match event? { + SegmentEvent::Begin(info) => { + anyhow::ensure!( + *state == SegmentState::AwaitingBegin, + "segment began before the previous segment ended" + ); + *state = SegmentState::Streaming; + debug!( + sequence = info.sequence, + width = info.width, + height = info.height, + "Segment begin" + ); + return Ok(Some(ServerMessage::SegmentStarted(info))); + } + SegmentEvent::Data(data) => { + anyhow::ensure!( + *state == SegmentState::Streaming, + "segment data arrived outside a segment" + ); + debug!(bytes = data.len(), "Segment data"); + return Ok(Some(ServerMessage::Chunk(data))); + } + SegmentEvent::End => { + anyhow::ensure!(*state == SegmentState::Streaming, "segment ended outside a segment"); + *state = SegmentState::AwaitingBegin; + debug!("Segment end"); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use futures_util::{StreamExt as _, stream}; + + use super::*; + + fn pending_after( + messages: impl IntoIterator>, + ) -> impl Stream> + Unpin { + stream::iter(messages).chain(stream::pending()) + } + + #[test] + fn protocol_codes_are_stable() { + assert_eq!( + encode_server_message(ServerMessage::SegmentStarted(SegmentInfo { + sequence: 7, + width: 1920, + height: 1080, + })), + Bytes::from_static(b"\x01{\"codec\":\"vp8\",\"sequence\":7,\"width\":1920,\"height\":1080}") + ); + assert_eq!( + encode_server_message(ServerMessage::Chunk(Bytes::from_static(b"webm"))), + Bytes::from_static(b"\x00webm") + ); + assert_eq!( + encode_server_message(ServerMessage::StreamEnded), + Bytes::from_static(b"\x03") + ); + } + + #[test] + fn client_messages_require_one_complete_transport_message() { + assert_eq!( + decode_client_message(b"\x00").expect("decode start"), + ClientMessage::Start + ); + assert_eq!( + decode_client_message(b"\x01").expect("decode pull"), + ClientMessage::Pull + ); + assert!(decode_client_message(b"\x00\x01").is_err()); + assert!(decode_client_message(b"").is_err()); + } + + #[tokio::test] + async fn segment_end_is_implicit_on_the_wire() { + let events = [ + Ok(SegmentEvent::Begin(SegmentInfo { + sequence: 0, + width: 640, + height: 480, + })), + Ok(SegmentEvent::Data(Bytes::from_static(b"first"))), + Ok(SegmentEvent::End), + Ok(SegmentEvent::Begin(SegmentInfo { + sequence: 1, + width: 800, + height: 600, + })), + Ok(SegmentEvent::Data(Bytes::from_static(b"second"))), + Ok(SegmentEvent::End), + ]; + let segments = stream::iter(events); + tokio::pin!(segments); + let mut state = SegmentState::AwaitingBegin; + + assert!(matches!( + next_segment_message(segments.as_mut(), &mut state) + .await + .expect("first begin"), + Some(ServerMessage::SegmentStarted(SegmentInfo { sequence: 0, .. })) + )); + assert_eq!( + next_segment_message(segments.as_mut(), &mut state) + .await + .expect("first data"), + Some(ServerMessage::Chunk(Bytes::from_static(b"first"))) + ); + assert!(matches!( + next_segment_message(segments.as_mut(), &mut state) + .await + .expect("second begin"), + Some(ServerMessage::SegmentStarted(SegmentInfo { sequence: 1, .. })) + )); + assert_eq!( + next_segment_message(segments.as_mut(), &mut state) + .await + .expect("second data"), + Some(ServerMessage::Chunk(Bytes::from_static(b"second"))) + ); + assert_eq!( + next_segment_message(segments.as_mut(), &mut state) + .await + .expect("stream end"), + Some(ServerMessage::StreamEnded) + ); + } + + #[tokio::test] + async fn start_response_buffers_one_pipelined_pull() { + let mut transport = pending_after([Ok::<_, std::io::Error>(Bytes::from_static(b"\x01"))]); + let segments = stream::once(async { + tokio::task::yield_now().await; + Ok(SegmentEvent::Begin(SegmentInfo { + sequence: 0, + width: 640, + height: 480, + })) + }); + tokio::pin!(segments); + let mut state = SegmentState::AwaitingBegin; + let mut queued_pulls = VecDeque::new(); + + let response = wait_for_response(&mut transport, segments.as_mut(), &mut state, &mut queued_pulls) + .await + .expect("wait for start response") + .expect("segment response"); + + assert!(matches!( + response, + ServerMessage::SegmentStarted(SegmentInfo { sequence: 0, .. }) + )); + assert_eq!(queued_pulls, VecDeque::from([ClientMessage::Pull])); + } + + #[tokio::test] + async fn extra_pull_while_waiting_for_chunk_is_queued() { + let mut transport = pending_after([Ok::<_, std::io::Error>(Bytes::from_static(b"\x01"))]); + let segments = stream::once(async { + tokio::task::yield_now().await; + Ok(SegmentEvent::Data(Bytes::from_static(b"chunk"))) + }); + tokio::pin!(segments); + let mut state = SegmentState::Streaming; + let mut queued_pulls = VecDeque::new(); + + let response = wait_for_response(&mut transport, segments.as_mut(), &mut state, &mut queued_pulls) + .await + .expect("wait for chunk") + .expect("chunk response"); + + assert_eq!(response, ServerMessage::Chunk(Bytes::from_static(b"chunk"))); + assert_eq!(queued_pulls, VecDeque::from([ClientMessage::Pull])); + } + + #[tokio::test] + async fn extra_start_while_waiting_is_still_rejected() { + let mut transport = pending_after([Ok::<_, std::io::Error>(Bytes::from_static(b"\x00"))]); + let segments = stream::once(async { + tokio::task::yield_now().await; + Ok(SegmentEvent::Data(Bytes::from_static(b"chunk"))) + }); + tokio::pin!(segments); + let mut state = SegmentState::Streaming; + let mut queued_pulls = VecDeque::new(); + + let error = wait_for_response(&mut transport, segments.as_mut(), &mut state, &mut queued_pulls) + .await + .expect_err("overlapping Start must fail"); + + assert!( + format!("{error:#}").contains("client sent another request before receiving a response"), + "{error:#}" + ); + } +} diff --git a/crates/video-streamer/src/session.rs b/crates/video-streamer/src/session.rs new file mode 100644 index 000000000..a8711b36f --- /dev/null +++ b/crates/video-streamer/src/session.rs @@ -0,0 +1,55 @@ +use std::error::Error; + +use bytes::Bytes; +use futures_util::{Sink, Stream}; + +/// A structural event from one append-only recording session. +/// +/// A clip starts, receives zero or more byte events, catches up exactly once, receives more bytes, +/// and ends before another clip starts. +/// The session ends only when no clip is active. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum RecordingEvent { + ClipStarted { sequence: u64, start_at: StartAt }, + Bytes(Bytes), + CaughtUp, + ClipEnded, + SessionEnded, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StartAt { + Beginning, + LiveEdge, +} + +#[derive(Clone, Copy, Debug)] +pub struct SessionConfig { + pub encoder_threads: u32, +} + +impl Default for SessionConfig { + fn default() -> Self { + Self { + encoder_threads: u32::try_from(num_cpus::get()).unwrap_or(1).max(1), + } + } +} + +/// Converts a recording session into fixed-size VP8 WebM segments over one pull-driven stream. +pub async fn stream_session(source: S, transport: T, config: SessionConfig) -> anyhow::Result<()> +where + S: Stream> + Send + 'static, + T: Stream> + Sink + Unpin, + E: Error + Send + Sync + 'static, +{ + let mut segments = crate::normalizer::normalize(source, config); + let stream_result = crate::protocol::stream_segments(transport, &mut segments).await; + let shutdown_result = segments.shutdown().await; + + match (stream_result, shutdown_result) { + (Err(error), _) => Err(error), + (Ok(()), Err(error)) => Err(error), + (Ok(()), Ok(())) => Ok(()), + } +} diff --git a/crates/video-streamer/src/streamer/block_tag.rs b/crates/video-streamer/src/streamer/block_tag.rs index 483dfc108..769657ed6 100644 --- a/crates/video-streamer/src/streamer/block_tag.rs +++ b/crates/video-streamer/src/streamer/block_tag.rs @@ -12,6 +12,7 @@ pub(crate) enum BlockTag { #[derive(Clone)] pub(crate) struct VideoBlock { + pub(crate) track: u64, pub(crate) cluster_timestamp: Option, pub(crate) timestamp: i16, pub(crate) is_key_frame: bool, @@ -22,6 +23,7 @@ impl fmt::Debug for VideoBlock { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("VideoBlock") .field("cluster_timestamp", &self.cluster_timestamp) + .field("track", &self.track) .field("timestamp", &self.timestamp) .field("is_key_frame", &self.is_key_frame) .field( @@ -58,6 +60,7 @@ impl VideoBlock { .any(|frame| is_vpx_key_frame(frame.data, codec)); Self { + track: block.track, cluster_timestamp, block_tag: BlockTag::BlockGroup(children), timestamp, @@ -67,6 +70,7 @@ impl VideoBlock { MatroskaSpec::SimpleBlock(data) => { let simple_block = SimpleBlock::try_from(&data)?; Self { + track: simple_block.track, cluster_timestamp, timestamp: simple_block.timestamp, is_key_frame: simple_block.keyframe, @@ -80,11 +84,13 @@ impl VideoBlock { } pub(crate) fn absolute_timestamp(&self) -> anyhow::Result { - let timestamp = u64::try_from(self.timestamp)?; - Ok(self + let cluster_timestamp = self .cluster_timestamp - .with_context(|| format!("Cluster timestamp not found for timestamp: {}", self.timestamp))? - + timestamp) + .with_context(|| format!("Cluster timestamp not found for timestamp: {}", self.timestamp))?; + let timestamp = i64::try_from(cluster_timestamp)? + .checked_add(i64::from(self.timestamp)) + .context("block timestamp overflow")?; + u64::try_from(timestamp).context("negative absolute block timestamp") } // We only handle non-lacing frames for now @@ -120,7 +126,7 @@ impl VideoBlock { } }; - assert!(frame.len() == 1); + anyhow::ensure!(frame.len() == 1, "laced video blocks are not supported"); Ok(frame[0].clone()) } } diff --git a/crates/video-streamer/src/streamer/signal_writer.rs b/crates/video-streamer/src/streamer/signal_writer.rs index e66af86ff..21bdac1cd 100644 --- a/crates/video-streamer/src/streamer/signal_writer.rs +++ b/crates/video-streamer/src/streamer/signal_writer.rs @@ -23,7 +23,11 @@ where cx: &mut std::task::Context<'_>, buf: &[u8], ) -> Poll> { - tokio::io::AsyncWrite::poll_write(std::pin::Pin::new(&mut self.writer), cx, buf) + let result = tokio::io::AsyncWrite::poll_write(std::pin::Pin::new(&mut self.writer), cx, buf); + if matches!(&result, Poll::Ready(Ok(written)) if *written > 0) { + self.notify.notify_one(); + } + result } fn poll_flush( @@ -34,7 +38,7 @@ where return Poll::Pending; }; - self.notify.notify_waiters(); + self.notify.notify_one(); Poll::Ready(res) } diff --git a/devolutions-gateway/src/api/jrec.rs b/devolutions-gateway/src/api/jrec.rs index b010f13a6..18df871e4 100644 --- a/devolutions-gateway/src/api/jrec.rs +++ b/devolutions-gateway/src/api/jrec.rs @@ -949,7 +949,11 @@ impl From for CloseFrame { } async fn shadow_recording( - State(DgwState { recordings, .. }): State, + State(DgwState { + recordings, + shutdown_signal, + .. + }): State, extract::Path(id): extract::Path, JrecToken(claims): JrecToken, ws: WebSocketUpgrade, @@ -962,31 +966,22 @@ async fn shadow_recording( return close_with_error(ws, StreamerCloseCode::StreamingEnded); } - let Ok(Some(crate::recording::OnGoingRecordingState::Connected)) = recordings.get_state(id).await else { - return close_with_error(ws, StreamerCloseCode::StreamingEnded); - }; - if !xmf::is_init() { warn!(%id, "Shadow recording rejected: XMF native library is not loaded"); return close_with_error(ws, StreamerCloseCode::InternalError); } - let Ok(notify) = recordings.subscribe_to_recording_finish(id).await else { - warn!(%id, "Shadow recording rejected: failed to subscribe to recording finish"); - return close_with_error(ws, StreamerCloseCode::InternalError); - }; - let Ok(recording_files) = recordings.list_files(id).await else { warn!(%id, "Shadow recording rejected: failed to list recording files"); return close_with_error(ws, StreamerCloseCode::InternalError); }; - let Some(recording_path) = recording_files.last() else { + if recording_files.is_empty() { warn!(%id, "Shadow recording rejected: no recording files found"); return close_with_error(ws, StreamerCloseCode::InternalError); - }; + } - return crate::streaming::stream_file(recording_path, ws, notify, recordings, id) + return crate::streaming::stream_recording(ws, shutdown_signal, recordings, id) .await .map_err(|_| HttpError::internal().msg("failed to stream file")); diff --git a/devolutions-gateway/src/recording.rs b/devolutions-gateway/src/recording.rs index f07913868..b68bc338c 100644 --- a/devolutions-gateway/src/recording.rs +++ b/devolutions-gateway/src/recording.rs @@ -14,7 +14,7 @@ use futures::future::Either; use parking_lot::Mutex; use serde::Serialize; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, BufWriter}; -use tokio::sync::{Notify, mpsc, oneshot}; +use tokio::sync::{Notify, mpsc, oneshot, watch}; use tokio::{fs, io}; use typed_builder::TypedBuilder; use uuid::Uuid; @@ -132,6 +132,7 @@ where let res = match open_options.open(&recording_file).await { Ok(file) => { + recordings.clip_started(session_id).await?; // Wrap SignalWriter inside a BufWriter to reduce the number of flushes. let (file, flush_signal) = SignalWriter::new(file); // larger buffer size to reduce the number of flushes @@ -144,7 +145,7 @@ where loop { tokio::select! { _ = flush_signal.notified() => { - recordings.new_chunk_appended(session_id)?; + recordings.new_chunk_appended(session_id).await?; }, _ = shutdown_signal_clone.wait() => { break; @@ -173,8 +174,22 @@ where }; signal_loop.abort(); + let _ = signal_loop.await; - res + let flush_result = file.flush().await; + if flush_result.is_ok() { + recordings.new_chunk_appended(session_id).await?; + } + + match (res, flush_result) { + (Err(error), _) => Err(error), + (Ok(_), Err(error)) if is_storage_full(&error) => { + warn!(%session_id, "Recording storage is full; closing push stream"); + Ok(PushOutcome::StorageFull) + } + (Ok(_), Err(error)) => Err(anyhow::Error::new(error).context("flush JREC recording file")), + (Ok(outcome), Ok(())) => Ok(outcome), + } } Err(e) => Err(anyhow::Error::new(e).context(format!("failed to open file at {recording_file}"))), }; @@ -241,6 +256,27 @@ struct OnGoingRecording { manifest_path: Utf8PathBuf, session_must_be_recorded: bool, disconnected_ttl: Duration, + stream_state: watch::Sender, +} + +#[derive(Clone, Debug)] +pub(crate) struct RecordingStreamClip { + pub(crate) sequence: u64, + pub(crate) path: Utf8PathBuf, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ActiveRecordingStreamClip { + pub(crate) sequence: u64, + pub(crate) ready: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct RecordingStreamState { + pub(crate) clips: Arc>, + pub(crate) active: Option, + pub(crate) ended: bool, + revision: u64, } enum RecordingManagerMessage { @@ -253,6 +289,12 @@ enum RecordingManagerMessage { Disconnect { id: Uuid, }, + ClipStarted { + id: Uuid, + }, + ChunkAppended { + id: Uuid, + }, GetState { id: Uuid, channel: oneshot::Sender>, @@ -272,6 +314,10 @@ enum RecordingManagerMessage { id: Uuid, channel: oneshot::Sender>, }, + SubscribeToStream { + id: Uuid, + channel: oneshot::Sender>, + }, } impl fmt::Debug for RecordingManagerMessage { @@ -289,6 +335,8 @@ impl fmt::Debug for RecordingManagerMessage { .field("disconnected_ttl", disconnected_ttl) .finish_non_exhaustive(), RecordingManagerMessage::Disconnect { id } => f.debug_struct("Disconnect").field("id", id).finish(), + RecordingManagerMessage::ClipStarted { id } => f.debug_struct("ClipStarted").field("id", id).finish(), + RecordingManagerMessage::ChunkAppended { id } => f.debug_struct("ChunkAppended").field("id", id).finish(), RecordingManagerMessage::GetState { id, channel: _ } => { f.debug_struct("GetState").field("id", id).finish_non_exhaustive() } @@ -307,6 +355,10 @@ impl fmt::Debug for RecordingManagerMessage { RecordingManagerMessage::ListFiles { id, channel: _ } => { f.debug_struct("ListFiles").field("id", id).finish() } + RecordingManagerMessage::SubscribeToStream { id, channel: _ } => f + .debug_struct("SubscribeToStream") + .field("id", id) + .finish_non_exhaustive(), } } } @@ -386,18 +438,28 @@ impl RecordingMessageSender { senders.push(tx); } - pub(crate) fn new_chunk_appended(&self, recording_id: Uuid) -> anyhow::Result<()> { - let senders = { self.flush_map.lock().remove(&recording_id) }; + async fn clip_started(&self, recording_id: Uuid) -> anyhow::Result<()> { + self.channel + .send(RecordingManagerMessage::ClipStarted { id: recording_id }) + .await + .ok() + .context("couldn't send ClipStarted message") + } - let Some(senders) = senders else { - return Ok(()); - }; + pub(crate) async fn new_chunk_appended(&self, recording_id: Uuid) -> anyhow::Result<()> { + let senders = { self.flush_map.lock().remove(&recording_id) }; - for tx in senders { - let _ = tx.send(()); + if let Some(senders) = senders { + for tx in senders { + let _ = tx.send(()); + } } - Ok(()) + self.channel + .send(RecordingManagerMessage::ChunkAppended { id: recording_id }) + .await + .ok() + .context("couldn't send ChunkAppended message") } pub(crate) async fn subscribe_to_recording_finish(&self, recording_id: Uuid) -> anyhow::Result> { @@ -411,6 +473,20 @@ impl RecordingMessageSender { Ok(rx.await?) } + pub(crate) async fn subscribe_to_stream( + &self, + recording_id: Uuid, + ) -> anyhow::Result> { + let (tx, rx) = oneshot::channel(); + self.channel + .send(RecordingManagerMessage::SubscribeToStream { + id: recording_id, + channel: tx, + }) + .await?; + Ok(rx.await?) + } + pub(crate) async fn list_files(&self, recording_id: Uuid) -> anyhow::Result> { let (tx, rx) = oneshot::channel(); self.channel @@ -516,6 +592,10 @@ impl RecordingManagerTask { anyhow::bail!("concurrent recording for the same session is not supported"); } + let existing_stream_state = self + .ongoing_recordings + .get(&id) + .map(|ongoing| ongoing.stream_state.clone()); let recording_path = self.recordings_path.join(id.to_string()); let manifest_path = recording_path.join("recording.json"); @@ -588,6 +668,45 @@ impl RecordingManagerTask { .map(|info| info.recording_policy) .unwrap_or(false); + let sequence = manifest + .files + .len() + .checked_sub(1) + .context("recording manifest has no files")?; + let sequence = u64::try_from(sequence).context("recording sequence does not fit in u64")?; + let clip = RecordingStreamClip { + sequence, + path: recording_file.clone(), + }; + let stream_state = if let Some(stream_state) = existing_stream_state { + stream_state.send_modify(|state| { + Arc::make_mut(&mut state.clips).push(clip.clone()); + state.active = Some(ActiveRecordingStreamClip { sequence, ready: false }); + state.ended = false; + state.revision = state.revision.saturating_add(1); + }); + stream_state + } else { + let clips = manifest + .files + .iter() + .enumerate() + .map(|(sequence, file)| { + Ok(RecordingStreamClip { + sequence: u64::try_from(sequence).context("recording sequence does not fit in u64")?, + path: recording_path.join(&file.file_name), + }) + }) + .collect::>>()?; + let state = RecordingStreamState { + clips: Arc::new(clips), + active: Some(ActiveRecordingStreamClip { sequence, ready: false }), + ended: false, + revision: 0, + }; + watch::channel(state).0 + }; + self.ongoing_recordings.insert( id, OnGoingRecording { @@ -596,6 +715,7 @@ impl RecordingManagerTask { manifest_path, session_must_be_recorded, disconnected_ttl, + stream_state, }, ); let ongoing_recording_count = self.ongoing_recordings.len(); @@ -612,6 +732,54 @@ impl RecordingManagerTask { Ok(recording_file) } + fn handle_clip_started(&mut self, id: Uuid) -> anyhow::Result<()> { + let ongoing = self + .ongoing_recordings + .get(&id) + .with_context(|| format!("unknown recording for ID {id}"))?; + let active = ongoing + .stream_state + .borrow() + .active + .context("recording has no active clip")?; + + if !matches!(ongoing.state, OnGoingRecordingState::Connected) || active.ready { + anyhow::bail!("recording clip can’t be started in its current state"); + } + + ongoing.stream_state.send_modify(|state| { + state.active = Some(ActiveRecordingStreamClip { + sequence: active.sequence, + ready: true, + }); + state.revision = state.revision.saturating_add(1); + }); + + Ok(()) + } + + fn handle_chunk_appended(&mut self, id: Uuid) -> anyhow::Result<()> { + let ongoing = self + .ongoing_recordings + .get(&id) + .with_context(|| format!("unknown recording for ID {id}"))?; + let active = ongoing + .stream_state + .borrow() + .active + .context("recording has no active clip")?; + + if !active.ready { + anyhow::bail!("recording clip is not ready"); + } + + ongoing.stream_state.send_modify(|state| { + state.revision = state.revision.saturating_add(1); + }); + + Ok(()) + } + async fn handle_disconnect(&mut self, id: Uuid) -> anyhow::Result<()> { let Some(ongoing) = self.ongoing_recordings.get_mut(&id) else { return Err(anyhow::anyhow!("unknown recording for ID {id}")); @@ -647,6 +815,12 @@ impl RecordingManagerTask { .save_to_file(&ongoing.manifest_path) .with_context(|| format!("write manifest at {}", ongoing.manifest_path))?; + ongoing.stream_state.send_modify(|state| { + state.active = None; + state.ended = true; + state.revision = state.revision.saturating_add(1); + }); + // Notify all the streamers that recording has ended. if let Some(notify) = self.recording_end_notifier.get(&id) { notify.notify_waiters(); @@ -686,6 +860,11 @@ impl RecordingManagerTask { OnGoingRecordingState::LastSeen { timestamp } if now >= timestamp + disconnected_ttl_secs - 1 => { debug!(%id, "Mark recording as terminated"); self.rx.active_recordings.remove(id); + ongoing.stream_state.send_modify(|state| { + state.active = None; + state.ended = true; + state.revision = state.revision.saturating_add(1); + }); // Check the recording policy of the associated session and kill it if necessary. if ongoing.session_must_be_recorded { @@ -745,6 +924,14 @@ impl RecordingManagerTask { Ok(notify) } } + + fn subscribe_stream(&self, id: Uuid) -> anyhow::Result> { + let ongoing = self + .ongoing_recordings + .get(&id) + .with_context(|| format!("unknown recording for ID {id}"))?; + Ok(ongoing.stream_state.subscribe()) + } } #[async_trait] @@ -822,6 +1009,16 @@ async fn recording_manager_task( } } } + RecordingManagerMessage::ClipStarted { id } => { + if let Err(error) = manager.handle_clip_started(id) { + error!(%error, "handle_clip_started"); + } + } + RecordingManagerMessage::ChunkAppended { id } => { + if let Err(error) = manager.handle_chunk_appended(id) { + error!(%error, "handle_chunk_appended"); + } + } RecordingManagerMessage::GetState { id, channel } => { let response = manager.ongoing_recordings.get(&id).map(|ongoing| ongoing.state.clone()); let _ = channel.send(response); @@ -847,6 +1044,14 @@ async fn recording_manager_task( Err(e) => error!(error = format!("{e:#}"), "subscribe to session end notification"), } }, + RecordingManagerMessage::SubscribeToStream { id, channel } => { + match manager.subscribe_stream(id) { + Ok(stream) => { + let _ = channel.send(stream); + } + Err(error) => error!(%error, "subscribe to recording stream"), + } + } RecordingManagerMessage::ListFiles { id, channel } => { match manager.ongoing_recordings.get(&id) { Some(recording) => { diff --git a/devolutions-gateway/src/streaming.rs b/devolutions-gateway/src/streaming.rs index b62f4994c..4bc280cb4 100644 --- a/devolutions-gateway/src/streaming.rs +++ b/devolutions-gateway/src/streaming.rs @@ -5,35 +5,43 @@ use anyhow::Context; use axum::body::Body; use axum::extract::ws::{CloseFrame, Utf8Bytes, WebSocket}; use axum::response::Response; -use futures::SinkExt; +use bytes::Bytes; +use devolutions_gateway_task::ShutdownSignal; +use futures::{SinkExt, Stream, stream}; use terminal_streamer::terminal_stream; -use tokio::fs::OpenOptions; -use tokio::sync::Notify; +use tokio::fs::{File, OpenOptions}; +use tokio::io::AsyncReadExt; +use tokio::sync::{Notify, watch}; use uuid::Uuid; -use video_streamer::config::CpuCount; -use video_streamer::{ReOpenableFile, webm_stream}; +use video_streamer::{RecordingEvent, SessionConfig, StartAt, stream_session}; +use crate::recording::{RecordingMessageSender, RecordingStreamState}; use crate::token::RecordingFileType; -pub(crate) async fn stream_file( - path: &camino::Utf8Path, +pub(crate) async fn stream_recording( ws: axum::extract::WebSocketUpgrade, - shutdown_notify: Arc, - recordings: crate::recording::RecordingMessageSender, + shutdown_signal: ShutdownSignal, + recordings: RecordingMessageSender, recording_id: Uuid, ) -> anyhow::Result> { - let streaming_type = validate_streaming_file(path).await?; - - let when_new_chunk_appended = move || { - let (tx, rx) = tokio::sync::oneshot::channel(); - recordings.add_new_chunk_listener(recording_id, tx); - rx - }; - - let path = Arc::new(path.to_owned()); + let stream_state = recordings.subscribe_to_stream(recording_id).await?; + let path = stream_state + .borrow() + .clips + .last() + .context("recording has no clips")? + .path + .clone(); + let streaming_type = validate_streaming_file(&path).await?; let upgrade_result = match streaming_type { StreamingType::Terminal => { - let shutdown_notify = Arc::clone(&shutdown_notify); + let shutdown_notify = recordings.subscribe_to_recording_finish(recording_id).await?; + let when_new_chunk_appended = move || { + let (tx, rx) = tokio::sync::oneshot::channel(); + recordings.add_new_chunk_listener(recording_id, tx); + rx + }; + let path = Arc::new(path); ws.on_upgrade(move |socket| async move { if let Err(e) = setup_terminal_streaming(&path, socket, shutdown_notify, when_new_chunk_appended).await { @@ -41,14 +49,11 @@ pub(crate) async fn stream_file( } }) } - StreamingType::WebM => { - let shutdown_notify = Arc::clone(&shutdown_notify); - ws.on_upgrade(move |socket| async move { - if let Err(e) = setup_webm_streaming(&path, socket, shutdown_notify, when_new_chunk_appended).await { - error!(error = ?e, "WebM streaming failed"); - } - }) - } + StreamingType::WebM => ws.on_upgrade(move |socket| async move { + if let Err(e) = setup_webm_streaming(stream_state, socket, shutdown_signal).await { + error!(error = ?e, "WebM streaming failed"); + } + }), }; Ok(upgrade_result) @@ -150,45 +155,168 @@ async fn setup_terminal_streaming( } async fn setup_webm_streaming( - path: &camino::Utf8Path, + stream_state: watch::Receiver, socket: WebSocket, - shutdown_notify: Arc, - when_new_chunk_appended: impl Fn() -> tokio::sync::oneshot::Receiver<()> + Send + 'static, + shutdown_signal: ShutdownSignal, ) -> anyhow::Result<()> { - let streaming_file = ReOpenableFile::open(path).with_context(|| format!("failed to open file: {path:?}"))?; - let streamer_config = video_streamer::StreamingConfig { - encoder_threads: CpuCount::default(), - adaptive_frame_skip: true, - }; - - let (websocket_stream, close_handle) = - crate::ws::handle(socket, Arc::clone(&shutdown_notify), Duration::from_secs(45)); - let streaming_result = tokio::task::spawn_blocking(move || { - webm_stream( - websocket_stream, - streaming_file, - shutdown_notify, - streamer_config, - when_new_chunk_appended, - ) - .context("webm_stream failed")?; - Ok::<_, anyhow::Error>(()) - }) - .await; + let source = recording_event_stream(stream_state)?; + let (websocket_stream, close_handle) = crate::ws::handle_messages( + socket, + crate::ws::KeepAliveShutdownSignal(shutdown_signal), + Duration::from_secs(45), + ); + let streaming_result = stream_session(source, websocket_stream, SessionConfig::default()).await; match streaming_result { - Err(e) => { - error!(error=?e, "Streaming file task join failed"); - Err(anyhow::anyhow!("Streaming task failed")) - } - Ok(Err(e)) => { + Err(error) => { close_handle.server_error("webm streaming failure".to_owned()).await; - error!(error = format!("{e:#}"), "Streaming file failed"); - Err(e) + error!(error = format!("{error:#}"), "WebM streaming failed"); + Err(error) } - Ok(Ok(())) => { + Ok(()) => { close_handle.normal_close().await; Ok(()) } } } + +struct CurrentRecordingClip { + sequence: u64, + file: File, + caught_up: bool, +} + +struct RecordingEventSource { + stream_state: watch::Receiver, + next_clip: usize, + current_clip: Option, + next_start_at: StartAt, + ended: bool, +} + +impl RecordingEventSource { + fn new(mut stream_state: watch::Receiver) -> anyhow::Result { + let state = stream_state.borrow_and_update().clone(); + let (next_clip, next_start_at) = match state.active { + Some(active) => ( + usize::try_from(active.sequence).context("recording sequence does not fit in usize")?, + if active.ready { + StartAt::LiveEdge + } else { + StartAt::Beginning + }, + ), + None => (state.clips.len(), StartAt::Beginning), + }; + + Ok(Self { + stream_state, + next_clip, + current_clip: None, + next_start_at, + ended: false, + }) + } + + async fn next_event(&mut self) -> anyhow::Result> { + const READ_BUFFER_SIZE: usize = 64 * 1024; + + if self.ended { + return Ok(None); + } + + loop { + let state = self.stream_state.borrow_and_update().clone(); + + if let Some(current_clip) = self.current_clip.as_mut() { + let mut bytes = vec![0; READ_BUFFER_SIZE]; + let read = current_clip.file.read(&mut bytes).await?; + if read > 0 { + bytes.truncate(read); + return Ok(Some(RecordingEvent::Bytes(Bytes::from(bytes)))); + } + + if !current_clip.caught_up { + current_clip.caught_up = true; + return Ok(Some(RecordingEvent::CaughtUp)); + } + + if state + .active + .is_some_and(|active| active.sequence == current_clip.sequence) + { + self.stream_state + .changed() + .await + .context("recording stream state closed")?; + continue; + } + + self.current_clip = None; + self.next_clip = self.next_clip.checked_add(1).context("recording clip index overflow")?; + return Ok(Some(RecordingEvent::ClipEnded)); + } + + if let Some(clip) = state.clips.get(self.next_clip) { + let expected_sequence = + u64::try_from(self.next_clip).context("recording clip index does not fit in u64")?; + if clip.sequence != expected_sequence { + anyhow::bail!("recording clip sequence is not contiguous"); + } + + if state + .active + .is_some_and(|active| active.sequence == clip.sequence && !active.ready) + { + self.stream_state + .changed() + .await + .context("recording stream state closed")?; + continue; + } + + if clip.path.extension() != Some(RecordingFileType::WebM.extension()) { + anyhow::bail!("recording clip is not WebM"); + } + + let file = File::open(&clip.path) + .await + .with_context(|| format!("failed to open recording clip: {}", clip.path))?; + let start_at = std::mem::replace(&mut self.next_start_at, StartAt::Beginning); + self.current_clip = Some(CurrentRecordingClip { + sequence: clip.sequence, + file, + caught_up: false, + }); + return Ok(Some(RecordingEvent::ClipStarted { + sequence: clip.sequence, + start_at, + })); + } + + if state.ended { + self.ended = true; + return Ok(Some(RecordingEvent::SessionEnded)); + } + + self.stream_state + .changed() + .await + .context("recording stream state closed")?; + } + } +} + +fn recording_event_stream( + stream_state: watch::Receiver, +) -> anyhow::Result> + Send + 'static> { + let source = RecordingEventSource::new(stream_state)?; + Ok(stream::unfold(Some(source), |source| async move { + let mut source = source?; + match source.next_event().await { + Ok(Some(event)) => Some((Ok(event), Some(source))), + Ok(None) => None, + Err(error) => Some((Err(error), None)), + } + })) +} diff --git a/devolutions-gateway/src/ws.rs b/devolutions-gateway/src/ws.rs index 59b667df6..a0ab14c6b 100644 --- a/devolutions-gateway/src/ws.rs +++ b/devolutions-gateway/src/ws.rs @@ -43,6 +43,51 @@ pub fn handle( (websocket_compat(ws), close_handle) } +pub fn handle_messages( + ws: WebSocket, + shutdown_signal: impl transport::KeepAliveShutdown, + keep_alive_interval: time::Duration, +) -> ( + impl futures::Stream> + + futures::Sink + + Unpin + + Send + + 'static, + transport::CloseWebSocketHandle, +) { + let ws = transport::Shared::new(ws); + + let close_handle = transport::spawn_websocket_sentinel_task( + ws.shared().with(|message: transport::WsWriteMsg| { + future::ready(Result::<_, axum::Error>::Ok(match message { + transport::WsWriteMsg::Ping => ws::Message::Ping(Bytes::new()), + transport::WsWriteMsg::Close(frame) => ws::Message::Close(Some(CloseFrame { + code: frame.code, + reason: frame.message.into(), + })), + })) + }), + shutdown_signal, + keep_alive_interval, + ); + + let messages = ws + .take_while(|item| future::ready(!matches!(item, Ok(ws::Message::Close(_))))) + .filter_map(|item| { + item.map(|msg| match msg { + ws::Message::Text(s) => Some(Bytes::from(s)), + ws::Message::Binary(data) => Some(data), + ws::Message::Ping(_) | ws::Message::Pong(_) => None, + ws::Message::Close(_) => None, + }) + .transpose() + .pipe(future::ready) + }) + .with(|item: Bytes| futures::future::ready(Ok::<_, axum::Error>(ws::Message::Binary(item)))); + + (messages, close_handle) +} + fn websocket_compat(ws: transport::Shared) -> impl AsyncRead + AsyncWrite + Unpin + Send + 'static { let ws_compat = ws .filter_map(|item| { From 948bd4ab3302bbf9d99d048173fe363034833ff3 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Mon, 24 Aug 2026 20:58:43 -0400 Subject: [PATCH 5/9] fix(dgw): preserve reconnecting stream segments Keep one response per Pull while recordings roll over to new WebM clips. Recover truncated tails and distinguish reconnectable capture disconnects from terminal recording end. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/video-streamer/README.md | 25 +- crates/video-streamer/src/normalizer.rs | 180 ++++++++++++-- crates/video-streamer/src/protocol.rs | 312 ++++++++++++++++++++++-- crates/video-streamer/src/session.rs | 4 +- devolutions-gateway/src/recording.rs | 72 +++++- devolutions-gateway/src/streaming.rs | 114 +++++++++ 6 files changed, 653 insertions(+), 54 deletions(-) diff --git a/crates/video-streamer/README.md b/crates/video-streamer/README.md index 31826d8d4..5112daf9a 100644 --- a/crates/video-streamer/README.md +++ b/crates/video-streamer/README.md @@ -1,6 +1,6 @@ # video-streamer -This crate takes an unseekable WebM recording and rewrites it into a stream that can start playing immediately. +This crate rewrites an unseekable WebM recording into a stream that can start playing immediately. `webm_stream` still serves one growing file over the original Start/Pull protocol. `stream_session` accepts a multi-clip recording event stream, reconnects across clips, and emits independent VP8 WebM segments over the same Start/Pull codes. @@ -11,8 +11,27 @@ The input event grammar is: (ClipStarted Bytes* CaughtUp Bytes* ClipEnded)* SessionEnded ``` -Pulls that arrive while a response is pending are queued. -`Stream ended` (type code 3) ends the session. +`ClipEnded` closes one input clip but does not end the recording session. +After `ClipEnded`, an existing viewer waits for a reconnecting `ClipStarted` until `SessionEnded` confirms the final end. + +## Session protocol + +The client sends `Start` (`00`) once. +After fully handling `Segment started` or `Chunk`, the client sends one `Pull` (`01`). +The client does not send `Pull` after `Error` or `Stream ended`. +The server sends exactly one response for each accepted request and buffers at most one early `Pull` while a response is pending. +An accepted queued `Pull` receives its own `Stream ended` response if the session ends before more segment data arrives. +If another overlapping request exceeds that limit, the current and queued requests receive `Error`, the excess request is rejected, and the stream fails. + +`Segment started` (`01` + JSON) carries `{codec,sequence,width,height}` and begins an independent WebM segment. +The output `sequence` starts at zero, is independent of the input `ClipStarted.sequence`, and increments for each output segment. +A reconnecting clip or resolution change starts the next output segment. +Another `Segment started` message implicitly closes the previous segment. +Legacy `{codec}` metadata remains valid for one segment with sequence zero. + +`Chunk` (`00` + bytes) belongs to the current segment. +`Stream ended` (`03`) cleanly closes the final segment and confirms that the recording session ended. +`Error` (`02` + JSON), an abrupt transport close, or a transport error does not confirm a clean session end. ## Prerequisites diff --git a/crates/video-streamer/src/normalizer.rs b/crates/video-streamer/src/normalizer.rs index 1630bde5c..96bd04d87 100644 --- a/crates/video-streamer/src/normalizer.rs +++ b/crates/video-streamer/src/normalizer.rs @@ -6,6 +6,7 @@ use anyhow::Context; use bytes::{Bytes, BytesMut}; use cadeau::xmf::vpx::{VpxCodec, VpxEncoder, VpxEncoderPreset, VpxImage}; use ebml_iterable::TagDecoder; +use ebml_iterable::error::TagIteratorError; use futures_util::{Stream, StreamExt}; use tokio::sync::mpsc; use webm_iterable::matroska_spec::{Master, MatroskaSpec, SimpleBlock}; @@ -311,10 +312,20 @@ impl ClipNormalizer { self.clip_sequence ); loop { - match self.decoder.decode_eof(&mut self.input)? { - Some(positioned) => self.handle_tag(positioned.tag)?, - None if self.decoder.is_finished() => break, - None => continue, + match self.decoder.decode_eof(&mut self.input) { + Ok(Some(positioned)) => self.handle_tag(positioned.tag)?, + Ok(None) if self.decoder.is_finished() => break, + Ok(None) => continue, + Err(TagIteratorError::UnexpectedEOF { .. }) => { + debug!( + clip_sequence = self.clip_sequence, + bytes = self.input.len(), + "Discard incomplete trailing EBML element" + ); + self.input.clear(); + break; + } + Err(error) => return Err(error.into()), } } @@ -374,27 +385,18 @@ impl ClipNormalizer { .get_or_insert_with(|| InputDecoder::new(frame.codec, self.config.encoder_threads)); let decoded = input_decoder.decode(&frame.data)?; let dimensions = decoded.dimensions; - let size_changed = self - .output_segment - .as_ref() - .is_some_and(|current| current.dimensions != dimensions); - if size_changed { + let new_segment = next_segment_info( + self.output_segment.as_ref().map(|segment| segment.dimensions), + dimensions, + self.next_segment_sequence, + ); + if self.output_segment.is_some() && new_segment.is_some() { self.output_segment .take() .context("missing active output segment")? .finish()?; } - let new_segment = if self.output_segment.is_none() { - Some(SegmentInfo { - sequence: self.next_segment_sequence, - width: dimensions.width, - height: dimensions.height, - }) - } else { - None - }; - if let Some(info) = new_segment { self.output_segment = Some(OutputSegment::new(self.sender.clone(), info, self.config)?); self.next_segment_sequence = self @@ -410,6 +412,18 @@ impl ClipNormalizer { } } +fn next_segment_info( + current_dimensions: Option, + frame_dimensions: Dimensions, + next_sequence: u64, +) -> Option { + (current_dimensions != Some(frame_dimensions)).then_some(SegmentInfo { + sequence: next_sequence, + width: frame_dimensions.width, + height: frame_dimensions.height, + }) +} + fn parse_video_track(children: &[MatroskaSpec]) -> anyhow::Result> { let is_video = children .iter() @@ -637,3 +651,131 @@ impl Write for EventWriter { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_clip_bytes() -> Vec { + let mut writer = WebmWriter::new(Vec::new()); + writer + .write(&MatroskaSpec::Ebml(Master::Full(vec![ + MatroskaSpec::EbmlVersion(1), + MatroskaSpec::EbmlReadVersion(1), + MatroskaSpec::EbmlMaxIdLength(4), + MatroskaSpec::EbmlMaxSizeLength(8), + MatroskaSpec::DocType("webm".to_owned()), + MatroskaSpec::DocTypeVersion(4), + MatroskaSpec::DocTypeReadVersion(2), + ]))) + .expect("write EBML header"); + writer + .write_advanced( + &MatroskaSpec::Segment(Master::Start), + WriteOptions::is_unknown_sized_element(), + ) + .expect("write segment start"); + writer + .write_advanced( + &MatroskaSpec::Cluster(Master::Start), + WriteOptions::is_unknown_sized_element(), + ) + .expect("write cluster start"); + writer + .write(&MatroskaSpec::Timestamp(0)) + .expect("write cluster timestamp"); + writer.into_inner().expect("finish clip bytes") + } + + #[test] + fn resolution_change_starts_the_next_output_segment() { + let first_dimensions = Dimensions { + width: 640, + height: 480, + }; + let second_dimensions = Dimensions { + width: 1280, + height: 720, + }; + + assert_eq!( + next_segment_info(None, first_dimensions, 0), + Some(SegmentInfo { + sequence: 0, + width: 640, + height: 480, + }) + ); + assert_eq!(next_segment_info(Some(first_dimensions), first_dimensions, 1), None); + assert_eq!( + next_segment_info(Some(first_dimensions), second_dimensions, 1), + Some(SegmentInfo { + sequence: 1, + width: 1280, + height: 720, + }) + ); + } + + #[test] + fn truncated_clip_tail_does_not_abort_the_following_clip() { + let mut truncated = empty_clip_bytes(); + truncated.extend_from_slice(&[0xa3, 0x84, 0x81, 0x00]); + let complete = empty_clip_bytes(); + let events = [ + RecordingEvent::ClipStarted { + sequence: 0, + start_at: StartAt::Beginning, + }, + RecordingEvent::Bytes(Bytes::from(truncated)), + RecordingEvent::CaughtUp, + RecordingEvent::ClipEnded, + RecordingEvent::ClipStarted { + sequence: 1, + start_at: StartAt::Beginning, + }, + RecordingEvent::Bytes(Bytes::from(complete)), + RecordingEvent::CaughtUp, + RecordingEvent::ClipEnded, + RecordingEvent::SessionEnded, + ]; + let (input_sender, input_receiver) = mpsc::channel(events.len()); + for event in events { + input_sender.blocking_send(Ok(event)).expect("queue recording event"); + } + drop(input_sender); + let (output_sender, mut output_receiver) = mpsc::channel(1); + + normalize_events(input_receiver, output_sender, SessionConfig { encoder_threads: 1 }) + .expect("normalize reconnecting clips"); + assert!(output_receiver.blocking_recv().is_none()); + } + + #[test] + fn corruption_before_an_incomplete_tail_still_fails() { + let mut corrupted = empty_clip_bytes(); + corrupted.extend_from_slice(&[0xff, 0x80]); + corrupted.extend_from_slice(&[0xa3, 0x84, 0x81, 0x00]); + let events = [ + RecordingEvent::ClipStarted { + sequence: 0, + start_at: StartAt::Beginning, + }, + RecordingEvent::Bytes(Bytes::from(corrupted)), + RecordingEvent::CaughtUp, + RecordingEvent::ClipEnded, + RecordingEvent::SessionEnded, + ]; + let (input_sender, input_receiver) = mpsc::channel(events.len()); + for event in events { + input_sender.blocking_send(Ok(event)).expect("queue recording event"); + } + drop(input_sender); + let (output_sender, _output_receiver) = mpsc::channel(1); + + let error = normalize_events(input_receiver, output_sender, SessionConfig { encoder_threads: 1 }) + .expect_err("corruption before the incomplete tail must fail"); + + assert!(format!("{error:#}").contains("corrupted"), "{error:#}"); + } +} diff --git a/crates/video-streamer/src/protocol.rs b/crates/video-streamer/src/protocol.rs index ea375a241..de6f2cea2 100644 --- a/crates/video-streamer/src/protocol.rs +++ b/crates/video-streamer/src/protocol.rs @@ -8,6 +8,8 @@ use futures_util::{Sink, SinkExt as _, Stream, StreamExt as _}; use crate::normalizer::{SegmentEvent, SegmentInfo}; +const MAX_QUEUED_PULLS: usize = 1; + #[derive(Debug, Eq, PartialEq)] pub(crate) enum ServerMessage { Chunk(Bytes), @@ -43,7 +45,7 @@ where { tokio::pin!(segments); let mut expected = ClientMessage::Start; - let mut segment_state = SegmentState::AwaitingBegin; + let mut segment_state = SegmentState::AwaitingBegin { next_sequence: 0 }; let mut queued_pulls = VecDeque::new(); loop { @@ -98,8 +100,12 @@ where Ok(None) => return Ok(()), Err(error) => { debug!(error = format!("{error:#}"), "Request failed while waiting"); - let _ = - send_server_message(&mut transport, ServerMessage::Error(UserFriendlyError::UnexpectedError)).await; + let pending_requests = 1 + queued_pulls.len(); + for _ in 0..pending_requests { + let _ = + send_server_message(&mut transport, ServerMessage::Error(UserFriendlyError::UnexpectedError)) + .await; + } return Err(error); } }; @@ -107,6 +113,9 @@ where let ended = response == ServerMessage::StreamEnded; send_server_message(&mut transport, response).await?; if ended { + while queued_pulls.pop_front().is_some() { + send_server_message(&mut transport, ServerMessage::StreamEnded).await?; + } return Ok(()); } @@ -173,8 +182,8 @@ fn encode_server_message(message: ServerMessage) -> Bytes { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum SegmentState { - AwaitingBegin, - Streaming, + AwaitingBegin { next_sequence: u64 }, + Streaming { next_sequence: u64 }, } async fn wait_for_response( @@ -191,14 +200,15 @@ where loop { tokio::select! { biased; - response = next_segment_message(segments.as_mut(), state) => { - return response; - } message = transport.next() => match message { None => return Ok(None), Some(Ok(message)) => { let message = decode_client_message(&message).context("decode pipelined client request")?; if message == ClientMessage::Pull { + anyhow::ensure!( + queued_pulls.len() < MAX_QUEUED_PULLS, + "too many pipelined Pull requests" + ); debug!( queued_pulls = queued_pulls.len() + 1, "Queued pipelined Pull while waiting for response" @@ -216,6 +226,9 @@ where return Err(anyhow::Error::new(error).context("read client stream message")); } }, + response = next_segment_message(segments.as_mut(), state) => { + return response; + } } } } @@ -230,7 +243,7 @@ where loop { let Some(event) = segments.as_mut().next().await else { anyhow::ensure!( - *state == SegmentState::AwaitingBegin, + matches!(*state, SegmentState::AwaitingBegin { .. }), "segment stream ended inside a segment" ); return Ok(Some(ServerMessage::StreamEnded)); @@ -238,11 +251,17 @@ where match event? { SegmentEvent::Begin(info) => { + let SegmentState::AwaitingBegin { next_sequence } = *state else { + anyhow::bail!("segment began before the previous segment ended"); + }; anyhow::ensure!( - *state == SegmentState::AwaitingBegin, - "segment began before the previous segment ended" + info.sequence == next_sequence, + "segment sequence is not contiguous: expected {next_sequence}, got {}", + info.sequence ); - *state = SegmentState::Streaming; + *state = SegmentState::Streaming { + next_sequence: next_sequence.checked_add(1).context("segment sequence overflow")?, + }; debug!( sequence = info.sequence, width = info.width, @@ -253,15 +272,17 @@ where } SegmentEvent::Data(data) => { anyhow::ensure!( - *state == SegmentState::Streaming, + matches!(*state, SegmentState::Streaming { .. }), "segment data arrived outside a segment" ); debug!(bytes = data.len(), "Segment data"); return Ok(Some(ServerMessage::Chunk(data))); } SegmentEvent::End => { - anyhow::ensure!(*state == SegmentState::Streaming, "segment ended outside a segment"); - *state = SegmentState::AwaitingBegin; + let SegmentState::Streaming { next_sequence } = *state else { + anyhow::bail!("segment ended outside a segment"); + }; + *state = SegmentState::AwaitingBegin { next_sequence }; debug!("Segment end"); } } @@ -271,8 +292,12 @@ where #[cfg(test)] mod tests { use std::collections::VecDeque; + use std::pin::Pin; + use std::task::{Context, Poll}; + use std::time::Duration; - use futures_util::{StreamExt as _, stream}; + use futures_util::{Sink, StreamExt as _, stream}; + use tokio::sync::mpsc; use super::*; @@ -282,6 +307,58 @@ mod tests { stream::iter(messages).chain(stream::pending()) } + struct ChannelTransport { + incoming: mpsc::UnboundedReceiver, + outgoing: mpsc::UnboundedSender, + } + + impl Stream for ChannelTransport { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.incoming.poll_recv(cx).map(|message| message.map(Ok)) + } + } + + impl Sink for ChannelTransport { + type Error = std::io::Error; + + fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn start_send(self: Pin<&mut Self>, message: Bytes) -> Result<(), Self::Error> { + self.outgoing + .send(message) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::BrokenPipe, "test receiver closed")) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + fn channel_transport() -> ( + ChannelTransport, + mpsc::UnboundedSender, + mpsc::UnboundedReceiver, + ) { + let (client_sender, incoming) = mpsc::unbounded_channel(); + let (outgoing, client_receiver) = mpsc::unbounded_channel(); + (ChannelTransport { incoming, outgoing }, client_sender, client_receiver) + } + + async fn receive_response(receiver: &mut mpsc::UnboundedReceiver) -> Bytes { + tokio::time::timeout(Duration::from_secs(1), receiver.recv()) + .await + .expect("timed out waiting for server response") + .expect("server response channel closed") + } + #[test] fn protocol_codes_are_stable() { assert_eq!( @@ -336,7 +413,7 @@ mod tests { ]; let segments = stream::iter(events); tokio::pin!(segments); - let mut state = SegmentState::AwaitingBegin; + let mut state = SegmentState::AwaitingBegin { next_sequence: 0 }; assert!(matches!( next_segment_message(segments.as_mut(), &mut state) @@ -382,7 +459,7 @@ mod tests { })) }); tokio::pin!(segments); - let mut state = SegmentState::AwaitingBegin; + let mut state = SegmentState::AwaitingBegin { next_sequence: 0 }; let mut queued_pulls = VecDeque::new(); let response = wait_for_response(&mut transport, segments.as_mut(), &mut state, &mut queued_pulls) @@ -405,7 +482,7 @@ mod tests { Ok(SegmentEvent::Data(Bytes::from_static(b"chunk"))) }); tokio::pin!(segments); - let mut state = SegmentState::Streaming; + let mut state = SegmentState::Streaming { next_sequence: 1 }; let mut queued_pulls = VecDeque::new(); let response = wait_for_response(&mut transport, segments.as_mut(), &mut state, &mut queued_pulls) @@ -425,7 +502,7 @@ mod tests { Ok(SegmentEvent::Data(Bytes::from_static(b"chunk"))) }); tokio::pin!(segments); - let mut state = SegmentState::Streaming; + let mut state = SegmentState::Streaming { next_sequence: 1 }; let mut queued_pulls = VecDeque::new(); let error = wait_for_response(&mut transport, segments.as_mut(), &mut state, &mut queued_pulls) @@ -437,4 +514,199 @@ mod tests { "{error:#}" ); } + + #[tokio::test] + async fn first_segment_sequence_must_be_zero() { + let segments = stream::iter([Ok(SegmentEvent::Begin(SegmentInfo { + sequence: 1, + width: 640, + height: 480, + }))]); + tokio::pin!(segments); + let mut state = SegmentState::AwaitingBegin { next_sequence: 0 }; + + let error = next_segment_message(segments.as_mut(), &mut state) + .await + .expect_err("nonzero first sequence must fail"); + + assert!( + format!("{error:#}").contains("segment sequence is not contiguous"), + "{error:#}" + ); + } + + #[tokio::test] + async fn segment_sequence_gap_is_rejected() { + let segments = stream::iter([ + Ok(SegmentEvent::Begin(SegmentInfo { + sequence: 0, + width: 640, + height: 480, + })), + Ok(SegmentEvent::End), + Ok(SegmentEvent::Begin(SegmentInfo { + sequence: 2, + width: 800, + height: 600, + })), + ]); + tokio::pin!(segments); + let mut state = SegmentState::AwaitingBegin { next_sequence: 0 }; + + assert!(matches!( + next_segment_message(segments.as_mut(), &mut state) + .await + .expect("first segment"), + Some(ServerMessage::SegmentStarted(SegmentInfo { sequence: 0, .. })) + )); + let error = next_segment_message(segments.as_mut(), &mut state) + .await + .expect_err("segment sequence gap must fail"); + + assert!( + format!("{error:#}").contains("segment sequence is not contiguous"), + "{error:#}" + ); + } + + #[tokio::test] + async fn pipelined_pull_queue_is_bounded() { + let mut transport = pending_after([ + Ok::<_, std::io::Error>(Bytes::from_static(b"\x01")), + Ok::<_, std::io::Error>(Bytes::from_static(b"\x01")), + ]); + let segments = stream::pending::>(); + tokio::pin!(segments); + let mut state = SegmentState::AwaitingBegin { next_sequence: 0 }; + let mut queued_pulls = VecDeque::new(); + + let error = tokio::time::timeout( + Duration::from_millis(100), + wait_for_response(&mut transport, segments.as_mut(), &mut state, &mut queued_pulls), + ) + .await + .expect("a second queued Pull must be rejected") + .expect_err("a second queued Pull must fail"); + + assert!( + format!("{error:#}").contains("too many pipelined Pull requests"), + "{error:#}" + ); + assert_eq!(queued_pulls, VecDeque::from([ClientMessage::Pull])); + } + + #[tokio::test] + async fn ready_output_does_not_bypass_the_pipelined_pull_limit() { + let (transport, client_sender, mut client_receiver) = channel_transport(); + client_sender.send(Bytes::from_static(b"\x00")).expect("send Start"); + client_sender + .send(Bytes::from_static(b"\x01")) + .expect("send first pipelined Pull"); + client_sender + .send(Bytes::from_static(b"\x01")) + .expect("send second pipelined Pull"); + let segments = stream::iter([ + Ok(SegmentEvent::Begin(SegmentInfo { + sequence: 0, + width: 640, + height: 480, + })), + Ok(SegmentEvent::End), + ]); + let task = tokio::spawn(stream_segments(transport, segments)); + + assert_eq!(receive_response(&mut client_receiver).await[0], 2); + assert_eq!(receive_response(&mut client_receiver).await[0], 2); + assert!(task.await.expect("stream task panicked").is_err()); + assert_eq!(client_receiver.recv().await, None); + } + + #[tokio::test] + async fn queued_pull_receives_stream_end() { + let (transport, client_sender, mut client_receiver) = channel_transport(); + client_sender.send(Bytes::from_static(b"\x00")).expect("send Start"); + client_sender + .send(Bytes::from_static(b"\x01")) + .expect("send queued Pull"); + let task = tokio::spawn(stream_segments(transport, stream::empty())); + + assert_eq!( + receive_response(&mut client_receiver).await, + Bytes::from_static(b"\x03") + ); + assert_eq!( + receive_response(&mut client_receiver).await, + Bytes::from_static(b"\x03") + ); + task.await + .expect("stream task panicked") + .expect("stream session failed"); + assert_eq!(client_receiver.recv().await, None); + } + + #[tokio::test] + async fn each_request_receives_exactly_one_response() { + let (transport, client_sender, mut client_receiver) = channel_transport(); + let segments = stream::iter([ + Ok(SegmentEvent::Begin(SegmentInfo { + sequence: 0, + width: 640, + height: 480, + })), + Ok(SegmentEvent::Data(Bytes::from_static(b"chunk"))), + Ok(SegmentEvent::End), + ]); + let task = tokio::spawn(stream_segments(transport, segments)); + + client_sender.send(Bytes::from_static(b"\x00")).expect("send Start"); + assert_eq!(receive_response(&mut client_receiver).await[0], 1); + assert!( + tokio::time::timeout(Duration::from_millis(10), client_receiver.recv()) + .await + .is_err(), + "server sent a response without another request" + ); + + client_sender + .send(Bytes::from_static(b"\x01")) + .expect("send first Pull"); + assert_eq!( + receive_response(&mut client_receiver).await, + Bytes::from_static(b"\x00chunk") + ); + assert!( + tokio::time::timeout(Duration::from_millis(10), client_receiver.recv()) + .await + .is_err(), + "server sent a response without another request" + ); + + client_sender + .send(Bytes::from_static(b"\x01")) + .expect("send final Pull"); + assert_eq!( + receive_response(&mut client_receiver).await, + Bytes::from_static(b"\x03") + ); + task.await + .expect("stream task panicked") + .expect("stream session failed"); + } + + #[tokio::test] + async fn segment_failure_sends_error_without_stream_end() { + let (transport, client_sender, mut client_receiver) = channel_transport(); + let segments = stream::iter([Err(anyhow::anyhow!("test segment failure"))]); + let task = tokio::spawn(stream_segments(transport, segments)); + + client_sender.send(Bytes::from_static(b"\x00")).expect("send Start"); + let response = receive_response(&mut client_receiver).await; + assert_eq!(response[0], 2); + assert!(task.await.expect("stream task panicked").is_err()); + assert_eq!( + client_receiver.recv().await, + None, + "error must not be followed by StreamEnded" + ); + } } diff --git a/crates/video-streamer/src/session.rs b/crates/video-streamer/src/session.rs index a8711b36f..cfaef3f73 100644 --- a/crates/video-streamer/src/session.rs +++ b/crates/video-streamer/src/session.rs @@ -36,7 +36,9 @@ impl Default for SessionConfig { } } -/// Converts a recording session into fixed-size VP8 WebM segments over one pull-driven stream. +/// Converts a recording session into independent VP8 WebM segments over one pull-driven stream. +/// +/// Each segment has one resolution, and output sequence numbers remain contiguous across input clips. pub async fn stream_session(source: S, transport: T, config: SessionConfig) -> anyhow::Result<()> where S: Stream> + Send + 'static, diff --git a/devolutions-gateway/src/recording.rs b/devolutions-gateway/src/recording.rs index b68bc338c..ea5c58da3 100644 --- a/devolutions-gateway/src/recording.rs +++ b/devolutions-gateway/src/recording.rs @@ -279,6 +279,62 @@ pub(crate) struct RecordingStreamState { revision: u64, } +impl RecordingStreamState { + pub(crate) fn mark_disconnected(&mut self) { + self.active = None; + self.ended = false; + self.revision = self.revision.saturating_add(1); + } + + pub(crate) fn mark_ended(&mut self) { + self.active = None; + self.ended = true; + self.revision = self.revision.saturating_add(1); + } + + #[cfg(test)] + pub(crate) fn for_test( + clips: Vec, + active: Option, + ended: bool, + ) -> Self { + Self { + clips: Arc::new(clips), + active, + ended, + revision: 0, + } + } +} + +#[cfg(test)] +mod stream_state_tests { + use super::*; + + #[test] + fn disconnect_is_not_a_confirmed_session_end() { + let mut state = RecordingStreamState { + clips: Arc::new(Vec::new()), + active: Some(ActiveRecordingStreamClip { + sequence: 0, + ready: true, + }), + ended: false, + revision: 0, + }; + + state.mark_disconnected(); + assert!(state.active.is_none()); + assert!(!state.ended); + assert_eq!(state.revision, 1); + + state.mark_ended(); + assert!(state.active.is_none()); + assert!(state.ended); + assert_eq!(state.revision, 2); + } +} + enum RecordingManagerMessage { Connect { id: Uuid, @@ -815,13 +871,11 @@ impl RecordingManagerTask { .save_to_file(&ongoing.manifest_path) .with_context(|| format!("write manifest at {}", ongoing.manifest_path))?; - ongoing.stream_state.send_modify(|state| { - state.active = None; - state.ended = true; - state.revision = state.revision.saturating_add(1); - }); + ongoing + .stream_state + .send_modify(RecordingStreamState::mark_disconnected); - // Notify all the streamers that recording has ended. + // Wake terminal-recording streamers waiting for this clip to stop. if let Some(notify) = self.recording_end_notifier.get(&id) { notify.notify_waiters(); } @@ -860,11 +914,7 @@ impl RecordingManagerTask { OnGoingRecordingState::LastSeen { timestamp } if now >= timestamp + disconnected_ttl_secs - 1 => { debug!(%id, "Mark recording as terminated"); self.rx.active_recordings.remove(id); - ongoing.stream_state.send_modify(|state| { - state.active = None; - state.ended = true; - state.revision = state.revision.saturating_add(1); - }); + ongoing.stream_state.send_modify(RecordingStreamState::mark_ended); // Check the recording policy of the associated session and kill it if necessary. if ongoing.session_must_be_recorded { diff --git a/devolutions-gateway/src/streaming.rs b/devolutions-gateway/src/streaming.rs index 4bc280cb4..d9bd2777e 100644 --- a/devolutions-gateway/src/streaming.rs +++ b/devolutions-gateway/src/streaming.rs @@ -320,3 +320,117 @@ fn recording_event_stream( } })) } + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + use crate::recording::{ActiveRecordingStreamClip, RecordingStreamClip}; + + struct ScratchDirectory(camino::Utf8PathBuf); + + impl Drop for ScratchDirectory { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + #[tokio::test] + async fn reconnect_waits_for_the_next_clip_before_ending_the_session() { + let scratch = camino::Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("target") + .join("streaming-tests") + .join(Uuid::new_v4().to_string()); + fs::create_dir_all(&scratch).expect("create test directory"); + let _cleanup = ScratchDirectory(scratch.clone()); + + let first_path = scratch.join("recording-0.webm"); + let second_path = scratch.join("recording-1.webm"); + fs::write(&first_path, b"first").expect("write first clip"); + fs::write(&second_path, b"second").expect("write second clip"); + + let first_clip = RecordingStreamClip { + sequence: 0, + path: first_path, + }; + let state = RecordingStreamState::for_test( + vec![first_clip], + Some(ActiveRecordingStreamClip { + sequence: 0, + ready: true, + }), + false, + ); + let (sender, receiver) = watch::channel(state); + let mut source = RecordingEventSource::new(receiver).expect("create recording event source"); + + assert_eq!( + source.next_event().await.expect("read first start"), + Some(RecordingEvent::ClipStarted { + sequence: 0, + start_at: StartAt::LiveEdge, + }) + ); + assert_eq!( + source.next_event().await.expect("read first bytes"), + Some(RecordingEvent::Bytes(Bytes::from_static(b"first"))) + ); + assert_eq!( + source.next_event().await.expect("catch up first clip"), + Some(RecordingEvent::CaughtUp) + ); + + sender.send_modify(RecordingStreamState::mark_disconnected); + assert_eq!( + source.next_event().await.expect("end first clip"), + Some(RecordingEvent::ClipEnded) + ); + assert!( + tokio::time::timeout(Duration::from_millis(25), source.next_event()) + .await + .is_err(), + "a reconnectable disconnect must not emit SessionEnded" + ); + + sender.send_modify(|state| { + Arc::make_mut(&mut state.clips).push(RecordingStreamClip { + sequence: 1, + path: second_path, + }); + state.active = Some(ActiveRecordingStreamClip { + sequence: 1, + ready: true, + }); + state.ended = false; + }); + assert_eq!( + source.next_event().await.expect("read second start"), + Some(RecordingEvent::ClipStarted { + sequence: 1, + start_at: StartAt::Beginning, + }) + ); + assert_eq!( + source.next_event().await.expect("read second bytes"), + Some(RecordingEvent::Bytes(Bytes::from_static(b"second"))) + ); + assert_eq!( + source.next_event().await.expect("catch up second clip"), + Some(RecordingEvent::CaughtUp) + ); + + sender.send_modify(RecordingStreamState::mark_disconnected); + assert_eq!( + source.next_event().await.expect("end second clip"), + Some(RecordingEvent::ClipEnded) + ); + sender.send_modify(RecordingStreamState::mark_ended); + assert_eq!( + source.next_event().await.expect("end session"), + Some(RecordingEvent::SessionEnded) + ); + assert_eq!(source.next_event().await.expect("finish source"), None); + } +} From eb201ab5fcbcdccff6f72a832d818b464e5c82a6 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Fri, 21 Aug 2026 16:23:07 -0400 Subject: [PATCH 6/9] feat(webapp): play session-protocol WebM streams The shadow player now understands Segment started, queued Pulls, and Stream ended so live clips can reconnect and change size on one websocket. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- webapp/apps/recording-player/index.css | 9 + .../public/locales/de/translation.json | 11 + .../public/locales/en/translation.json | 11 + .../public/locales/es/translation.json | 11 + .../public/locales/fr/translation.json | 11 + webapp/apps/recording-player/src/i18n.ts | 9 + .../recording-player/src/streamers/webm.ts | 18 +- .../shadow-player/demo-src/apiClient.ts | 4 +- webapp/packages/shadow-player/index.html | 4 +- .../shadow-player/src/playbackClip.ts | 80 ++ .../shadow-player/src/playbackControls.css | 149 ++++ .../shadow-player/src/playbackControls.ts | 306 ++++++++ webapp/packages/shadow-player/src/protocol.ts | 101 ++- .../shadow-player/src/sourceBuffer.ts | 113 +-- .../packages/shadow-player/src/streamer.css | 26 +- webapp/packages/shadow-player/src/streamer.ts | 690 ++++++++++++------ .../packages/shadow-player/src/websocket.ts | 51 +- 17 files changed, 1238 insertions(+), 366 deletions(-) create mode 100644 webapp/packages/shadow-player/src/playbackClip.ts create mode 100644 webapp/packages/shadow-player/src/playbackControls.css create mode 100644 webapp/packages/shadow-player/src/playbackControls.ts diff --git a/webapp/apps/recording-player/index.css b/webapp/apps/recording-player/index.css index b8fbcf612..d00a7055d 100644 --- a/webapp/apps/recording-player/index.css +++ b/webapp/apps/recording-player/index.css @@ -2,9 +2,18 @@ html, body { + width: 100%; + height: 100%; + margin: 0; background-color: black; } +shadow-player { + display: block; + width: 100%; + height: 100%; +} + #terminal { height: 100%; } diff --git a/webapp/apps/recording-player/public/locales/de/translation.json b/webapp/apps/recording-player/public/locales/de/translation.json index bd387d25a..2a6732f4c 100644 --- a/webapp/apps/recording-player/public/locales/de/translation.json +++ b/webapp/apps/recording-player/public/locales/de/translation.json @@ -6,6 +6,17 @@ "unknownError": "Unbekannter Fehler, bitte versuchen Sie es erneut", "protocolError": "Ein Fehler ist aufgetreten: {{error}}" }, + "controls": { + "play": "Wiedergabe", + "pause": "Pause", + "mute": "Stummschalten", + "unmute": "Stummschaltung aufheben", + "volume": "Lautstärke", + "timeline": "Aufzeichnungszeitachse", + "fullscreen": "Vollbild", + "exitFullscreen": "Vollbild beenden", + "clip": "Clip" + }, "ui": { "close": "Schließen" } diff --git a/webapp/apps/recording-player/public/locales/en/translation.json b/webapp/apps/recording-player/public/locales/en/translation.json index 9802ff8a1..33c645ece 100644 --- a/webapp/apps/recording-player/public/locales/en/translation.json +++ b/webapp/apps/recording-player/public/locales/en/translation.json @@ -6,6 +6,17 @@ "unknownError": "Unknown error, please try again", "protocolError": "An error occurred: {{error}}" }, + "controls": { + "play": "Play", + "pause": "Pause", + "mute": "Mute", + "unmute": "Unmute", + "volume": "Volume", + "timeline": "Recording timeline", + "fullscreen": "Fullscreen", + "exitFullscreen": "Exit fullscreen", + "clip": "Clip" + }, "ui": { "close": "Close" } diff --git a/webapp/apps/recording-player/public/locales/es/translation.json b/webapp/apps/recording-player/public/locales/es/translation.json index 9d12b8d06..811714f1e 100644 --- a/webapp/apps/recording-player/public/locales/es/translation.json +++ b/webapp/apps/recording-player/public/locales/es/translation.json @@ -6,6 +6,17 @@ "unknownError": "Error desconocido, por favor intente de nuevo", "protocolError": "Se produjo un error: {{error}}" }, + "controls": { + "play": "Reproducir", + "pause": "Pausar", + "mute": "Silenciar", + "unmute": "Activar sonido", + "volume": "Volumen", + "timeline": "Línea de tiempo de la grabación", + "fullscreen": "Pantalla completa", + "exitFullscreen": "Salir de pantalla completa", + "clip": "Clip" + }, "ui": { "close": "Cerrar" } diff --git a/webapp/apps/recording-player/public/locales/fr/translation.json b/webapp/apps/recording-player/public/locales/fr/translation.json index a6eb01a40..28cabf7d5 100644 --- a/webapp/apps/recording-player/public/locales/fr/translation.json +++ b/webapp/apps/recording-player/public/locales/fr/translation.json @@ -6,6 +6,17 @@ "unknownError": "Erreur inconnue, veuillez réessayer", "protocolError": "Une erreur s'est produite: {{error}}" }, + "controls": { + "play": "Lire", + "pause": "Pause", + "mute": "Couper le son", + "unmute": "Réactiver le son", + "volume": "Volume", + "timeline": "Chronologie de l'enregistrement", + "fullscreen": "Plein écran", + "exitFullscreen": "Quitter le plein écran", + "clip": "Séquence" + }, "ui": { "close": "Fermer" } diff --git a/webapp/apps/recording-player/src/i18n.ts b/webapp/apps/recording-player/src/i18n.ts index c2aac9b65..b1547be30 100644 --- a/webapp/apps/recording-player/src/i18n.ts +++ b/webapp/apps/recording-player/src/i18n.ts @@ -8,6 +8,15 @@ export type TranslationKeys = | 'notifications.unauthorized' | 'notifications.unknownError' | 'notifications.protocolError' + | 'controls.play' + | 'controls.pause' + | 'controls.mute' + | 'controls.unmute' + | 'controls.volume' + | 'controls.timeline' + | 'controls.fullscreen' + | 'controls.exitFullscreen' + | 'controls.clip' | 'ui.close'; /** diff --git a/webapp/apps/recording-player/src/streamers/webm.ts b/webapp/apps/recording-player/src/streamers/webm.ts index 9ddee0a4c..f889b522e 100644 --- a/webapp/apps/recording-player/src/streamers/webm.ts +++ b/webapp/apps/recording-player/src/streamers/webm.ts @@ -5,19 +5,25 @@ import { t } from '../i18n'; import { showNotification } from '../notification'; export async function handleWebm(gatewayAccessApi: GatewayAccessApi) { - // Create element with correct spelling const shadowPlayer = document.createElement('shadow-player') as ShadowPlayer; + shadowPlayer.setAttribute('controls', ''); + shadowPlayer.setControlLabels({ + play: t('controls.play'), + pause: t('controls.pause'), + mute: t('controls.mute'), + unmute: t('controls.unmute'), + volume: t('controls.volume'), + timeline: t('controls.timeline'), + fullscreen: t('controls.fullscreen'), + exitFullscreen: t('controls.exitFullscreen'), + clip: t('controls.clip'), + }); - // Append to DOM document.body.appendChild(shadowPlayer); - // Wait for element to be initialized await customElements.whenDefined('shadow-player'); - - // Wait for next microtask to ensure connectedCallback has run await new Promise((resolve) => setTimeout(resolve, 0)); - // Now safe to call methods shadowPlayer.srcChange(gatewayAccessApi.sessionShadowingUrl()); shadowPlayer.play(); diff --git a/webapp/packages/shadow-player/demo-src/apiClient.ts b/webapp/packages/shadow-player/demo-src/apiClient.ts index 9f3de2d1d..46950ad4f 100644 --- a/webapp/packages/shadow-player/demo-src/apiClient.ts +++ b/webapp/packages/shadow-player/demo-src/apiClient.ts @@ -1,6 +1,6 @@ // Base URL of the API -const TOKEN_SERVER_BASE_URL = 'http://localhost:8080'; -const GATEWAY_BASE_URL = 'http://localhost:7171'; +const TOKEN_SERVER_BASE_URL = import.meta.env.VITE_TOKEN_SERVER_BASE_URL ?? 'http://localhost:8080'; +const GATEWAY_BASE_URL = import.meta.env.VITE_GATEWAY_BASE_URL ?? 'http://localhost:7171'; // Common request fields interface CommonRequest { diff --git a/webapp/packages/shadow-player/index.html b/webapp/packages/shadow-player/index.html index 578f4abe2..541e75c8d 100644 --- a/webapp/packages/shadow-player/index.html +++ b/webapp/packages/shadow-player/index.html @@ -94,7 +94,7 @@ background-color: #e9ecef; } - webm-stream-player { + shadow-player { width: 80%; height: 80%; background-color: #000; @@ -120,7 +120,7 @@

Streaming Files

- +
diff --git a/webapp/packages/shadow-player/src/playbackClip.ts b/webapp/packages/shadow-player/src/playbackClip.ts new file mode 100644 index 000000000..3844bae72 --- /dev/null +++ b/webapp/packages/shadow-player/src/playbackClip.ts @@ -0,0 +1,80 @@ +import type { SegmentStartedMessage } from './protocol'; +import { ReactiveSourceBuffer } from './sourceBuffer'; + +export class PlaybackClip { + readonly video = document.createElement('video'); + + private readonly mediaSource = new MediaSource(); + private readonly objectUrl = URL.createObjectURL(this.mediaSource); + private readonly opened: Promise; + private sourceBuffer: ReactiveSourceBuffer | null = null; + private debug = false; + private complete = false; + + constructor(readonly metadata: SegmentStartedMessage) { + this.video.src = this.objectUrl; + this.opened = new Promise((resolve, reject) => { + const cleanup = () => { + this.mediaSource.removeEventListener('sourceopen', onOpen); + this.mediaSource.removeEventListener('sourceclose', onClose); + }; + const onOpen = () => { + cleanup(); + try { + this.sourceBuffer = new ReactiveSourceBuffer(this.mediaSource, metadata.codec); + this.sourceBuffer.setDebug(this.debug); + resolve(); + } catch (error) { + reject(error); + } + }; + const onClose = () => { + cleanup(); + reject(new Error('MediaSource closed before it opened')); + }; + + this.mediaSource.addEventListener('sourceopen', onOpen); + this.mediaSource.addEventListener('sourceclose', onClose); + }); + } + + async open(): Promise { + await this.opened; + } + + async append(data: Uint8Array): Promise { + await this.opened; + if (this.complete || !this.sourceBuffer) { + throw new Error('Cannot append to a completed clip'); + } + await this.sourceBuffer.appendBuffer(data); + } + + finish(): void { + if (this.complete) { + return; + } + if (this.mediaSource.readyState !== 'open') { + throw new Error('Cannot finish a MediaSource that is not open'); + } + this.mediaSource.endOfStream(); + this.complete = true; + } + + setDebug(debug: boolean): void { + this.debug = debug; + this.sourceBuffer?.setDebug(debug); + } + + downloadBufferedFile(): void { + this.sourceBuffer?.downloadBufferedFile(); + } + + dispose(): void { + this.video.pause(); + this.video.removeAttribute('src'); + this.video.load(); + this.video.remove(); + URL.revokeObjectURL(this.objectUrl); + } +} diff --git a/webapp/packages/shadow-player/src/playbackControls.css b/webapp/packages/shadow-player/src/playbackControls.css new file mode 100644 index 000000000..9c85d1555 --- /dev/null +++ b/webapp/packages/shadow-player/src/playbackControls.css @@ -0,0 +1,149 @@ +.control-bar { + position: absolute; + z-index: 3; + right: 0; + bottom: 0; + left: 0; + display: flex; + height: 30px; + align-items: stretch; + color: #fff; + background: rgba(43, 51, 63, 0.7); + font-family: Arial, Helvetica, sans-serif; +} + +:host(:not([controls])) .control-bar { + display: none; +} + +.control-button { + position: relative; + display: grid; + width: 40px; + min-width: 40px; + height: 30px; + padding: 7px 10px; + place-items: center; + color: inherit; + background: transparent; + border: 0; + cursor: pointer; +} + +.control-button:hover, +.control-button:focus-visible { + color: #fff; + background: rgba(255, 255, 255, 0.12); + outline: none; +} + +.control-button:focus-visible, +.timeline-segment:focus-visible, +.volume-input:focus-visible { + box-shadow: inset 0 0 0 2px #fff; +} + +.control-button svg { + width: 16px; + height: 16px; + fill: currentColor; +} + +.volume-control { + display: flex; + width: 40px; + min-width: 40px; + overflow: hidden; + align-items: center; + transition: width 120ms ease; +} + +.volume-control:hover, +.volume-control:focus-within { + width: 105px; +} + +.volume-input { + width: 0; + height: 3px; + margin: 0; + opacity: 0; + accent-color: #fff; + cursor: pointer; + transition: + width 120ms ease, + opacity 120ms ease; +} + +.volume-control:hover .volume-input, +.volume-control:focus-within .volume-input { + width: 58px; + opacity: 1; +} + +.timeline { + display: flex; + min-width: 4em; + flex: 1; + align-items: center; + touch-action: none; +} + +.timeline-segment { + position: relative; + height: 3px; + min-width: 3px; + margin-left: 3px; + flex-basis: 0; + overflow: visible; + background: rgba(115, 133, 159, 0.5); + cursor: pointer; + transition: height 80ms ease; +} + +.timeline-segment:hover, +.timeline-segment:focus-visible { + height: 10px; + outline: none; +} + +.timeline-segment[aria-disabled="true"] { + cursor: wait; +} + +.timeline-progress { + position: absolute; + inset: 0 auto 0 0; + width: 0; + background: #fff; + pointer-events: none; +} + +.time-tooltip { + position: absolute; + bottom: 15px; + left: 0; + visibility: hidden; + padding: 5px 8px; + color: #fff; + background: rgba(0, 0, 0, 0.8); + border-radius: 2px; + font-size: 12px; + line-height: 1; + pointer-events: none; + transform: translateX(-50%); + white-space: nowrap; +} + +.timeline-segment:hover .time-tooltip, +.timeline-segment:focus-visible .time-tooltip { + visibility: visible; +} + +@media (prefers-reduced-motion: reduce) { + .volume-control, + .volume-input, + .timeline-segment { + transition: none; + } +} diff --git a/webapp/packages/shadow-player/src/playbackControls.ts b/webapp/packages/shadow-player/src/playbackControls.ts new file mode 100644 index 000000000..b116c033e --- /dev/null +++ b/webapp/packages/shadow-player/src/playbackControls.ts @@ -0,0 +1,306 @@ +import styles from './playbackControls.css?inline'; + +export interface PlaybackControlLabels { + play: string; + pause: string; + mute: string; + unmute: string; + volume: string; + timeline: string; + fullscreen: string; + exitFullscreen: string; + clip: string; +} + +export const defaultPlaybackControlLabels: PlaybackControlLabels = { + play: 'Play', + pause: 'Pause', + mute: 'Mute', + unmute: 'Unmute', + volume: 'Volume', + timeline: 'Recording timeline', + fullscreen: 'Fullscreen', + exitFullscreen: 'Exit fullscreen', + clip: 'Clip', +}; + +export type PlaybackControlsAction = + | { type: 'toggle-playback' } + | { type: 'toggle-muted' } + | { type: 'set-volume'; volume: number } + | { type: 'seek'; sequence: number; percentage: number } + | { type: 'toggle-fullscreen' }; + +export type PlaybackControlsSnapshot = + | { + type: 'player'; + playing: boolean; + muted: boolean; + volume: number; + fullscreen: boolean; + } + | { + type: 'segment'; + sequence: number; + startTime: number; + duration: number; + currentTime: number; + progress: number; + playable: boolean; + } + | { type: 'labels'; labels: PlaybackControlLabels } + | { type: 'reset' }; + +const icons = { + play: '', + pause: '', + muted: + '', + volume: + '', + fullscreen: + '', + exitFullscreen: + '', +} as const; + +interface SegmentView { + state: Extract; + track: HTMLDivElement; + fill: HTMLDivElement; + tooltip: HTMLSpanElement; +} + +export class PlaybackControls { + private readonly style: HTMLStyleElement; + private readonly controlBar: HTMLDivElement; + private readonly playButton: HTMLButtonElement; + private readonly muteButton: HTMLButtonElement; + private readonly volumeInput: HTMLInputElement; + private readonly timeline: HTMLDivElement; + private readonly fullscreenButton: HTMLButtonElement; + private readonly segments = new Map(); + private labels = defaultPlaybackControlLabels; + private player = { + playing: false, + muted: true, + volume: 1, + fullscreen: false, + }; + private actionCallback: ((action: PlaybackControlsAction) => void) | null = null; + + constructor(container: HTMLElement) { + this.style = document.createElement('style'); + this.style.textContent = styles; + container.appendChild(this.style); + + this.controlBar = document.createElement('div'); + this.controlBar.className = 'control-bar'; + + this.playButton = this.createControlButton(); + this.playButton.addEventListener('click', () => this.emit({ type: 'toggle-playback' })); + this.controlBar.appendChild(this.playButton); + + const volumeControl = document.createElement('div'); + volumeControl.className = 'volume-control'; + this.muteButton = this.createControlButton(); + this.muteButton.addEventListener('click', () => this.emit({ type: 'toggle-muted' })); + volumeControl.appendChild(this.muteButton); + + this.volumeInput = document.createElement('input'); + this.volumeInput.className = 'volume-input'; + this.volumeInput.type = 'range'; + this.volumeInput.min = '0'; + this.volumeInput.max = '1'; + this.volumeInput.step = '0.05'; + this.volumeInput.addEventListener('input', () => { + this.emit({ type: 'set-volume', volume: Number.parseFloat(this.volumeInput.value) }); + }); + volumeControl.appendChild(this.volumeInput); + this.controlBar.appendChild(volumeControl); + + this.timeline = document.createElement('div'); + this.timeline.className = 'timeline'; + this.timeline.setAttribute('role', 'group'); + this.controlBar.appendChild(this.timeline); + + this.fullscreenButton = this.createControlButton(); + this.fullscreenButton.addEventListener('click', () => this.emit({ type: 'toggle-fullscreen' })); + this.controlBar.appendChild(this.fullscreenButton); + + container.appendChild(this.controlBar); + this.render({ type: 'labels', labels: this.labels }); + this.render({ type: 'player', ...this.player }); + } + + onAction(callback: (action: PlaybackControlsAction) => void): void { + this.actionCallback = callback; + } + + render(snapshot: PlaybackControlsSnapshot): void { + if (snapshot.type === 'player') { + this.renderPlayer(snapshot); + return; + } + if (snapshot.type === 'segment') { + this.renderSegment(snapshot); + return; + } + if (snapshot.type === 'labels') { + this.renderLabels(snapshot.labels); + return; + } + this.segments.clear(); + this.timeline.replaceChildren(); + } + + dispose(): void { + this.actionCallback = null; + this.segments.clear(); + this.controlBar.remove(); + this.style.remove(); + } + + private createControlButton(): HTMLButtonElement { + const button = document.createElement('button'); + button.className = 'control-button'; + button.type = 'button'; + return button; + } + + private renderPlayer(snapshot: Extract): void { + this.player = snapshot; + this.setButton( + this.playButton, + snapshot.playing ? this.labels.pause : this.labels.play, + snapshot.playing ? icons.pause : icons.play, + ); + const silent = snapshot.muted || snapshot.volume === 0; + this.setButton( + this.muteButton, + silent ? this.labels.unmute : this.labels.mute, + silent ? icons.muted : icons.volume, + ); + this.volumeInput.value = String(snapshot.volume); + this.setButton( + this.fullscreenButton, + snapshot.fullscreen ? this.labels.exitFullscreen : this.labels.fullscreen, + snapshot.fullscreen ? icons.exitFullscreen : icons.fullscreen, + ); + } + + private renderLabels(labels: PlaybackControlLabels): void { + this.labels = labels; + this.volumeInput.setAttribute('aria-label', labels.volume); + this.timeline.setAttribute('aria-label', labels.timeline); + this.render({ type: 'player', ...this.player }); + for (const view of this.segments.values()) { + this.renderSegment(view.state); + } + } + + private renderSegment(snapshot: Extract): void { + const view = this.segments.get(snapshot.sequence) ?? this.createSegment(snapshot); + view.state = snapshot; + view.track.style.flexGrow = String(Math.max(1, snapshot.duration)); + view.track.setAttribute('aria-label', `${this.labels.clip} ${snapshot.sequence + 1}`); + view.track.setAttribute('aria-disabled', String(!snapshot.playable)); + view.track.setAttribute('aria-valuenow', String(Math.round(snapshot.progress * 100))); + view.track.setAttribute('aria-valuetext', formatTime(snapshot.startTime + snapshot.currentTime)); + view.fill.style.width = `${snapshot.progress * 100}%`; + } + + private createSegment(snapshot: Extract): SegmentView { + const track = document.createElement('div'); + track.className = 'timeline-segment'; + track.tabIndex = 0; + track.setAttribute('role', 'slider'); + track.setAttribute('aria-valuemin', '0'); + track.setAttribute('aria-valuemax', '100'); + + const fill = document.createElement('div'); + fill.className = 'timeline-progress'; + track.appendChild(fill); + + const tooltip = document.createElement('span'); + tooltip.className = 'time-tooltip'; + track.appendChild(tooltip); + + const view = { state: snapshot, track, fill, tooltip }; + track.addEventListener('click', (event) => this.seekFromPointer(view, event)); + track.addEventListener('pointermove', (event) => this.renderTooltip(view, event)); + track.addEventListener('keydown', (event) => this.seekFromKeyboard(view, event)); + + this.segments.set(snapshot.sequence, view); + this.timeline.appendChild(track); + return view; + } + + private seekFromPointer(view: SegmentView, event: MouseEvent | PointerEvent): void { + if (!view.state.playable) { + return; + } + this.emit({ + type: 'seek', + sequence: view.state.sequence, + percentage: pointerPercentage(view.track, event), + }); + } + + private renderTooltip(view: SegmentView, event: PointerEvent): void { + const percentage = pointerPercentage(view.track, event); + view.tooltip.style.left = `${percentage * 100}%`; + view.tooltip.textContent = formatTime(view.state.startTime + view.state.duration * percentage); + } + + private seekFromKeyboard(view: SegmentView, event: KeyboardEvent): void { + if (!view.state.playable) { + return; + } + let percentage: number | null = null; + if (event.key === 'ArrowLeft' || event.key === 'ArrowDown') { + percentage = view.state.progress - 0.05; + } else if (event.key === 'ArrowRight' || event.key === 'ArrowUp') { + percentage = view.state.progress + 0.05; + } else if (event.key === 'Home') { + percentage = 0; + } else if (event.key === 'End') { + percentage = 1; + } + if (percentage === null) { + return; + } + event.preventDefault(); + this.emit({ + type: 'seek', + sequence: view.state.sequence, + percentage: Math.max(0, Math.min(1, percentage)), + }); + } + + private setButton(button: HTMLButtonElement, label: string, icon: string): void { + button.title = label; + button.setAttribute('aria-label', label); + button.innerHTML = icon; + } + + private emit(action: PlaybackControlsAction): void { + this.actionCallback?.(action); + } +} + +function pointerPercentage(element: HTMLElement, event: MouseEvent | PointerEvent): number { + const bounds = element.getBoundingClientRect(); + return Math.max(0, Math.min(1, (event.clientX - bounds.left) / bounds.width)); +} + +function formatTime(value: number): string { + const seconds = Math.max(0, Math.floor(value)); + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const remainder = seconds % 60; + if (hours > 0) { + return `${hours}:${String(minutes).padStart(2, '0')}:${String(remainder).padStart(2, '0')}`; + } + return `${minutes}:${String(remainder).padStart(2, '0')}`; +} diff --git a/webapp/packages/shadow-player/src/protocol.ts b/webapp/packages/shadow-player/src/protocol.ts index f942f32a3..debe367f3 100644 --- a/webapp/packages/shadow-player/src/protocol.ts +++ b/webapp/packages/shadow-player/src/protocol.ts @@ -1,81 +1,110 @@ -// Define the message types -export type ServerMessage = ChunkMessage | MetaDataMessage | ErrorMessage | EndMessage; +export type ServerMessage = ChunkMessage | SegmentStartedMessage | ErrorMessage | StreamEndedMessage; export interface ChunkMessage { type: 'chunk'; data: Uint8Array; } -export interface ErrorMessage { - type: 'error'; - error: 'UnexpectedError' | 'UnexpectedEOF'; -} - -export interface MetaDataMessage { - type: 'metadata'; +export interface SegmentStartedMessage { + type: 'segment-started'; codec: 'vp8' | 'vp9'; + sequence: number; + width?: number; + height?: number; } -export interface EndMessage { - type: 'end'; +export interface ErrorMessage { + type: 'error'; + error: 'UnexpectedError'; } -export type ClientMessageTypes = 'start' | 'pull'; +export interface StreamEndedMessage { + type: 'stream-ended'; +} export interface ClientMessage { - type: ClientMessageTypes; + type: 'start' | 'pull'; } -// Function to parse the message export function parseServerMessage(buffer: ArrayBuffer): ServerMessage { - const view = new DataView(buffer); - const typeCode = view.getUint8(0); // Read the first byte as the type code + if (buffer.byteLength === 0) { + throw new Error('Empty server message'); + } + const typeCode = new DataView(buffer).getUint8(0); if (typeCode === 0) { - // Chunk message - const chunkData = new Uint8Array(buffer, 1); // The rest is the chunk data return { type: 'chunk', - data: chunkData, + data: new Uint8Array(buffer, 1), }; } + if (typeCode === 1) { - // Metadata message (JSON) - const jsonString = new TextDecoder().decode(new Uint8Array(buffer, 1)); // Decode the rest as a string - const json = JSON.parse(jsonString); + const metadata = parseJsonPayload(buffer); + if (metadata.sequence === undefined && metadata.width === undefined && metadata.height === undefined) { + if (metadata.codec !== 'vp8' && metadata.codec !== 'vp9') { + throw new Error('Unsupported stream codec'); + } + return { + type: 'segment-started', + codec: metadata.codec, + sequence: 0, + }; + } + + if (metadata.codec !== 'vp8') { + throw new Error('Unsupported stream codec'); + } return { - type: 'metadata', - codec: json.codec === 'vp8' ? 'vp8' : 'vp9', + type: 'segment-started', + codec: metadata.codec, + sequence: readInteger(metadata.sequence, 'sequence', 0), + width: readInteger(metadata.width, 'width', 1), + height: readInteger(metadata.height, 'height', 1), }; } if (typeCode === 2) { - // Metadata message (JSON) - const jsonString = new TextDecoder().decode(new Uint8Array(buffer, 1)); // Decode the rest as a string - const json = JSON.parse(jsonString); - + const payload = parseJsonPayload(buffer); + if (payload.error !== 'UnexpectedError') { + throw new Error('Unknown server error'); + } return { type: 'error', - error: json.error, + error: payload.error, }; } if (typeCode === 3) { - return { - type: 'end', - }; + if (buffer.byteLength !== 1) { + throw new Error('Invalid stream-ended message'); + } + return { type: 'stream-ended' }; } - throw new Error('Unknown message type'); + throw new Error('Unknown server message type'); } export function parseClientMessage(message: ClientMessage): Uint8Array { if (message.type === 'start') { return new Uint8Array([0]); } - if (message.type === 'pull') { - return new Uint8Array([1]); + return new Uint8Array([1]); +} + +function parseJsonPayload(buffer: ArrayBuffer): Record { + const text = new TextDecoder('utf-8', { fatal: true }).decode(new Uint8Array(buffer, 1)); + const value: unknown = JSON.parse(text); + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid server message payload'); + } + return value as Record; +} + +function readInteger(value: unknown, field: string, minimum: number): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) { + throw new Error(`Invalid ${field}`); } - throw new Error('Unknown message type'); + return value; } diff --git a/webapp/packages/shadow-player/src/sourceBuffer.ts b/webapp/packages/shadow-player/src/sourceBuffer.ts index 8f115c7bc..3daa39f5a 100644 --- a/webapp/packages/shadow-player/src/sourceBuffer.ts +++ b/webapp/packages/shadow-player/src/sourceBuffer.ts @@ -1,105 +1,58 @@ export class ReactiveSourceBuffer { - sourceBuffer: SourceBuffer; - bufferQueue: Uint8Array[] = []; - isAppending = false; - next = () => {}; - allBuffers: Blob[] = []; // Store all buffers for file creation - debug = false; + private readonly sourceBuffer: SourceBuffer; + private readonly allBuffers: Blob[] = []; + private debug = false; - private readonly onUpdateEnd: () => void; - - constructor( - mediaSource: MediaSource, - codec: string, - next: () => void, - onUpdateEnd?: () => void - ) { + constructor(mediaSource: MediaSource, codec: string) { this.sourceBuffer = mediaSource.addSourceBuffer(`video/webm; codecs="${codec}"`); - this.next = next; - this.onUpdateEnd = onUpdateEnd ?? (() => {}); - - this.sourceBuffer.addEventListener('updateend', () => { - try { - this.onUpdateEnd(); - } finally { - this.tryAppendBuffer(); - } - }); - - // Handle errors and trigger download of the file - this.sourceBuffer.addEventListener('error', (event) => { - this.logErrorDetails(event); - this.downloadBufferedFile(); - }); } - setDebug(debug: boolean) { + setDebug(debug: boolean): void { this.debug = debug; } - appendBuffer(buffer: Uint8Array) { - this.bufferQueue.push(buffer); + async appendBuffer(buffer: Uint8Array): Promise { + if (this.sourceBuffer.updating) { + throw new Error('SourceBuffer is already updating'); + } + if (this.debug) { - this.allBuffers.push(new Blob([buffer], { type: 'video/webm' })); // Save each buffer - console.log( - `[sourceBuffer] appendBuffer: size=${buffer.length} queueLen=${this.bufferQueue.length} bufferedRanges=${this.getBufferedRanges() || '(empty)'}` - ); + this.allBuffers.push(new Blob([buffer], { type: 'video/webm' })); } - this.tryAppendBuffer(); - } - private tryAppendBuffer() { - if (!this.isAppending && !this.sourceBuffer.updating && this.bufferQueue.length > 0) { - this.isAppending = true; + await new Promise((resolve, reject) => { + const cleanup = () => { + this.sourceBuffer.removeEventListener('updateend', onUpdateEnd); + this.sourceBuffer.removeEventListener('error', onError); + }; + const onUpdateEnd = () => { + cleanup(); + resolve(); + }; + const onError = () => { + cleanup(); + reject(new Error('SourceBuffer append failed')); + }; + + this.sourceBuffer.addEventListener('updateend', onUpdateEnd); + this.sourceBuffer.addEventListener('error', onError); try { - const buffer = this.bufferQueue.shift() as Uint8Array; this.sourceBuffer.appendBuffer(buffer); } catch (error) { - this.logErrorDetails(error); - } finally { - this.next(); - this.isAppending = false; + cleanup(); + reject(error); } - } + }); } - public downloadBufferedFile() { - const completeBlob = new Blob(this.allBuffers, { type: 'video/webm' }); - const url = URL.createObjectURL(completeBlob); - - // Create a download link + downloadBufferedFile(): void { + const url = URL.createObjectURL(new Blob(this.allBuffers, { type: 'video/webm' })); const link = document.createElement('a'); link.href = url; link.download = 'buffered-video.webm'; document.body.appendChild(link); link.click(); - - // Cleanup - document.body.removeChild(link); + link.remove(); URL.revokeObjectURL(url); - console.log('Buffered file downloaded.'); - } - - private logErrorDetails(error: unknown) { - console.error('Error encountered in ReactiveSourceBuffer:'); - - // Log the error object with stack trace - console.error('Error object:', error); - - // Log the state of the bufferQueue - console.log('Current bufferQueue length:', this.bufferQueue.length); - - // Log the sourceBuffer state - console.log('SourceBuffer updating:', this.sourceBuffer.updating); - console.log('SourceBuffer buffered ranges:', this.getBufferedRanges()); - } - - private getBufferedRanges(): string { - const ranges = this.sourceBuffer.buffered; - let rangeStr = ''; - for (let i = 0; i < ranges.length; i++) { - rangeStr += `[${ranges.start(i)} - ${ranges.end(i)}] `; - } - return rangeStr.trim(); } } diff --git a/webapp/packages/shadow-player/src/streamer.css b/webapp/packages/shadow-player/src/streamer.css index 1712cc44e..41f50ca9b 100644 --- a/webapp/packages/shadow-player/src/streamer.css +++ b/webapp/packages/shadow-player/src/streamer.css @@ -1,16 +1,32 @@ +:host { + display: block; + background: #000; +} + .container { position: relative; width: 100%; height: 100%; + overflow: hidden; + background: #000; } video { + position: absolute; + inset: 0; + display: none; width: 100%; height: 100%; + object-fit: contain; +} + +video.active { + display: block; } .replay-button { position: absolute; + z-index: 2; top: 50%; left: 50%; transform: translate(-50%, -50%); @@ -22,7 +38,9 @@ video { border-radius: 50%; cursor: pointer; display: none; - transition: transform 0.2s, background-color 0.2s; + transition: + transform 0.2s, + background-color 0.2s; } .replay-button:hover { @@ -39,3 +57,9 @@ video { .replay-button.visible { display: block; } + +@media (prefers-reduced-motion: reduce) { + .replay-button { + transition: none; + } +} diff --git a/webapp/packages/shadow-player/src/streamer.ts b/webapp/packages/shadow-player/src/streamer.ts index 9bfd29da8..ac5488dd9 100644 --- a/webapp/packages/shadow-player/src/streamer.ts +++ b/webapp/packages/shadow-player/src/streamer.ts @@ -1,8 +1,16 @@ -import { ErrorMessage } from './protocol'; -import { ReactiveSourceBuffer } from './sourceBuffer'; +import { PlaybackClip } from './playbackClip'; +import { + defaultPlaybackControlLabels, + type PlaybackControlLabels, + PlaybackControls, + type PlaybackControlsAction, +} from './playbackControls'; +import type { ErrorMessage, SegmentStartedMessage, ServerMessage } from './protocol'; import styles from './streamer.css?inline'; import { ServerWebSocket } from './websocket'; +export type { PlaybackControlLabels } from './playbackControls'; + export type ShadowPlayerError = | { type: 'websocket'; @@ -15,324 +23,574 @@ export type ShadowPlayerError = | { type: 'session-not-found'; message: string; + } + | { + type: 'player'; + inner: Error; }; type ShadowPlayerErrorCallback = (error: ShadowPlayerError) => void; -const LIVE_EDGE_THRESHOLD_SECONDS = 5; -const LIVE_EDGE_SAFETY_MARGIN_SECONDS = 0.25; - export class ShadowPlayer extends HTMLElement { - shadowRoot: ShadowRoot | null = null; _videoElement: HTMLVideoElement | null = null; _src: string | null = null; - _buffer: ReactiveSourceBuffer | null = null; onErrorCallback: ShadowPlayerErrorCallback | null = null; onEndCallback: (() => void) | null = null; debug = false; _container: HTMLDivElement | null = null; _replayButton: HTMLButtonElement | null = null; + private root: ShadowRoot | null = null; private websocket: ServerWebSocket | null = null; - private isDisconnecting = false; - - static get observedAttributes() { - return ['src', 'autoplay', 'loop', 'muted', 'poster', 'preload', 'style', 'width', 'height']; + private readonly clips: PlaybackClip[] = []; + private readonly playableClips = new Set(); + private receivingClip: PlaybackClip | null = null; + private activeClip: PlaybackClip | null = null; + private awaitingResponse = false; + private shouldPlay = false; + private streamEnded = false; + private muted = true; + private volume = 1; + private controls: PlaybackControls | null = null; + private controlLabels = defaultPlaybackControlLabels; + private readonly segmentStartTimes = new Map(); + private readonly onFullscreenChange = () => this.renderPlayerControls(); + + static get observedAttributes(): string[] { + return ['src', 'autoplay', 'controls', 'loop', 'muted', 'poster', 'preload', 'style', 'width', 'height']; } - setDebug(debug: boolean) { + setDebug(debug: boolean): void { this.debug = debug; - if (this._buffer) { - this._buffer.setDebug(debug); + for (const clip of this.clips) { + clip.setDebug(debug); } } - onError(callback: ShadowPlayerErrorCallback) { + onError(callback: ShadowPlayerErrorCallback): void { this.onErrorCallback = callback; } - onEnd(callback: () => void) { - if (this._videoElement) { - this._videoElement.controls = true; - } + onEnd(callback: () => void): void { this.onEndCallback = callback; } - attributeChangedCallback(name: string, _oldValue: string, newValue: string) { + setControlLabels(labels: Partial): void { + this.controlLabels = { ...this.controlLabels, ...labels }; + this.controls?.render({ type: 'labels', labels: this.controlLabels }); + } + + attributeChangedCallback(name: string, _oldValue: string | null, newValue: string | null): void { if (name === 'src') { - this.srcChange(newValue); + if (newValue === null) { + this.disconnect(); + this._src = null; + } else if (this._container) { + this.srcChange(newValue); + } else { + this._src = newValue; + } return; } - if (this._videoElement && Object.prototype.hasOwnProperty.call(this._videoElement, name)) { - this._videoElement.setAttribute(name, newValue !== null ? newValue : ''); + if (name === 'autoplay' && newValue !== null) { + this.shouldPlay = true; + } + if (name === 'controls') { + return; + } + if (name === 'muted') { + this.setMuted(newValue !== null); + return; + } + for (const clip of this.clips) { + this.applyVideoAttribute(clip.video, name, newValue); } } - connectedCallback() { + connectedCallback(): void { this.init(); + document.addEventListener('fullscreenchange', this.onFullscreenChange); + const src = this.getAttribute('src'); + if (src !== null && !this.websocket) { + this.srcChange(src); + } } - init() { - this.shadowRoot = this.attachShadow({ mode: 'open' }); - - // Add styles - const style = document.createElement('style'); - style.textContent = styles; - this.shadowRoot.appendChild(style); + disconnectedCallback(): void { + document.removeEventListener('fullscreenchange', this.onFullscreenChange); + this.disconnect(); + this.controls?.dispose(); + this.controls = null; + } - this._container = document.createElement('div'); - this._container.className = 'container'; + init(): void { + if (!this.root) { + this.root = this.attachShadow({ mode: 'open' }); + const style = document.createElement('style'); + style.textContent = styles; + this.root.appendChild(style); - this.videoElement = document.createElement('video'); - // Set muted to true so that the browser security policy will allow autoplay. - this.videoElement.muted = true; - this._container.appendChild(this.videoElement); + this._container = document.createElement('div'); + this._container.className = 'container'; - this._replayButton = document.createElement('button'); - this._replayButton.className = 'replay-button'; - this._replayButton.innerHTML = ` + this._replayButton = document.createElement('button'); + this._replayButton.className = 'replay-button'; + this._replayButton.innerHTML = ` `; - this._replayButton.onclick = () => this.replay(); - this._container.appendChild(this._replayButton); + this._replayButton.onclick = () => this.replay(); + this._container.appendChild(this._replayButton); + this.root.appendChild(this._container); + } - this.shadowRoot.appendChild(this._container); - this.syncAttributes(); + if (!this.controls && this._container) { + this.controls = new PlaybackControls(this._container); + this.controls.onAction((action) => this.handleControlsAction(action)); + this.controls.render({ type: 'labels', labels: this.controlLabels }); + } + this.shouldPlay = this.hasAttribute('autoplay'); + this.renderPlayerControls(); } - syncAttributes() { - for (const attr of ShadowPlayer.observedAttributes) { - const value = this.getAttribute(attr); - if (attr === 'src' && value !== null) { - this.srcChange(value); + private handleControlsAction(action: PlaybackControlsAction): void { + if (action.type === 'toggle-playback') { + if (this.shouldPlay) { + this.pause(); + } else { + this.play(); } - if (value !== null && this._videoElement) { - this._videoElement.setAttribute(attr, value); + return; + } + if (action.type === 'toggle-muted') { + if (this.volume === 0) { + this.setVolume(1); } + this.setMuted(!this.muted); + return; } + if (action.type === 'set-volume') { + this.setVolume(action.volume); + this.setMuted(this.volume === 0); + return; + } + if (action.type === 'seek') { + const clip = this.clips[action.sequence]; + if (clip?.metadata.sequence === action.sequence) { + this.seekToClip(clip, action.percentage); + } + return; + } + void this.toggleFullscreen().catch((error: unknown) => this.reportPlayerError(error)); } - private get videoElement() { - return this._videoElement as HTMLVideoElement; + private setMuted(muted: boolean): void { + this.muted = muted; + for (const clip of this.clips) { + clip.video.muted = muted; + } + this.renderPlayerControls(); } - private set videoElement(value: HTMLVideoElement) { - this._videoElement = value; + private setVolume(volume: number): void { + this.volume = Math.max(0, Math.min(1, volume)); + for (const clip of this.clips) { + clip.video.volume = this.volume; + } + this.renderPlayerControls(); } - public play() { - if (this._videoElement) { - this._videoElement.play(); + private async toggleFullscreen(): Promise { + if (document.fullscreenElement === this) { + await document.exitFullscreen(); + } else { + await this.requestFullscreen(); } } - private replay() { - if (this._replayButton) { - this._replayButton.classList.remove('visible'); + public play(): void { + this.shouldPlay = true; + this.renderPlayerControls(); + if (this.activeClip && !this.activeClip.video.ended) { + void this.activeClip.video.play(); + return; } - this._videoElement?.play(); + if (this.activateNextClip()) { + return; + } + if (this.streamEnded && this.activeClip?.video.ended) { + this.replay(); + } + } + + public pause(): void { + this.shouldPlay = false; + this.activeClip?.video.pause(); + this.renderPlayerControls(); } - public srcChange(value: string) { - if (!this._videoElement) { + private replay(): void { + this._replayButton?.classList.remove('visible'); + const firstClip = this.clips[0]; + if (!firstClip) { return; } - this.isDisconnecting = false; - const mediaSource = new MediaSource(); - this._src = value; - this._videoElement.src = URL.createObjectURL(mediaSource); - mediaSource.addEventListener('sourceopen', () => { - this.handleSourceOpen(mediaSource); - }); + for (const clip of this.clips) { + clip.video.currentTime = 0; + } + this.shouldPlay = true; + this.activateClip(firstClip); + this.renderAllSegments(); + this.renderPlayerControls(); } - private async handleSourceOpen(mediaSource: MediaSource) { - this.websocket = new ServerWebSocket(this._src as string); - let reactiveSourceBuffer: ReactiveSourceBuffer | null = null; + public srcChange(value: string): void { + this.closeSession(); + this._src = value; + if (!this._container) { + return; + } - this.websocket.onopen(() => { - this.websocket!.send({ type: 'start' }); - this.websocket!.send({ type: 'pull' }); + this.streamEnded = false; + this._replayButton?.classList.remove('visible'); + this.renderPlayerControls(); + const websocket = new ServerWebSocket(value); + this.websocket = websocket; - this._videoElement?.addEventListener('ended', () => { - this.showReplayButton(); - }); + websocket.onopen(() => { + if (this.websocket === websocket) { + this.sendRequest(websocket, 'start'); + } }); + websocket.onmessage( + async (message) => this.handleServerMessage(websocket, message), + (error) => this.handlePlayerFailure(websocket, error), + ); + websocket.onclose((event) => this.handleSocketClose(websocket, event)); + websocket.onerror((event) => this.handleSocketError(websocket, event)); + } - this.websocket.onmessage((ev) => { - if (mediaSource.readyState === 'closed') { - return; - } - if (ev.type === 'metadata') { - const codec = ev.codec; - reactiveSourceBuffer = new ReactiveSourceBuffer( - mediaSource, - codec, - () => { - this.websocket?.send({ type: 'pull' }); - }, - () => this.catchUpToLiveEdge() - ); - this._buffer = reactiveSourceBuffer; - } + private async handleServerMessage(websocket: ServerWebSocket, message: ServerMessage): Promise { + if (this.websocket !== websocket) { + return; + } + if (!this.awaitingResponse) { + throw new Error('Received a server message without a pending request'); + } + this.awaitingResponse = false; - if (ev.type === 'chunk') { - if (!reactiveSourceBuffer) { - return; - } - - reactiveSourceBuffer.appendBuffer(ev.data); - - if (!this._videoElement) { - return; - } - - if (this.debug) { - const v = this._videoElement; - const buffered = v.buffered.length > 0 - ? `[${v.buffered.start(0).toFixed(2)}-${v.buffered.end(0).toFixed(2)}]` - : '(empty)'; - console.log( - `[shadow-player] chunk appended: duration=${v.duration.toFixed(2)} currentTime=${v.currentTime.toFixed(2)} buffered=${buffered} readyState=${v.readyState}` - ); - } + if (message.type === 'segment-started') { + this.sendRequest(websocket, 'pull'); + await this.startSegment(message); + return; + } + if (message.type === 'chunk') { + if (!this.receivingClip) { + throw new Error('Received a chunk before a segment started'); } + this.sendRequest(websocket, 'pull'); + await this.receivingClip.append(message.data); + return; + } + if (message.type === 'error') { + this.onErrorCallback?.({ type: 'protocol', inner: message }); + return; + } - if (ev.type === 'error') { - this.onErrorCallback?.({ - type: 'protocol', - inner: ev, - }); - } + this.finishReceivingClip(); + this.streamEnded = true; + this.renderPlayerControls(); + if (this.activeClip?.video.ended) { + this.showReplayButton(); + } + this.onEndCallback?.(); + } + + private async startSegment(metadata: SegmentStartedMessage): Promise { + if (metadata.sequence !== this.clips.length) { + throw new Error(`Expected segment ${this.clips.length}, received ${metadata.sequence}`); + } + + this.finishReceivingClip(); + const clip = new PlaybackClip(metadata); + clip.setDebug(this.debug); + this.configureVideo(clip); + this.clips.push(clip); + this.receivingClip = clip; + this._container?.insertBefore(clip.video, this._replayButton); + this.renderAllSegments(); + await clip.open(); + } + + private finishReceivingClip(): void { + const clip = this.receivingClip; + if (!clip) { + return; + } + clip.finish(); + this.renderAllSegments(); + } - if (ev.type === 'end') { - this.onEndCallback?.(); + private configureVideo(clip: PlaybackClip): void { + const video = clip.video; + video.className = 'clip'; + video.muted = this.muted; + video.volume = this.volume; + for (const attribute of ShadowPlayer.observedAttributes) { + if (attribute !== 'src' && attribute !== 'controls' && attribute !== 'muted') { + this.applyVideoAttribute(video, attribute, this.getAttribute(attribute)); + } + } + video.addEventListener( + 'loadeddata', + () => { + this.playableClips.add(clip); + this.activateNextClip(); + this.renderAllSegments(); + }, + { once: true }, + ); + video.addEventListener('play', () => { + if (this.activeClip === clip) { + this.shouldPlay = true; + this.renderPlayerControls(); } }); - - this.websocket.onclose((ev) => { - if (this.isDisconnecting) { - this.websocket = null; - return; + video.addEventListener('pause', () => { + if (this.activeClip === clip && !video.ended) { + this.shouldPlay = false; + this.renderPlayerControls(); } - - if (ev.code === 4001) { - this.onErrorCallback?.({ - type: 'session-not-found', - message: 'Recording session is no longer active', - }); + }); + video.addEventListener('ended', () => { + if (this.activeClip !== clip) { + return; } - - this.videoElement.controls = true; - if (reactiveSourceBuffer && mediaSource.readyState === 'open') { - try { - if (this.debug && this._videoElement) { - const v = this._videoElement; - const buffered = v.buffered.length > 0 - ? `[${v.buffered.start(0).toFixed(2)}-${v.buffered.end(0).toFixed(2)}]` - : '(empty)'; - console.log( - `[shadow-player] BEFORE endOfStream: duration=${v.duration} currentTime=${v.currentTime.toFixed(2)} buffered=${buffered} mediaSource.readyState=${mediaSource.readyState}` - ); - } - mediaSource.endOfStream(); - if (this.debug && this._videoElement) { - const v = this._videoElement; - const buffered = v.buffered.length > 0 - ? `[${v.buffered.start(0).toFixed(2)}-${v.buffered.end(0).toFixed(2)}]` - : '(empty)'; - console.log( - `[shadow-player] AFTER endOfStream: duration=${v.duration} currentTime=${v.currentTime.toFixed(2)} buffered=${buffered} mediaSource.readyState=${mediaSource.readyState}` - ); - } - } catch (error) { - if (this.debug) { - console.error('[shadow-player] endOfStream error:', error); - } - } + if (!this.activateNextClip() && this.streamEnded) { + this.showReplayButton(); } - this.websocket = null; + this.renderClipControls(clip); + this.renderPlayerControls(); }); + video.addEventListener('timeupdate', () => this.renderClipControls(clip)); + video.addEventListener('durationchange', () => this.renderAllSegments()); + video.addEventListener('progress', () => this.renderClipControls(clip)); + video.addEventListener('click', () => this.handleControlsAction({ type: 'toggle-playback' })); + } - this.websocket.onerror((ev) => { - if (this.isDisconnecting) { - return; + private activateNextClip(): boolean { + const sequence = this.activeClip ? this.activeClip.metadata.sequence + 1 : 0; + const next = this.clips[sequence]; + if (!next || !this.playableClips.has(next)) { + return false; + } + if (this.activeClip && !this.activeClip.video.ended) { + return false; + } + this.activateClip(next); + return true; + } + + private activateClip(clip: PlaybackClip): void { + if (this.activeClip === clip) { + if (this.shouldPlay) { + void clip.video.play(); } + this.renderClipControls(clip); + this.renderPlayerControls(); + return; + } + const previous = this.activeClip; + this.activeClip = clip; + if (previous) { + previous.video.pause(); + previous.video.classList.remove('active'); + } + this._videoElement = clip.video; + clip.video.classList.add('active'); + if (this.shouldPlay) { + void clip.video.play(); + } + if (previous) { + this.renderClipControls(previous); + } + this.renderClipControls(clip); + this.renderPlayerControls(); + } - this.onErrorCallback?.({ - type: 'websocket', - inner: ev as unknown as ErrorEvent, - }); + private seekToClip(clip: PlaybackClip, percentage: number): void { + if (!this.playableClips.has(clip)) { + return; + } + const duration = this.clipDuration(clip); + if (duration <= 0) { + return; + } - if (reactiveSourceBuffer && mediaSource.readyState === 'open') { - try { - mediaSource.endOfStream(); - } catch (error) { - console.error('endOfStream error:', error); - } + for (const laterClip of this.clips) { + if (laterClip.metadata.sequence > clip.metadata.sequence && this.playableClips.has(laterClip)) { + laterClip.video.currentTime = 0; } + } + clip.video.currentTime = duration * Math.max(0, Math.min(1, percentage)); + this._replayButton?.classList.remove('visible'); + this.activateClip(clip); + this.renderAllSegments(); + } + + private clipDuration(clip: PlaybackClip): number { + if (Number.isFinite(clip.video.duration) && clip.video.duration > 0) { + return clip.video.duration; + } + const buffered = clip.video.buffered; + return buffered.length > 0 ? buffered.end(buffered.length - 1) : 0; + } + + private clipProgress(clip: PlaybackClip): number { + if (!this.activeClip) { + return 0; + } + if (clip.metadata.sequence < this.activeClip.metadata.sequence) { + return 1; + } + if (clip !== this.activeClip) { + return 0; + } + const duration = this.clipDuration(clip); + return duration > 0 ? Math.max(0, Math.min(1, clip.video.currentTime / duration)) : 0; + } + + private renderPlayerControls(): void { + this.controls?.render({ + type: 'player', + playing: this.shouldPlay, + muted: this.muted, + volume: this.volume, + fullscreen: document.fullscreenElement === this, + }); + } + + private renderClipControls(clip: PlaybackClip): void { + const startTime = this.segmentStartTimes.get(clip); + if (startTime === undefined) { + return; + } + const duration = this.clipDuration(clip); + const progress = this.clipProgress(clip); + this.controls?.render({ + type: 'segment', + sequence: clip.metadata.sequence, + startTime, + duration, + currentTime: duration * progress, + progress, + playable: this.playableClips.has(clip), }); } - public downloadBUfferAsFile() { - if (this._buffer && this.debug) { - this._buffer.downloadBufferedFile(); + private renderAllSegments(): void { + let startTime = 0; + for (const clip of this.clips) { + this.segmentStartTimes.set(clip, startTime); + this.renderClipControls(clip); + startTime += this.clipDuration(clip); } } - private showReplayButton() { - if (this._replayButton) { - this._replayButton.classList.add('visible'); + private applyVideoAttribute(video: HTMLVideoElement, name: string, value: string | null): void { + if (value === null) { + video.removeAttribute(name); + } else { + video.setAttribute(name, value); } } - private catchUpToLiveEdge() { - const video = this._videoElement; - if (!video || video.buffered.length === 0) { + private sendRequest(websocket: ServerWebSocket, type: 'start' | 'pull'): void { + if (this.websocket !== websocket) { return; } + if (this.awaitingResponse) { + throw new Error('A stream request is already pending'); + } + this.awaitingResponse = true; + websocket.send({ type }); + } - const latestRangeIndex = video.buffered.length - 1; - const latestRangeStart = video.buffered.start(latestRangeIndex); - const latestRangeEnd = video.buffered.end(latestRangeIndex); - const isOutsideLatestRange = - video.currentTime < latestRangeStart || video.currentTime >= latestRangeEnd; + private handleSocketClose(websocket: ServerWebSocket, event: CloseEvent): void { + if (this.websocket !== websocket) { + return; + } + this.awaitingResponse = false; + this.websocket = null; + if (event.code === 4001) { + this.onErrorCallback?.({ + type: 'session-not-found', + message: 'Recording session is no longer active', + }); + } + this.renderPlayerControls(); + } - if ( - isOutsideLatestRange || - latestRangeEnd - video.currentTime > LIVE_EDGE_THRESHOLD_SECONDS - ) { - video.currentTime = Math.max( - latestRangeStart, - latestRangeEnd - LIVE_EDGE_SAFETY_MARGIN_SECONDS - ); + private handleSocketError(websocket: ServerWebSocket, event: Event): void { + if (this.websocket !== websocket) { + return; } + this.onErrorCallback?.({ + type: 'websocket', + inner: event as ErrorEvent, + }); } - public disconnect(): void { - this.isDisconnecting = true; + private handlePlayerFailure(websocket: ServerWebSocket, value: unknown): void { + if (this.websocket !== websocket) { + return; + } + const error = value instanceof Error ? value : new Error(String(value)); + this.awaitingResponse = false; + this.onErrorCallback?.({ type: 'player', inner: error }); + websocket.close(1000, 'Player failure'); + this.websocket = null; + this.renderPlayerControls(); + } - if (this.websocket) { - try { - this.websocket.ws.close(1000, 'Component cleanup'); - } catch (error) { - // Intentionally ignored: WebSocket may already be closed - } - this.websocket = null; + private reportPlayerError(value: unknown): void { + const error = value instanceof Error ? value : new Error(String(value)); + this.onErrorCallback?.({ type: 'player', inner: error }); + } + + public downloadBUfferAsFile(): void { + if (this.debug) { + (this.receivingClip ?? this.activeClip)?.downloadBufferedFile(); } + } - if (this._videoElement) { - try { - this._videoElement.pause(); - this._videoElement.src = ''; - this._videoElement.load(); - } catch (error) { - // Intentionally ignored: Video element may already be in an invalid state - } + private showReplayButton(): void { + this._replayButton?.classList.add('visible'); + this.renderPlayerControls(); + } + + public disconnect(): void { + this.closeSession(); + } + + private closeSession(): void { + const websocket = this.websocket; + this.websocket = null; + this.awaitingResponse = false; + websocket?.close(1000, 'Component cleanup'); + for (const clip of this.clips) { + clip.dispose(); } + this.clips.length = 0; + this.playableClips.clear(); + this.segmentStartTimes.clear(); + this.receivingClip = null; + this.activeClip = null; + this._videoElement = null; + this.controls?.render({ type: 'reset' }); + this.renderPlayerControls(); } } diff --git a/webapp/packages/shadow-player/src/websocket.ts b/webapp/packages/shadow-player/src/websocket.ts index 0d690b26c..99211c712 100644 --- a/webapp/packages/shadow-player/src/websocket.ts +++ b/webapp/packages/shadow-player/src/websocket.ts @@ -1,41 +1,46 @@ -import { ClientMessage, ServerMessage, parseClientMessage, parseServerMessage } from './protocol'; +import { ClientMessage, parseClientMessage, parseServerMessage, ServerMessage } from './protocol'; export class ServerWebSocket { - ws: WebSocket; + private readonly socket: WebSocket; + constructor(url: string) { - this.ws = new WebSocket(url); + this.socket = new WebSocket(url); + this.socket.binaryType = 'arraybuffer'; } - onopen(callback: (ev: Event) => unknown) { - this.ws.onopen = callback; + onopen(callback: (event: Event) => void): void { + this.socket.onopen = callback; } - onmessage(callback: (ev: ServerMessage) => unknown) { - this.ws.onmessage = (ev) => { - const reader = new FileReader(); - reader.onload = () => { - const arrayBuffer = reader.result as ArrayBuffer; - const serverResponse = parseServerMessage(arrayBuffer); - callback(serverResponse); - }; - - reader.readAsArrayBuffer(ev.data); + onmessage(callback: (message: ServerMessage) => Promise | void, onFailure: (error: unknown) => void): void { + this.socket.onmessage = (event) => { + try { + if (!(event.data instanceof ArrayBuffer)) { + throw new Error('Server sent a non-binary message'); + } + Promise.resolve(callback(parseServerMessage(event.data))).catch(onFailure); + } catch (error) { + onFailure(error); + } }; } - onclose(callback: (ev: CloseEvent) => unknown) { - this.ws.onclose = callback; + onclose(callback: (event: CloseEvent) => void): void { + this.socket.onclose = callback; } - onerror(callback: (ev: Event) => unknown) { - this.ws.onerror = callback; + onerror(callback: (event: Event) => void): void { + this.socket.onerror = callback; } - send(data: T) { - this.ws.send(parseClientMessage(data)); + send(message: ClientMessage): void { + if (this.socket.readyState !== WebSocket.OPEN) { + throw new Error('WebSocket is not open'); + } + this.socket.send(parseClientMessage(message)); } - isClosed() { - return this.ws.readyState === WebSocket.CLOSED; + close(code: number, reason: string): void { + this.socket.close(code, reason); } } From 8dfc9ff39389419241767e8e8d1bbef02926371c Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Mon, 24 Aug 2026 20:58:01 -0400 Subject: [PATCH 7/9] fix(webapp): play every live stream segment Queue metadata-delimited WebM clips as independent playback sources. Serialize SourceBuffer completion and terminal events so delayed segments cannot be skipped. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- webapp/packages/shadow-player/package.json | 7 +- .../shadow-player/src/playbackClip.test.ts | 93 ++++ .../shadow-player/src/playbackClip.ts | 23 +- .../shadow-player/src/protocol.test.ts | 39 ++ .../shadow-player/src/sourceBuffer.test.ts | 53 ++ .../shadow-player/src/sourceBuffer.ts | 13 +- .../shadow-player/src/streamer.test.ts | 512 ++++++++++++++++++ webapp/packages/shadow-player/src/streamer.ts | 144 ++++- .../shadow-player/src/websocket.test.ts | 142 +++++ .../packages/shadow-player/src/websocket.ts | 36 +- webapp/packages/shadow-player/vite.config.ts | 4 +- webapp/pnpm-lock.yaml | 6 + 12 files changed, 1022 insertions(+), 50 deletions(-) create mode 100644 webapp/packages/shadow-player/src/playbackClip.test.ts create mode 100644 webapp/packages/shadow-player/src/protocol.test.ts create mode 100644 webapp/packages/shadow-player/src/sourceBuffer.test.ts create mode 100644 webapp/packages/shadow-player/src/streamer.test.ts create mode 100644 webapp/packages/shadow-player/src/websocket.test.ts diff --git a/webapp/packages/shadow-player/package.json b/webapp/packages/shadow-player/package.json index fddd1d8a6..d8ee79f78 100644 --- a/webapp/packages/shadow-player/package.json +++ b/webapp/packages/shadow-player/package.json @@ -8,13 +8,16 @@ "scripts": { "dev": "vite", "build": "tsc && vite build", - "preview": "vite preview" + "preview": "vite preview", + "test": "vitest run" }, "devDependencies": { + "jsdom": "^20.0.3", "ts-node": "^10.9.2", "typescript": "~5.6.2", "vite": "^5.4.9", "vite-plugin-dts": "^4.3.0", - "vite-plugin-static-copy": "^2.3.0" + "vite-plugin-static-copy": "^2.3.0", + "vitest": "^3.1.1" } } diff --git a/webapp/packages/shadow-player/src/playbackClip.test.ts b/webapp/packages/shadow-player/src/playbackClip.test.ts new file mode 100644 index 000000000..d636d9893 --- /dev/null +++ b/webapp/packages/shadow-player/src/playbackClip.test.ts @@ -0,0 +1,93 @@ +// @vitest-environment jsdom + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { PlaybackClip } from './playbackClip'; + +class FakeSourceBuffer extends EventTarget { + updating = false; + + appendBuffer(): void { + if (this.updating) { + throw new Error('concurrent append'); + } + this.updating = true; + } + + completeAppend(): void { + this.updating = false; + this.dispatchEvent(new Event('updateend')); + } +} + +class FakeMediaSource extends EventTarget { + static latest: FakeMediaSource | null = null; + + readyState: ReadyState = 'closed'; + readonly sourceBuffer = new FakeSourceBuffer(); + endOfStreamCalls = 0; + + constructor() { + super(); + FakeMediaSource.latest = this; + } + + addSourceBuffer(): SourceBuffer { + return this.sourceBuffer as unknown as SourceBuffer; + } + + open(): void { + this.readyState = 'open'; + this.dispatchEvent(new Event('sourceopen')); + } + + endOfStream(): void { + if (this.sourceBuffer.updating) { + throw new Error('endOfStream during append'); + } + this.endOfStreamCalls += 1; + this.readyState = 'ended'; + } +} + +describe('PlaybackClip', () => { + beforeEach(() => { + vi.stubGlobal('MediaSource', FakeMediaSource); + vi.stubGlobal('URL', { + createObjectURL: vi.fn(() => 'blob:test'), + revokeObjectURL: vi.fn(), + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + FakeMediaSource.latest = null; + }); + + it('waits for pending SourceBuffer work before ending the MediaSource', async () => { + const clip = new PlaybackClip({ + type: 'segment-started', + codec: 'vp8', + sequence: 0, + width: 640, + height: 480, + }); + const mediaSource = FakeMediaSource.latest; + expect(mediaSource).not.toBeNull(); + mediaSource?.open(); + await clip.open(); + + const append = clip.append(new Uint8Array([1])); + const finish = Promise.resolve().then(() => clip.finish()); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + expect(mediaSource?.sourceBuffer.updating).toBe(true); + expect(mediaSource?.endOfStreamCalls).toBe(0); + + mediaSource?.sourceBuffer.completeAppend(); + await append; + await finish; + expect(mediaSource?.endOfStreamCalls).toBe(1); + }); +}); diff --git a/webapp/packages/shadow-player/src/playbackClip.ts b/webapp/packages/shadow-player/src/playbackClip.ts index 3844bae72..5b08b5328 100644 --- a/webapp/packages/shadow-player/src/playbackClip.ts +++ b/webapp/packages/shadow-player/src/playbackClip.ts @@ -10,6 +10,7 @@ export class PlaybackClip { private sourceBuffer: ReactiveSourceBuffer | null = null; private debug = false; private complete = false; + private finishing: Promise | null = null; constructor(readonly metadata: SegmentStartedMessage) { this.video.src = this.objectUrl; @@ -50,15 +51,25 @@ export class PlaybackClip { await this.sourceBuffer.appendBuffer(data); } - finish(): void { - if (this.complete) { - return; + async finish(): Promise { + await this.opened; + if (this.finishing) { + return this.finishing; } - if (this.mediaSource.readyState !== 'open') { - throw new Error('Cannot finish a MediaSource that is not open'); + const sourceBuffer = this.sourceBuffer; + if (this.complete || !sourceBuffer) { + return; } - this.mediaSource.endOfStream(); + this.complete = true; + this.finishing = (async () => { + await sourceBuffer.whenIdle(); + if (this.mediaSource.readyState !== 'open') { + throw new Error('Cannot finish a MediaSource that is not open'); + } + this.mediaSource.endOfStream(); + })(); + return this.finishing; } setDebug(debug: boolean): void { diff --git a/webapp/packages/shadow-player/src/protocol.test.ts b/webapp/packages/shadow-player/src/protocol.test.ts new file mode 100644 index 000000000..92fba7065 --- /dev/null +++ b/webapp/packages/shadow-player/src/protocol.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { parseServerMessage } from './protocol'; + +function encodedMessage(type: number, payload = ''): ArrayBuffer { + const encodedPayload = new TextEncoder().encode(payload); + const message = new Uint8Array(1 + encodedPayload.length); + message[0] = type; + message.set(encodedPayload, 1); + return message.buffer; +} + +describe('parseServerMessage', () => { + it('accepts legacy one-segment metadata', () => { + expect(parseServerMessage(encodedMessage(1, '{"codec":"vp9"}'))).toEqual({ + type: 'segment-started', + codec: 'vp9', + sequence: 0, + }); + }); + + it('parses independent segment metadata', () => { + expect(parseServerMessage(encodedMessage(1, '{"codec":"vp8","sequence":2,"width":1280,"height":720}'))).toEqual({ + type: 'segment-started', + codec: 'vp8', + sequence: 2, + width: 1280, + height: 720, + }); + }); + + it('requires stream-ended to have no payload', () => { + expect(parseServerMessage(encodedMessage(3))).toEqual({ type: 'stream-ended' }); + expect(() => parseServerMessage(encodedMessage(3, 'unexpected'))).toThrow('Invalid stream-ended message'); + }); + + it('rejects partially extended metadata', () => { + expect(() => parseServerMessage(encodedMessage(1, '{"codec":"vp8","sequence":0}'))).toThrow('Invalid width'); + }); +}); diff --git a/webapp/packages/shadow-player/src/sourceBuffer.test.ts b/webapp/packages/shadow-player/src/sourceBuffer.test.ts new file mode 100644 index 000000000..a98cd4c04 --- /dev/null +++ b/webapp/packages/shadow-player/src/sourceBuffer.test.ts @@ -0,0 +1,53 @@ +// @vitest-environment jsdom + +import { describe, expect, it } from 'vitest'; +import { ReactiveSourceBuffer } from './sourceBuffer'; + +class FakeSourceBuffer extends EventTarget { + updating = false; + readonly appended: Uint8Array[] = []; + + appendBuffer(buffer: BufferSource): void { + if (this.updating) { + throw new Error('concurrent append'); + } + this.updating = true; + const bytes = + buffer instanceof ArrayBuffer + ? new Uint8Array(buffer) + : new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + this.appended.push(Uint8Array.from(bytes)); + } + + completeAppend(): void { + this.updating = false; + this.dispatchEvent(new Event('updateend')); + } +} + +describe('ReactiveSourceBuffer', () => { + it('serializes append operations', async () => { + const sourceBuffer = new FakeSourceBuffer(); + const mediaSource = { + addSourceBuffer: () => sourceBuffer, + } as unknown as MediaSource; + const reactive = new ReactiveSourceBuffer(mediaSource, 'vp8'); + + const first = reactive.appendBuffer(new Uint8Array([1])); + const secondResult = reactive.appendBuffer(new Uint8Array([2])).then( + () => null, + (error: unknown) => error, + ); + + await Promise.resolve(); + expect(sourceBuffer.appended).toEqual([new Uint8Array([1])]); + + sourceBuffer.completeAppend(); + await first; + await Promise.resolve(); + expect(sourceBuffer.appended).toEqual([new Uint8Array([1]), new Uint8Array([2])]); + + sourceBuffer.completeAppend(); + expect(await secondResult).toBeNull(); + }); +}); diff --git a/webapp/packages/shadow-player/src/sourceBuffer.ts b/webapp/packages/shadow-player/src/sourceBuffer.ts index 3daa39f5a..c86cfcd66 100644 --- a/webapp/packages/shadow-player/src/sourceBuffer.ts +++ b/webapp/packages/shadow-player/src/sourceBuffer.ts @@ -1,6 +1,7 @@ export class ReactiveSourceBuffer { private readonly sourceBuffer: SourceBuffer; private readonly allBuffers: Blob[] = []; + private pendingOperation = Promise.resolve(); private debug = false; constructor(mediaSource: MediaSource, codec: string) { @@ -11,7 +12,17 @@ export class ReactiveSourceBuffer { this.debug = debug; } - async appendBuffer(buffer: Uint8Array): Promise { + appendBuffer(buffer: Uint8Array): Promise { + const operation = this.pendingOperation.then(() => this.append(buffer)); + this.pendingOperation = operation; + return operation; + } + + whenIdle(): Promise { + return this.pendingOperation; + } + + private async append(buffer: Uint8Array): Promise { if (this.sourceBuffer.updating) { throw new Error('SourceBuffer is already updating'); } diff --git a/webapp/packages/shadow-player/src/streamer.test.ts b/webapp/packages/shadow-player/src/streamer.test.ts new file mode 100644 index 000000000..5a077aa53 --- /dev/null +++ b/webapp/packages/shadow-player/src/streamer.test.ts @@ -0,0 +1,512 @@ +// @vitest-environment jsdom + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { ClientMessage, SegmentStartedMessage, ServerMessage } from './protocol'; + +interface MockServerWebSocket { + sent: ClientMessage[]; + emitOpen: () => void; + emitMessage: (message: ServerMessage) => Promise; + emitClose: (code?: number, reason?: string) => void; + emitError: () => void; +} + +interface MockPlaybackClip { + metadata: SegmentStartedMessage; + video: HTMLVideoElement; + play: ReturnType; + pause: ReturnType; + open: ReturnType; + append: ReturnType; + finish: ReturnType; + resolveOpen: () => void; + resolveAppend: () => void; + resolveFinish: () => void; + setDuration: (duration: number) => void; + loaded: () => void; + end: () => void; +} + +const mocks = vi.hoisted(() => ({ + sockets: [] as MockServerWebSocket[], + clips: [] as MockPlaybackClip[], +})); + +vi.mock('./websocket', () => ({ + ServerWebSocket: class { + readonly sent: ClientMessage[] = []; + private openCallback: (() => void) | null = null; + private messageCallback: ((message: ServerMessage) => Promise | void) | null = null; + private closeCallback: ((event: CloseEvent) => void) | null = null; + private errorCallback: ((event: Event) => void) | null = null; + private failureCallback: ((error: unknown) => void) | null = null; + + constructor(_url: string) { + mocks.sockets.push(this); + } + + onopen(callback: () => void): void { + this.openCallback = callback; + } + + onmessage(callback: (message: ServerMessage) => Promise | void, onFailure: (error: unknown) => void): void { + this.messageCallback = callback; + this.failureCallback = onFailure; + } + + onclose(callback: (event: CloseEvent) => void): void { + this.closeCallback = callback; + } + + onerror(callback: (event: Event) => void): void { + this.errorCallback = callback; + } + + send(message: ClientMessage): void { + this.sent.push(message); + } + + isOpen(): boolean { + return true; + } + + close(): void {} + + emitOpen(): void { + this.openCallback?.(); + } + + async emitMessage(message: ServerMessage): Promise { + try { + await this.messageCallback?.(message); + } catch (error) { + this.failureCallback?.(error); + throw error; + } + } + + emitClose(code = 1006, reason = ''): void { + this.closeCallback?.(new CloseEvent('close', { code, reason, wasClean: false })); + } + + emitError(): void { + this.errorCallback?.(new Event('error')); + } + }, +})); + +vi.mock('./playbackClip', () => ({ + PlaybackClip: class { + readonly video = document.createElement('video'); + readonly play = vi.fn(async () => undefined); + readonly pause = vi.fn(); + readonly open: ReturnType; + readonly append: ReturnType; + readonly finish: ReturnType; + private readonly openPromise: Promise; + private readonly appendPromise: Promise; + private readonly finishPromise: Promise; + private openResolver!: () => void; + private appendResolver!: () => void; + private finishResolver!: () => void; + private duration = 0; + private ended = false; + + constructor(readonly metadata: SegmentStartedMessage) { + this.openPromise = new Promise((resolve) => { + this.openResolver = resolve; + }); + this.appendPromise = new Promise((resolve) => { + this.appendResolver = resolve; + }); + this.finishPromise = new Promise((resolve) => { + this.finishResolver = resolve; + }); + this.open = vi.fn(() => this.openPromise); + this.append = vi.fn(() => this.appendPromise); + this.finish = vi.fn(() => this.finishPromise); + Object.defineProperties(this.video, { + play: { configurable: true, value: this.play }, + pause: { configurable: true, value: this.pause }, + load: { configurable: true, value: vi.fn() }, + duration: { configurable: true, get: () => this.duration }, + ended: { configurable: true, get: () => this.ended }, + }); + mocks.clips.push(this); + } + + resolveOpen(): void { + this.openResolver(); + } + + resolveAppend(): void { + this.appendResolver(); + } + + resolveFinish(): void { + this.finishResolver(); + } + + setDuration(duration: number): void { + this.duration = duration; + this.video.dispatchEvent(new Event('durationchange')); + } + + loaded(): void { + this.video.dispatchEvent(new Event('loadeddata')); + } + + end(): void { + this.ended = true; + this.video.dispatchEvent(new Event('ended')); + } + + setDebug(): void {} + + downloadBufferedFile(): void {} + + dispose(): void { + this.video.remove(); + } + }, +})); + +import { ShadowPlayer } from './streamer'; + +async function flushMicrotasks(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +function createPlayer(attributes: string[] = []): { player: ShadowPlayer; socket: MockServerWebSocket } { + const player = new ShadowPlayer(); + for (const attribute of attributes) { + player.setAttribute(attribute, ''); + } + player.setAttribute('src', 'ws://example.test'); + document.body.appendChild(player); + const socket = mocks.sockets.at(-1); + if (!socket) { + throw new Error('ShadowPlayer did not create a websocket'); + } + socket.emitOpen(); + return { player, socket }; +} + +const firstMetadata: SegmentStartedMessage = { + type: 'segment-started', + codec: 'vp8', + sequence: 0, + width: 640, + height: 480, +}; + +const secondMetadata: SegmentStartedMessage = { + type: 'segment-started', + codec: 'vp8', + sequence: 1, + width: 1280, + height: 720, +}; + +describe('ShadowPlayer', () => { + beforeEach(() => { + mocks.sockets.length = 0; + mocks.clips.length = 0; + }); + + afterEach(() => { + document.body.replaceChildren(); + }); + + it('pulls only after segment and append work completes', async () => { + const { player, socket } = createPlayer(); + const onEnd = vi.fn(); + player.onEnd(onEnd); + expect(socket.sent).toEqual([{ type: 'start' }]); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + expect(firstClip).toBeDefined(); + expect(socket.sent).toEqual([{ type: 'start' }]); + + firstClip.resolveOpen(); + await firstStart; + expect(socket.sent).toEqual([{ type: 'start' }, { type: 'pull' }]); + + const chunk = socket.emitMessage({ type: 'chunk', data: new Uint8Array([1]) }); + await flushMicrotasks(); + expect(socket.sent).toHaveLength(2); + firstClip.resolveAppend(); + await chunk; + expect(socket.sent).toHaveLength(3); + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + expect(firstClip.finish).toHaveBeenCalledOnce(); + expect(mocks.clips).toHaveLength(1); + + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + expect(secondClip).toBeDefined(); + secondClip.resolveOpen(); + await secondStart; + expect(socket.sent).toHaveLength(4); + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + expect(secondClip.finish).toHaveBeenCalledOnce(); + expect(onEnd).not.toHaveBeenCalled(); + + secondClip.resolveFinish(); + await streamEnd; + expect(onEnd).toHaveBeenCalledOnce(); + expect(socket.sent).toHaveLength(4); + }); + + it('does not turn an abrupt close into a clean stream end', async () => { + const { player, socket } = createPlayer(); + const onEnd = vi.fn(); + player.onEnd(onEnd); + + const start = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const clip = mocks.clips[0]; + clip.resolveOpen(); + await start; + + socket.emitClose(); + expect(clip.finish).not.toHaveBeenCalled(); + expect(onEnd).not.toHaveBeenCalled(); + }); + + it.each([4002, 4003, 1011])('surfaces unexpected close code %i', (code) => { + const { player, socket } = createPlayer(); + const onError = vi.fn(); + player.onError(onError); + + socket.emitClose(code, `close ${code}`); + + expect(onError).toHaveBeenCalledOnce(); + expect(onError).toHaveBeenCalledWith({ + type: 'websocket-close', + code, + reason: `close ${code}`, + wasClean: false, + }); + }); + + it('reports only the clean End when a socket error follows it', async () => { + const { player, socket } = createPlayer(); + const onEnd = vi.fn(); + const onError = vi.fn(); + player.onEnd(onEnd); + player.onError(onError); + + const start = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const clip = mocks.clips[0]; + clip.resolveOpen(); + await start; + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + clip.resolveFinish(); + await streamEnd; + socket.emitError(); + + expect(onEnd).toHaveBeenCalledOnce(); + expect(onError).not.toHaveBeenCalled(); + }); + + it('rejects a noncontiguous segment sequence', async () => { + const { socket } = createPlayer(); + + await expect(socket.emitMessage({ ...firstMetadata, sequence: 1 })).rejects.toThrow( + 'Expected segment 0, received 1', + ); + expect(mocks.clips).toHaveLength(0); + }); + + it('does not loop from stream completion while the next segment is not yet playable', async () => { + const { player, socket } = createPlayer(['autoplay', 'loop']); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + firstClip.resolveOpen(); + await firstStart; + firstClip.loaded(); + expect(firstClip.play).toHaveBeenCalledOnce(); + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + secondClip.resolveOpen(); + await secondStart; + + firstClip.end(); + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + secondClip.resolveFinish(); + await streamEnd; + + expect(player._videoElement).toBe(firstClip.video); + expect(firstClip.play).toHaveBeenCalledOnce(); + expect(secondClip.play).not.toHaveBeenCalled(); + + secondClip.loaded(); + expect(player._videoElement).toBe(secondClip.video); + expect(firstClip.play).toHaveBeenCalledOnce(); + expect(secondClip.play).toHaveBeenCalledOnce(); + }); + + it('does not show replay from the ended handler while the next segment is not yet playable', async () => { + const { player, socket } = createPlayer(['autoplay']); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + firstClip.resolveOpen(); + await firstStart; + firstClip.loaded(); + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + secondClip.resolveOpen(); + await secondStart; + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + secondClip.resolveFinish(); + await streamEnd; + firstClip.end(); + + const replayButton = player.shadowRoot?.querySelector('.replay-button'); + expect(player._videoElement).toBe(firstClip.video); + expect(replayButton?.classList.contains('visible')).toBe(false); + expect(secondClip.play).not.toHaveBeenCalled(); + + secondClip.loaded(); + expect(player._videoElement).toBe(secondClip.video); + expect(secondClip.play).toHaveBeenCalledOnce(); + expect(replayButton?.classList.contains('visible')).toBe(false); + }); + + it('coordinates autoplay and loop across the full segment sequence', async () => { + const { player, socket } = createPlayer(['autoplay', 'loop']); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + firstClip.resolveOpen(); + await firstStart; + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + secondClip.resolveOpen(); + await secondStart; + + expect(firstClip.video.hasAttribute('autoplay')).toBe(false); + expect(firstClip.video.hasAttribute('loop')).toBe(false); + expect(secondClip.video.hasAttribute('autoplay')).toBe(false); + expect(secondClip.video.hasAttribute('loop')).toBe(false); + + secondClip.loaded(); + expect(secondClip.play).not.toHaveBeenCalled(); + firstClip.loaded(); + expect(firstClip.play).toHaveBeenCalledOnce(); + + firstClip.end(); + expect(player._videoElement).toBe(secondClip.video); + expect(secondClip.play).toHaveBeenCalledOnce(); + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + secondClip.resolveFinish(); + await streamEnd; + secondClip.end(); + + expect(player._videoElement).toBe(firstClip.video); + expect(firstClip.play).toHaveBeenCalledTimes(2); + expect(player.shadowRoot?.querySelector('.replay-button')?.classList.contains('visible')).toBe(false); + }); + + it('keeps segment playback chronological while preserving pause, seek, and replay intent', async () => { + const { player, socket } = createPlayer(); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + firstClip.resolveOpen(); + await firstStart; + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + secondClip.resolveOpen(); + await secondStart; + + firstClip.setDuration(10); + secondClip.setDuration(20); + secondClip.loaded(); + expect(player._videoElement).toBeNull(); + firstClip.loaded(); + expect(player._videoElement).toBe(firstClip.video); + + player.play(); + expect(firstClip.play).toHaveBeenCalledOnce(); + player.pause(); + firstClip.end(); + expect(player._videoElement).toBe(secondClip.video); + expect(secondClip.play).not.toHaveBeenCalled(); + + player.play(); + expect(secondClip.play).toHaveBeenCalledOnce(); + + const firstTimelineSegment = player.shadowRoot?.querySelector('.timeline-segment'); + expect(firstTimelineSegment).not.toBeNull(); + vi.spyOn(firstTimelineSegment as HTMLElement, 'getBoundingClientRect').mockReturnValue({ + x: 0, + y: 0, + width: 100, + height: 10, + top: 0, + right: 100, + bottom: 10, + left: 0, + toJSON: () => ({}), + }); + firstTimelineSegment?.dispatchEvent(new MouseEvent('click', { clientX: 25 })); + expect(player._videoElement).toBe(firstClip.video); + expect(firstClip.video.currentTime).toBe(2.5); + expect(secondClip.video.currentTime).toBe(0); + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + secondClip.resolveFinish(); + await streamEnd; + + firstClip.end(); + secondClip.end(); + const replayButton = player.shadowRoot?.querySelector('.replay-button'); + expect(replayButton?.classList.contains('visible')).toBe(true); + replayButton?.click(); + expect(player._videoElement).toBe(firstClip.video); + expect(firstClip.video.currentTime).toBe(0); + expect(secondClip.video.currentTime).toBe(0); + }); +}); diff --git a/webapp/packages/shadow-player/src/streamer.ts b/webapp/packages/shadow-player/src/streamer.ts index ac5488dd9..046b366c5 100644 --- a/webapp/packages/shadow-player/src/streamer.ts +++ b/webapp/packages/shadow-player/src/streamer.ts @@ -16,6 +16,12 @@ export type ShadowPlayerError = type: 'websocket'; inner: ErrorEvent; } + | { + type: 'websocket-close'; + code: number; + reason: string; + wasClean: boolean; + } | { type: 'protocol'; inner: ErrorMessage; @@ -30,6 +36,7 @@ export type ShadowPlayerError = }; type ShadowPlayerErrorCallback = (error: ShadowPlayerError) => void; +type TerminalOutcome = 'none' | 'end' | 'error' | 'closed'; export class ShadowPlayer extends HTMLElement { _videoElement: HTMLVideoElement | null = null; @@ -49,6 +56,7 @@ export class ShadowPlayer extends HTMLElement { private awaitingResponse = false; private shouldPlay = false; private streamEnded = false; + private terminalOutcome: TerminalOutcome = 'closed'; private muted = true; private volume = 1; private controls: PlaybackControls | null = null; @@ -93,10 +101,13 @@ export class ShadowPlayer extends HTMLElement { return; } - if (name === 'autoplay' && newValue !== null) { - this.shouldPlay = true; + if (name === 'autoplay') { + if (newValue !== null) { + this.shouldPlay = true; + } + return; } - if (name === 'controls') { + if (name === 'controls' || name === 'loop') { return; } if (name === 'muted') { @@ -220,7 +231,7 @@ export class ShadowPlayer extends HTMLElement { if (this.activateNextClip()) { return; } - if (this.streamEnded && this.activeClip?.video.ended) { + if (this.isSequencePlaybackComplete()) { this.replay(); } } @@ -253,6 +264,7 @@ export class ShadowPlayer extends HTMLElement { return; } + this.terminalOutcome = 'none'; this.streamEnded = false; this._replayButton?.classList.remove('visible'); this.renderPlayerControls(); @@ -273,7 +285,7 @@ export class ShadowPlayer extends HTMLElement { } private async handleServerMessage(websocket: ServerWebSocket, message: ServerMessage): Promise { - if (this.websocket !== websocket) { + if (this.websocket !== websocket || this.terminalOutcome !== 'none') { return; } if (!this.awaitingResponse) { @@ -282,38 +294,40 @@ export class ShadowPlayer extends HTMLElement { this.awaitingResponse = false; if (message.type === 'segment-started') { + await this.startSegment(websocket, message); this.sendRequest(websocket, 'pull'); - await this.startSegment(message); return; } if (message.type === 'chunk') { - if (!this.receivingClip) { + const clip = this.receivingClip; + if (!clip) { throw new Error('Received a chunk before a segment started'); } + await clip.append(message.data); this.sendRequest(websocket, 'pull'); - await this.receivingClip.append(message.data); return; } if (message.type === 'error') { - this.onErrorCallback?.({ type: 'protocol', inner: message }); + this.reportTerminalError({ type: 'protocol', inner: message }); return; } - this.finishReceivingClip(); - this.streamEnded = true; - this.renderPlayerControls(); - if (this.activeClip?.video.ended) { - this.showReplayButton(); + await this.finishReceivingClip(); + if (this.websocket !== websocket || this.terminalOutcome !== 'none') { + return; } - this.onEndCallback?.(); + this.completeStream(); } - private async startSegment(metadata: SegmentStartedMessage): Promise { + private async startSegment(websocket: ServerWebSocket, metadata: SegmentStartedMessage): Promise { if (metadata.sequence !== this.clips.length) { throw new Error(`Expected segment ${this.clips.length}, received ${metadata.sequence}`); } - this.finishReceivingClip(); + await this.finishReceivingClip(); + if (this.websocket !== websocket || this.terminalOutcome !== 'none') { + return; + } const clip = new PlaybackClip(metadata); clip.setDebug(this.debug); this.configureVideo(clip); @@ -324,12 +338,13 @@ export class ShadowPlayer extends HTMLElement { await clip.open(); } - private finishReceivingClip(): void { + private async finishReceivingClip(): Promise { const clip = this.receivingClip; if (!clip) { return; } - clip.finish(); + this.receivingClip = null; + await clip.finish(); this.renderAllSegments(); } @@ -339,7 +354,13 @@ export class ShadowPlayer extends HTMLElement { video.muted = this.muted; video.volume = this.volume; for (const attribute of ShadowPlayer.observedAttributes) { - if (attribute !== 'src' && attribute !== 'controls' && attribute !== 'muted') { + if ( + attribute !== 'src' && + attribute !== 'autoplay' && + attribute !== 'controls' && + attribute !== 'loop' && + attribute !== 'muted' + ) { this.applyVideoAttribute(video, attribute, this.getAttribute(attribute)); } } @@ -368,8 +389,8 @@ export class ShadowPlayer extends HTMLElement { if (this.activeClip !== clip) { return; } - if (!this.activateNextClip() && this.streamEnded) { - this.showReplayButton(); + if (!this.activateNextClip()) { + this.handleSequencePlaybackEnd(); } this.renderClipControls(clip); this.renderPlayerControls(); @@ -508,7 +529,10 @@ export class ShadowPlayer extends HTMLElement { } private sendRequest(websocket: ServerWebSocket, type: 'start' | 'pull'): void { - if (this.websocket !== websocket) { + if (this.websocket !== websocket || this.terminalOutcome !== 'none') { + return; + } + if (!websocket.isOpen()) { return; } if (this.awaitingResponse) { @@ -524,11 +548,20 @@ export class ShadowPlayer extends HTMLElement { } this.awaitingResponse = false; this.websocket = null; - if (event.code === 4001) { - this.onErrorCallback?.({ - type: 'session-not-found', - message: 'Recording session is no longer active', - }); + if (this.terminalOutcome === 'none') { + const error: ShadowPlayerError = + event.code === 4001 + ? { + type: 'session-not-found', + message: 'Recording session is no longer active', + } + : { + type: 'websocket-close', + code: event.code, + reason: event.reason, + wasClean: event.wasClean, + }; + this.reportTerminalError(error); } this.renderPlayerControls(); } @@ -537,7 +570,7 @@ export class ShadowPlayer extends HTMLElement { if (this.websocket !== websocket) { return; } - this.onErrorCallback?.({ + this.reportTerminalError({ type: 'websocket', inner: event as ErrorEvent, }); @@ -548,13 +581,62 @@ export class ShadowPlayer extends HTMLElement { return; } const error = value instanceof Error ? value : new Error(String(value)); + if ( + !this.reportTerminalError({ + type: 'player', + inner: error, + }) + ) { + return; + } this.awaitingResponse = false; - this.onErrorCallback?.({ type: 'player', inner: error }); websocket.close(1000, 'Player failure'); this.websocket = null; this.renderPlayerControls(); } + private completeStream(): void { + if (this.terminalOutcome !== 'none') { + return; + } + this.terminalOutcome = 'end'; + this.streamEnded = true; + this.renderPlayerControls(); + this.handleSequencePlaybackEnd(); + this.onEndCallback?.(); + } + + private reportTerminalError(error: ShadowPlayerError): boolean { + if (this.terminalOutcome !== 'none') { + return false; + } + this.terminalOutcome = 'error'; + this.onErrorCallback?.(error); + this.renderPlayerControls(); + return true; + } + + private handleSequencePlaybackEnd(): void { + if (!this.isSequencePlaybackComplete()) { + return; + } + if (this.hasAttribute('loop') && this.shouldPlay) { + this.replay(); + } else { + this.showReplayButton(); + } + } + + private isSequencePlaybackComplete(): boolean { + const activeClip = this.activeClip; + return ( + this.streamEnded && + activeClip !== null && + activeClip.video.ended && + activeClip.metadata.sequence === this.clips.length - 1 + ); + } + private reportPlayerError(value: unknown): void { const error = value instanceof Error ? value : new Error(String(value)); this.onErrorCallback?.({ type: 'player', inner: error }); @@ -579,6 +661,8 @@ export class ShadowPlayer extends HTMLElement { const websocket = this.websocket; this.websocket = null; this.awaitingResponse = false; + this.terminalOutcome = 'closed'; + this.streamEnded = false; websocket?.close(1000, 'Component cleanup'); for (const clip of this.clips) { clip.dispose(); diff --git a/webapp/packages/shadow-player/src/websocket.test.ts b/webapp/packages/shadow-player/src/websocket.test.ts new file mode 100644 index 000000000..f72372c4d --- /dev/null +++ b/webapp/packages/shadow-player/src/websocket.test.ts @@ -0,0 +1,142 @@ +// @vitest-environment jsdom + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ServerWebSocket } from './websocket'; + +interface Deferred { + promise: Promise; + resolve: (value: T | PromiseLike) => void; +} + +function deferred(): Deferred { + let resolve!: Deferred['resolve']; + const promise = new Promise((promiseResolve) => { + resolve = promiseResolve; + }); + return { promise, resolve }; +} + +function encodedMessage(type: number, payload = ''): ArrayBuffer { + const encodedPayload = new TextEncoder().encode(payload); + const message = new Uint8Array(1 + encodedPayload.length); + message[0] = type; + message.set(encodedPayload, 1); + return message.buffer; +} + +class FakeWebSocket { + static readonly OPEN = 1; + static latest: FakeWebSocket | null = null; + + binaryType: BinaryType = 'blob'; + readyState = FakeWebSocket.OPEN; + onopen: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + onclose: ((event: CloseEvent) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + + constructor(readonly url: string) { + FakeWebSocket.latest = this; + } + + send(): void {} + + close(): void {} + + emitMessage(data: ArrayBuffer): void { + this.onmessage?.(new MessageEvent('message', { data })); + } + + emitClose(): void { + this.onclose?.(new CloseEvent('close', { code: 1006 })); + } + + emitError(): void { + this.onerror?.(new Event('error')); + } +} + +describe('ServerWebSocket', () => { + beforeEach(() => { + vi.stubGlobal('WebSocket', FakeWebSocket); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + FakeWebSocket.latest = null; + }); + + it('serializes messages and dispatches close after pending message work', async () => { + const websocket = new ServerWebSocket('ws://example.test'); + const socket = FakeWebSocket.latest; + expect(socket).not.toBeNull(); + + const firstStarted = deferred(); + const releaseFirst = deferred(); + const secondStarted = deferred(); + const closed = deferred(); + const calls: string[] = []; + + websocket.onmessage(async (message) => { + calls.push(message.type); + if (message.type === 'segment-started') { + firstStarted.resolve(); + await releaseFirst.promise; + } else { + secondStarted.resolve(); + } + }, vi.fn()); + websocket.onclose(() => closed.resolve()); + + socket?.emitMessage(encodedMessage(1, '{"codec":"vp8","sequence":0,"width":640,"height":480}')); + socket?.emitMessage(encodedMessage(0, 'chunk')); + socket?.emitClose(); + + await firstStarted.promise; + await Promise.resolve(); + expect(calls).toEqual(['segment-started']); + + let closeDispatched = false; + void closed.promise.then(() => { + closeDispatched = true; + }); + await Promise.resolve(); + expect(closeDispatched).toBe(false); + + releaseFirst.resolve(); + await secondStarted.promise; + await closed.promise; + expect(calls).toEqual(['segment-started', 'chunk']); + }); + + it('serializes an error after a queued stream end', async () => { + const websocket = new ServerWebSocket('ws://example.test'); + const socket = FakeWebSocket.latest; + expect(socket).not.toBeNull(); + + const endStarted = deferred(); + const releaseEnd = deferred(); + const errorDispatched = deferred(); + + websocket.onmessage(async (message) => { + expect(message).toEqual({ type: 'stream-ended' }); + endStarted.resolve(); + await releaseEnd.promise; + }, vi.fn()); + websocket.onerror(() => errorDispatched.resolve()); + + socket?.emitMessage(encodedMessage(3)); + socket?.emitError(); + + await endStarted.promise; + let errorObserved = false; + void errorDispatched.promise.then(() => { + errorObserved = true; + }); + await Promise.resolve(); + expect(errorObserved).toBe(false); + + releaseEnd.resolve(); + await errorDispatched.promise; + }); +}); diff --git a/webapp/packages/shadow-player/src/websocket.ts b/webapp/packages/shadow-player/src/websocket.ts index 99211c712..91e1d698f 100644 --- a/webapp/packages/shadow-player/src/websocket.ts +++ b/webapp/packages/shadow-player/src/websocket.ts @@ -2,6 +2,8 @@ import { ClientMessage, parseClientMessage, parseServerMessage, ServerMessage } export class ServerWebSocket { private readonly socket: WebSocket; + private pendingEvent = Promise.resolve(); + private closed = false; constructor(url: string) { this.socket = new WebSocket(url); @@ -14,33 +16,47 @@ export class ServerWebSocket { onmessage(callback: (message: ServerMessage) => Promise | void, onFailure: (error: unknown) => void): void { this.socket.onmessage = (event) => { - try { - if (!(event.data instanceof ArrayBuffer)) { - throw new Error('Server sent a non-binary message'); + this.enqueueEvent(async () => { + try { + if (!(event.data instanceof ArrayBuffer)) { + throw new Error('Server sent a non-binary message'); + } + await callback(parseServerMessage(event.data)); + } catch (error) { + onFailure(error); } - Promise.resolve(callback(parseServerMessage(event.data))).catch(onFailure); - } catch (error) { - onFailure(error); - } + }); }; } onclose(callback: (event: CloseEvent) => void): void { - this.socket.onclose = callback; + this.socket.onclose = (event) => { + this.closed = true; + this.enqueueEvent(() => callback(event)); + }; } onerror(callback: (event: Event) => void): void { - this.socket.onerror = callback; + this.socket.onerror = (event) => this.enqueueEvent(() => callback(event)); } send(message: ClientMessage): void { - if (this.socket.readyState !== WebSocket.OPEN) { + if (!this.isOpen()) { throw new Error('WebSocket is not open'); } this.socket.send(parseClientMessage(message)); } + isOpen(): boolean { + return !this.closed && this.socket.readyState === WebSocket.OPEN; + } + close(code: number, reason: string): void { this.socket.close(code, reason); } + + private enqueueEvent(callback: () => Promise | void): void { + const event = this.pendingEvent.then(callback); + this.pendingEvent = event.catch(() => undefined); + } } diff --git a/webapp/packages/shadow-player/vite.config.ts b/webapp/packages/shadow-player/vite.config.ts index 632a03336..10e685ef8 100644 --- a/webapp/packages/shadow-player/vite.config.ts +++ b/webapp/packages/shadow-player/vite.config.ts @@ -1,5 +1,5 @@ import path from 'node:path'; -import { UserConfig, defineConfig } from 'vite'; +import { defineConfig, UserConfig } from 'vite'; import dts from 'vite-plugin-dts'; import { viteStaticCopy } from 'vite-plugin-static-copy'; @@ -51,12 +51,14 @@ const staticCopyPlugin = viteStaticCopy({ const Plugins = { debug: [ dts({ + exclude: ['src/**/*.test.ts'], insertTypesEntry: true, }), staticCopyPlugin, ], release: [ dts({ + exclude: ['src/**/*.test.ts'], insertTypesEntry: true, }), staticCopyPlugin, diff --git a/webapp/pnpm-lock.yaml b/webapp/pnpm-lock.yaml index 295ff1c75..28cd4e092 100644 --- a/webapp/pnpm-lock.yaml +++ b/webapp/pnpm-lock.yaml @@ -268,6 +268,9 @@ importers: packages/shadow-player: devDependencies: + jsdom: + specifier: ^20.0.3 + version: 20.0.3 ts-node: specifier: ^10.9.2 version: 10.9.2(@types/node@22.19.3)(typescript@5.6.3) @@ -283,6 +286,9 @@ importers: vite-plugin-static-copy: specifier: ^2.3.0 version: 2.3.2(vite@5.4.21(@types/node@22.19.3)(less@4.4.0)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.43.1)) + vitest: + specifier: ^3.1.1 + version: 3.2.7(@types/node@20.19.27)(jiti@2.6.1)(jsdom@20.0.3)(less@4.4.0)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.43.1) packages/web-recorder: devDependencies: From c7618b3a61b9b7073cc85961e32f81376f329ee7 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Fri, 21 Aug 2026 16:23:07 -0400 Subject: [PATCH 8/9] feat(webapp): play session-protocol WebM streams The shadow player now understands Segment started, queued Pulls, and Stream ended so live clips can reconnect and change size on one websocket. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- webapp/apps/recording-player/index.css | 9 + .../public/locales/de/translation.json | 11 + .../public/locales/en/translation.json | 11 + .../public/locales/es/translation.json | 11 + .../public/locales/fr/translation.json | 11 + webapp/apps/recording-player/src/i18n.ts | 9 + .../recording-player/src/streamers/webm.ts | 18 +- .../shadow-player/demo-src/apiClient.ts | 4 +- webapp/packages/shadow-player/index.html | 4 +- .../shadow-player/src/playbackClip.ts | 80 ++ .../shadow-player/src/playbackControls.css | 149 ++++ .../shadow-player/src/playbackControls.ts | 306 ++++++++ webapp/packages/shadow-player/src/protocol.ts | 101 ++- .../shadow-player/src/sourceBuffer.ts | 113 +-- .../packages/shadow-player/src/streamer.css | 26 +- webapp/packages/shadow-player/src/streamer.ts | 690 ++++++++++++------ .../packages/shadow-player/src/websocket.ts | 51 +- 17 files changed, 1238 insertions(+), 366 deletions(-) create mode 100644 webapp/packages/shadow-player/src/playbackClip.ts create mode 100644 webapp/packages/shadow-player/src/playbackControls.css create mode 100644 webapp/packages/shadow-player/src/playbackControls.ts diff --git a/webapp/apps/recording-player/index.css b/webapp/apps/recording-player/index.css index b8fbcf612..d00a7055d 100644 --- a/webapp/apps/recording-player/index.css +++ b/webapp/apps/recording-player/index.css @@ -2,9 +2,18 @@ html, body { + width: 100%; + height: 100%; + margin: 0; background-color: black; } +shadow-player { + display: block; + width: 100%; + height: 100%; +} + #terminal { height: 100%; } diff --git a/webapp/apps/recording-player/public/locales/de/translation.json b/webapp/apps/recording-player/public/locales/de/translation.json index bd387d25a..2a6732f4c 100644 --- a/webapp/apps/recording-player/public/locales/de/translation.json +++ b/webapp/apps/recording-player/public/locales/de/translation.json @@ -6,6 +6,17 @@ "unknownError": "Unbekannter Fehler, bitte versuchen Sie es erneut", "protocolError": "Ein Fehler ist aufgetreten: {{error}}" }, + "controls": { + "play": "Wiedergabe", + "pause": "Pause", + "mute": "Stummschalten", + "unmute": "Stummschaltung aufheben", + "volume": "Lautstärke", + "timeline": "Aufzeichnungszeitachse", + "fullscreen": "Vollbild", + "exitFullscreen": "Vollbild beenden", + "clip": "Clip" + }, "ui": { "close": "Schließen" } diff --git a/webapp/apps/recording-player/public/locales/en/translation.json b/webapp/apps/recording-player/public/locales/en/translation.json index 9802ff8a1..33c645ece 100644 --- a/webapp/apps/recording-player/public/locales/en/translation.json +++ b/webapp/apps/recording-player/public/locales/en/translation.json @@ -6,6 +6,17 @@ "unknownError": "Unknown error, please try again", "protocolError": "An error occurred: {{error}}" }, + "controls": { + "play": "Play", + "pause": "Pause", + "mute": "Mute", + "unmute": "Unmute", + "volume": "Volume", + "timeline": "Recording timeline", + "fullscreen": "Fullscreen", + "exitFullscreen": "Exit fullscreen", + "clip": "Clip" + }, "ui": { "close": "Close" } diff --git a/webapp/apps/recording-player/public/locales/es/translation.json b/webapp/apps/recording-player/public/locales/es/translation.json index 9d12b8d06..811714f1e 100644 --- a/webapp/apps/recording-player/public/locales/es/translation.json +++ b/webapp/apps/recording-player/public/locales/es/translation.json @@ -6,6 +6,17 @@ "unknownError": "Error desconocido, por favor intente de nuevo", "protocolError": "Se produjo un error: {{error}}" }, + "controls": { + "play": "Reproducir", + "pause": "Pausar", + "mute": "Silenciar", + "unmute": "Activar sonido", + "volume": "Volumen", + "timeline": "Línea de tiempo de la grabación", + "fullscreen": "Pantalla completa", + "exitFullscreen": "Salir de pantalla completa", + "clip": "Clip" + }, "ui": { "close": "Cerrar" } diff --git a/webapp/apps/recording-player/public/locales/fr/translation.json b/webapp/apps/recording-player/public/locales/fr/translation.json index a6eb01a40..28cabf7d5 100644 --- a/webapp/apps/recording-player/public/locales/fr/translation.json +++ b/webapp/apps/recording-player/public/locales/fr/translation.json @@ -6,6 +6,17 @@ "unknownError": "Erreur inconnue, veuillez réessayer", "protocolError": "Une erreur s'est produite: {{error}}" }, + "controls": { + "play": "Lire", + "pause": "Pause", + "mute": "Couper le son", + "unmute": "Réactiver le son", + "volume": "Volume", + "timeline": "Chronologie de l'enregistrement", + "fullscreen": "Plein écran", + "exitFullscreen": "Quitter le plein écran", + "clip": "Séquence" + }, "ui": { "close": "Fermer" } diff --git a/webapp/apps/recording-player/src/i18n.ts b/webapp/apps/recording-player/src/i18n.ts index c2aac9b65..b1547be30 100644 --- a/webapp/apps/recording-player/src/i18n.ts +++ b/webapp/apps/recording-player/src/i18n.ts @@ -8,6 +8,15 @@ export type TranslationKeys = | 'notifications.unauthorized' | 'notifications.unknownError' | 'notifications.protocolError' + | 'controls.play' + | 'controls.pause' + | 'controls.mute' + | 'controls.unmute' + | 'controls.volume' + | 'controls.timeline' + | 'controls.fullscreen' + | 'controls.exitFullscreen' + | 'controls.clip' | 'ui.close'; /** diff --git a/webapp/apps/recording-player/src/streamers/webm.ts b/webapp/apps/recording-player/src/streamers/webm.ts index 9ddee0a4c..f889b522e 100644 --- a/webapp/apps/recording-player/src/streamers/webm.ts +++ b/webapp/apps/recording-player/src/streamers/webm.ts @@ -5,19 +5,25 @@ import { t } from '../i18n'; import { showNotification } from '../notification'; export async function handleWebm(gatewayAccessApi: GatewayAccessApi) { - // Create element with correct spelling const shadowPlayer = document.createElement('shadow-player') as ShadowPlayer; + shadowPlayer.setAttribute('controls', ''); + shadowPlayer.setControlLabels({ + play: t('controls.play'), + pause: t('controls.pause'), + mute: t('controls.mute'), + unmute: t('controls.unmute'), + volume: t('controls.volume'), + timeline: t('controls.timeline'), + fullscreen: t('controls.fullscreen'), + exitFullscreen: t('controls.exitFullscreen'), + clip: t('controls.clip'), + }); - // Append to DOM document.body.appendChild(shadowPlayer); - // Wait for element to be initialized await customElements.whenDefined('shadow-player'); - - // Wait for next microtask to ensure connectedCallback has run await new Promise((resolve) => setTimeout(resolve, 0)); - // Now safe to call methods shadowPlayer.srcChange(gatewayAccessApi.sessionShadowingUrl()); shadowPlayer.play(); diff --git a/webapp/packages/shadow-player/demo-src/apiClient.ts b/webapp/packages/shadow-player/demo-src/apiClient.ts index 9f3de2d1d..46950ad4f 100644 --- a/webapp/packages/shadow-player/demo-src/apiClient.ts +++ b/webapp/packages/shadow-player/demo-src/apiClient.ts @@ -1,6 +1,6 @@ // Base URL of the API -const TOKEN_SERVER_BASE_URL = 'http://localhost:8080'; -const GATEWAY_BASE_URL = 'http://localhost:7171'; +const TOKEN_SERVER_BASE_URL = import.meta.env.VITE_TOKEN_SERVER_BASE_URL ?? 'http://localhost:8080'; +const GATEWAY_BASE_URL = import.meta.env.VITE_GATEWAY_BASE_URL ?? 'http://localhost:7171'; // Common request fields interface CommonRequest { diff --git a/webapp/packages/shadow-player/index.html b/webapp/packages/shadow-player/index.html index 578f4abe2..541e75c8d 100644 --- a/webapp/packages/shadow-player/index.html +++ b/webapp/packages/shadow-player/index.html @@ -94,7 +94,7 @@ background-color: #e9ecef; } - webm-stream-player { + shadow-player { width: 80%; height: 80%; background-color: #000; @@ -120,7 +120,7 @@

Streaming Files

- +
diff --git a/webapp/packages/shadow-player/src/playbackClip.ts b/webapp/packages/shadow-player/src/playbackClip.ts new file mode 100644 index 000000000..3844bae72 --- /dev/null +++ b/webapp/packages/shadow-player/src/playbackClip.ts @@ -0,0 +1,80 @@ +import type { SegmentStartedMessage } from './protocol'; +import { ReactiveSourceBuffer } from './sourceBuffer'; + +export class PlaybackClip { + readonly video = document.createElement('video'); + + private readonly mediaSource = new MediaSource(); + private readonly objectUrl = URL.createObjectURL(this.mediaSource); + private readonly opened: Promise; + private sourceBuffer: ReactiveSourceBuffer | null = null; + private debug = false; + private complete = false; + + constructor(readonly metadata: SegmentStartedMessage) { + this.video.src = this.objectUrl; + this.opened = new Promise((resolve, reject) => { + const cleanup = () => { + this.mediaSource.removeEventListener('sourceopen', onOpen); + this.mediaSource.removeEventListener('sourceclose', onClose); + }; + const onOpen = () => { + cleanup(); + try { + this.sourceBuffer = new ReactiveSourceBuffer(this.mediaSource, metadata.codec); + this.sourceBuffer.setDebug(this.debug); + resolve(); + } catch (error) { + reject(error); + } + }; + const onClose = () => { + cleanup(); + reject(new Error('MediaSource closed before it opened')); + }; + + this.mediaSource.addEventListener('sourceopen', onOpen); + this.mediaSource.addEventListener('sourceclose', onClose); + }); + } + + async open(): Promise { + await this.opened; + } + + async append(data: Uint8Array): Promise { + await this.opened; + if (this.complete || !this.sourceBuffer) { + throw new Error('Cannot append to a completed clip'); + } + await this.sourceBuffer.appendBuffer(data); + } + + finish(): void { + if (this.complete) { + return; + } + if (this.mediaSource.readyState !== 'open') { + throw new Error('Cannot finish a MediaSource that is not open'); + } + this.mediaSource.endOfStream(); + this.complete = true; + } + + setDebug(debug: boolean): void { + this.debug = debug; + this.sourceBuffer?.setDebug(debug); + } + + downloadBufferedFile(): void { + this.sourceBuffer?.downloadBufferedFile(); + } + + dispose(): void { + this.video.pause(); + this.video.removeAttribute('src'); + this.video.load(); + this.video.remove(); + URL.revokeObjectURL(this.objectUrl); + } +} diff --git a/webapp/packages/shadow-player/src/playbackControls.css b/webapp/packages/shadow-player/src/playbackControls.css new file mode 100644 index 000000000..9c85d1555 --- /dev/null +++ b/webapp/packages/shadow-player/src/playbackControls.css @@ -0,0 +1,149 @@ +.control-bar { + position: absolute; + z-index: 3; + right: 0; + bottom: 0; + left: 0; + display: flex; + height: 30px; + align-items: stretch; + color: #fff; + background: rgba(43, 51, 63, 0.7); + font-family: Arial, Helvetica, sans-serif; +} + +:host(:not([controls])) .control-bar { + display: none; +} + +.control-button { + position: relative; + display: grid; + width: 40px; + min-width: 40px; + height: 30px; + padding: 7px 10px; + place-items: center; + color: inherit; + background: transparent; + border: 0; + cursor: pointer; +} + +.control-button:hover, +.control-button:focus-visible { + color: #fff; + background: rgba(255, 255, 255, 0.12); + outline: none; +} + +.control-button:focus-visible, +.timeline-segment:focus-visible, +.volume-input:focus-visible { + box-shadow: inset 0 0 0 2px #fff; +} + +.control-button svg { + width: 16px; + height: 16px; + fill: currentColor; +} + +.volume-control { + display: flex; + width: 40px; + min-width: 40px; + overflow: hidden; + align-items: center; + transition: width 120ms ease; +} + +.volume-control:hover, +.volume-control:focus-within { + width: 105px; +} + +.volume-input { + width: 0; + height: 3px; + margin: 0; + opacity: 0; + accent-color: #fff; + cursor: pointer; + transition: + width 120ms ease, + opacity 120ms ease; +} + +.volume-control:hover .volume-input, +.volume-control:focus-within .volume-input { + width: 58px; + opacity: 1; +} + +.timeline { + display: flex; + min-width: 4em; + flex: 1; + align-items: center; + touch-action: none; +} + +.timeline-segment { + position: relative; + height: 3px; + min-width: 3px; + margin-left: 3px; + flex-basis: 0; + overflow: visible; + background: rgba(115, 133, 159, 0.5); + cursor: pointer; + transition: height 80ms ease; +} + +.timeline-segment:hover, +.timeline-segment:focus-visible { + height: 10px; + outline: none; +} + +.timeline-segment[aria-disabled="true"] { + cursor: wait; +} + +.timeline-progress { + position: absolute; + inset: 0 auto 0 0; + width: 0; + background: #fff; + pointer-events: none; +} + +.time-tooltip { + position: absolute; + bottom: 15px; + left: 0; + visibility: hidden; + padding: 5px 8px; + color: #fff; + background: rgba(0, 0, 0, 0.8); + border-radius: 2px; + font-size: 12px; + line-height: 1; + pointer-events: none; + transform: translateX(-50%); + white-space: nowrap; +} + +.timeline-segment:hover .time-tooltip, +.timeline-segment:focus-visible .time-tooltip { + visibility: visible; +} + +@media (prefers-reduced-motion: reduce) { + .volume-control, + .volume-input, + .timeline-segment { + transition: none; + } +} diff --git a/webapp/packages/shadow-player/src/playbackControls.ts b/webapp/packages/shadow-player/src/playbackControls.ts new file mode 100644 index 000000000..b116c033e --- /dev/null +++ b/webapp/packages/shadow-player/src/playbackControls.ts @@ -0,0 +1,306 @@ +import styles from './playbackControls.css?inline'; + +export interface PlaybackControlLabels { + play: string; + pause: string; + mute: string; + unmute: string; + volume: string; + timeline: string; + fullscreen: string; + exitFullscreen: string; + clip: string; +} + +export const defaultPlaybackControlLabels: PlaybackControlLabels = { + play: 'Play', + pause: 'Pause', + mute: 'Mute', + unmute: 'Unmute', + volume: 'Volume', + timeline: 'Recording timeline', + fullscreen: 'Fullscreen', + exitFullscreen: 'Exit fullscreen', + clip: 'Clip', +}; + +export type PlaybackControlsAction = + | { type: 'toggle-playback' } + | { type: 'toggle-muted' } + | { type: 'set-volume'; volume: number } + | { type: 'seek'; sequence: number; percentage: number } + | { type: 'toggle-fullscreen' }; + +export type PlaybackControlsSnapshot = + | { + type: 'player'; + playing: boolean; + muted: boolean; + volume: number; + fullscreen: boolean; + } + | { + type: 'segment'; + sequence: number; + startTime: number; + duration: number; + currentTime: number; + progress: number; + playable: boolean; + } + | { type: 'labels'; labels: PlaybackControlLabels } + | { type: 'reset' }; + +const icons = { + play: '', + pause: '', + muted: + '', + volume: + '', + fullscreen: + '', + exitFullscreen: + '', +} as const; + +interface SegmentView { + state: Extract; + track: HTMLDivElement; + fill: HTMLDivElement; + tooltip: HTMLSpanElement; +} + +export class PlaybackControls { + private readonly style: HTMLStyleElement; + private readonly controlBar: HTMLDivElement; + private readonly playButton: HTMLButtonElement; + private readonly muteButton: HTMLButtonElement; + private readonly volumeInput: HTMLInputElement; + private readonly timeline: HTMLDivElement; + private readonly fullscreenButton: HTMLButtonElement; + private readonly segments = new Map(); + private labels = defaultPlaybackControlLabels; + private player = { + playing: false, + muted: true, + volume: 1, + fullscreen: false, + }; + private actionCallback: ((action: PlaybackControlsAction) => void) | null = null; + + constructor(container: HTMLElement) { + this.style = document.createElement('style'); + this.style.textContent = styles; + container.appendChild(this.style); + + this.controlBar = document.createElement('div'); + this.controlBar.className = 'control-bar'; + + this.playButton = this.createControlButton(); + this.playButton.addEventListener('click', () => this.emit({ type: 'toggle-playback' })); + this.controlBar.appendChild(this.playButton); + + const volumeControl = document.createElement('div'); + volumeControl.className = 'volume-control'; + this.muteButton = this.createControlButton(); + this.muteButton.addEventListener('click', () => this.emit({ type: 'toggle-muted' })); + volumeControl.appendChild(this.muteButton); + + this.volumeInput = document.createElement('input'); + this.volumeInput.className = 'volume-input'; + this.volumeInput.type = 'range'; + this.volumeInput.min = '0'; + this.volumeInput.max = '1'; + this.volumeInput.step = '0.05'; + this.volumeInput.addEventListener('input', () => { + this.emit({ type: 'set-volume', volume: Number.parseFloat(this.volumeInput.value) }); + }); + volumeControl.appendChild(this.volumeInput); + this.controlBar.appendChild(volumeControl); + + this.timeline = document.createElement('div'); + this.timeline.className = 'timeline'; + this.timeline.setAttribute('role', 'group'); + this.controlBar.appendChild(this.timeline); + + this.fullscreenButton = this.createControlButton(); + this.fullscreenButton.addEventListener('click', () => this.emit({ type: 'toggle-fullscreen' })); + this.controlBar.appendChild(this.fullscreenButton); + + container.appendChild(this.controlBar); + this.render({ type: 'labels', labels: this.labels }); + this.render({ type: 'player', ...this.player }); + } + + onAction(callback: (action: PlaybackControlsAction) => void): void { + this.actionCallback = callback; + } + + render(snapshot: PlaybackControlsSnapshot): void { + if (snapshot.type === 'player') { + this.renderPlayer(snapshot); + return; + } + if (snapshot.type === 'segment') { + this.renderSegment(snapshot); + return; + } + if (snapshot.type === 'labels') { + this.renderLabels(snapshot.labels); + return; + } + this.segments.clear(); + this.timeline.replaceChildren(); + } + + dispose(): void { + this.actionCallback = null; + this.segments.clear(); + this.controlBar.remove(); + this.style.remove(); + } + + private createControlButton(): HTMLButtonElement { + const button = document.createElement('button'); + button.className = 'control-button'; + button.type = 'button'; + return button; + } + + private renderPlayer(snapshot: Extract): void { + this.player = snapshot; + this.setButton( + this.playButton, + snapshot.playing ? this.labels.pause : this.labels.play, + snapshot.playing ? icons.pause : icons.play, + ); + const silent = snapshot.muted || snapshot.volume === 0; + this.setButton( + this.muteButton, + silent ? this.labels.unmute : this.labels.mute, + silent ? icons.muted : icons.volume, + ); + this.volumeInput.value = String(snapshot.volume); + this.setButton( + this.fullscreenButton, + snapshot.fullscreen ? this.labels.exitFullscreen : this.labels.fullscreen, + snapshot.fullscreen ? icons.exitFullscreen : icons.fullscreen, + ); + } + + private renderLabels(labels: PlaybackControlLabels): void { + this.labels = labels; + this.volumeInput.setAttribute('aria-label', labels.volume); + this.timeline.setAttribute('aria-label', labels.timeline); + this.render({ type: 'player', ...this.player }); + for (const view of this.segments.values()) { + this.renderSegment(view.state); + } + } + + private renderSegment(snapshot: Extract): void { + const view = this.segments.get(snapshot.sequence) ?? this.createSegment(snapshot); + view.state = snapshot; + view.track.style.flexGrow = String(Math.max(1, snapshot.duration)); + view.track.setAttribute('aria-label', `${this.labels.clip} ${snapshot.sequence + 1}`); + view.track.setAttribute('aria-disabled', String(!snapshot.playable)); + view.track.setAttribute('aria-valuenow', String(Math.round(snapshot.progress * 100))); + view.track.setAttribute('aria-valuetext', formatTime(snapshot.startTime + snapshot.currentTime)); + view.fill.style.width = `${snapshot.progress * 100}%`; + } + + private createSegment(snapshot: Extract): SegmentView { + const track = document.createElement('div'); + track.className = 'timeline-segment'; + track.tabIndex = 0; + track.setAttribute('role', 'slider'); + track.setAttribute('aria-valuemin', '0'); + track.setAttribute('aria-valuemax', '100'); + + const fill = document.createElement('div'); + fill.className = 'timeline-progress'; + track.appendChild(fill); + + const tooltip = document.createElement('span'); + tooltip.className = 'time-tooltip'; + track.appendChild(tooltip); + + const view = { state: snapshot, track, fill, tooltip }; + track.addEventListener('click', (event) => this.seekFromPointer(view, event)); + track.addEventListener('pointermove', (event) => this.renderTooltip(view, event)); + track.addEventListener('keydown', (event) => this.seekFromKeyboard(view, event)); + + this.segments.set(snapshot.sequence, view); + this.timeline.appendChild(track); + return view; + } + + private seekFromPointer(view: SegmentView, event: MouseEvent | PointerEvent): void { + if (!view.state.playable) { + return; + } + this.emit({ + type: 'seek', + sequence: view.state.sequence, + percentage: pointerPercentage(view.track, event), + }); + } + + private renderTooltip(view: SegmentView, event: PointerEvent): void { + const percentage = pointerPercentage(view.track, event); + view.tooltip.style.left = `${percentage * 100}%`; + view.tooltip.textContent = formatTime(view.state.startTime + view.state.duration * percentage); + } + + private seekFromKeyboard(view: SegmentView, event: KeyboardEvent): void { + if (!view.state.playable) { + return; + } + let percentage: number | null = null; + if (event.key === 'ArrowLeft' || event.key === 'ArrowDown') { + percentage = view.state.progress - 0.05; + } else if (event.key === 'ArrowRight' || event.key === 'ArrowUp') { + percentage = view.state.progress + 0.05; + } else if (event.key === 'Home') { + percentage = 0; + } else if (event.key === 'End') { + percentage = 1; + } + if (percentage === null) { + return; + } + event.preventDefault(); + this.emit({ + type: 'seek', + sequence: view.state.sequence, + percentage: Math.max(0, Math.min(1, percentage)), + }); + } + + private setButton(button: HTMLButtonElement, label: string, icon: string): void { + button.title = label; + button.setAttribute('aria-label', label); + button.innerHTML = icon; + } + + private emit(action: PlaybackControlsAction): void { + this.actionCallback?.(action); + } +} + +function pointerPercentage(element: HTMLElement, event: MouseEvent | PointerEvent): number { + const bounds = element.getBoundingClientRect(); + return Math.max(0, Math.min(1, (event.clientX - bounds.left) / bounds.width)); +} + +function formatTime(value: number): string { + const seconds = Math.max(0, Math.floor(value)); + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const remainder = seconds % 60; + if (hours > 0) { + return `${hours}:${String(minutes).padStart(2, '0')}:${String(remainder).padStart(2, '0')}`; + } + return `${minutes}:${String(remainder).padStart(2, '0')}`; +} diff --git a/webapp/packages/shadow-player/src/protocol.ts b/webapp/packages/shadow-player/src/protocol.ts index f942f32a3..debe367f3 100644 --- a/webapp/packages/shadow-player/src/protocol.ts +++ b/webapp/packages/shadow-player/src/protocol.ts @@ -1,81 +1,110 @@ -// Define the message types -export type ServerMessage = ChunkMessage | MetaDataMessage | ErrorMessage | EndMessage; +export type ServerMessage = ChunkMessage | SegmentStartedMessage | ErrorMessage | StreamEndedMessage; export interface ChunkMessage { type: 'chunk'; data: Uint8Array; } -export interface ErrorMessage { - type: 'error'; - error: 'UnexpectedError' | 'UnexpectedEOF'; -} - -export interface MetaDataMessage { - type: 'metadata'; +export interface SegmentStartedMessage { + type: 'segment-started'; codec: 'vp8' | 'vp9'; + sequence: number; + width?: number; + height?: number; } -export interface EndMessage { - type: 'end'; +export interface ErrorMessage { + type: 'error'; + error: 'UnexpectedError'; } -export type ClientMessageTypes = 'start' | 'pull'; +export interface StreamEndedMessage { + type: 'stream-ended'; +} export interface ClientMessage { - type: ClientMessageTypes; + type: 'start' | 'pull'; } -// Function to parse the message export function parseServerMessage(buffer: ArrayBuffer): ServerMessage { - const view = new DataView(buffer); - const typeCode = view.getUint8(0); // Read the first byte as the type code + if (buffer.byteLength === 0) { + throw new Error('Empty server message'); + } + const typeCode = new DataView(buffer).getUint8(0); if (typeCode === 0) { - // Chunk message - const chunkData = new Uint8Array(buffer, 1); // The rest is the chunk data return { type: 'chunk', - data: chunkData, + data: new Uint8Array(buffer, 1), }; } + if (typeCode === 1) { - // Metadata message (JSON) - const jsonString = new TextDecoder().decode(new Uint8Array(buffer, 1)); // Decode the rest as a string - const json = JSON.parse(jsonString); + const metadata = parseJsonPayload(buffer); + if (metadata.sequence === undefined && metadata.width === undefined && metadata.height === undefined) { + if (metadata.codec !== 'vp8' && metadata.codec !== 'vp9') { + throw new Error('Unsupported stream codec'); + } + return { + type: 'segment-started', + codec: metadata.codec, + sequence: 0, + }; + } + + if (metadata.codec !== 'vp8') { + throw new Error('Unsupported stream codec'); + } return { - type: 'metadata', - codec: json.codec === 'vp8' ? 'vp8' : 'vp9', + type: 'segment-started', + codec: metadata.codec, + sequence: readInteger(metadata.sequence, 'sequence', 0), + width: readInteger(metadata.width, 'width', 1), + height: readInteger(metadata.height, 'height', 1), }; } if (typeCode === 2) { - // Metadata message (JSON) - const jsonString = new TextDecoder().decode(new Uint8Array(buffer, 1)); // Decode the rest as a string - const json = JSON.parse(jsonString); - + const payload = parseJsonPayload(buffer); + if (payload.error !== 'UnexpectedError') { + throw new Error('Unknown server error'); + } return { type: 'error', - error: json.error, + error: payload.error, }; } if (typeCode === 3) { - return { - type: 'end', - }; + if (buffer.byteLength !== 1) { + throw new Error('Invalid stream-ended message'); + } + return { type: 'stream-ended' }; } - throw new Error('Unknown message type'); + throw new Error('Unknown server message type'); } export function parseClientMessage(message: ClientMessage): Uint8Array { if (message.type === 'start') { return new Uint8Array([0]); } - if (message.type === 'pull') { - return new Uint8Array([1]); + return new Uint8Array([1]); +} + +function parseJsonPayload(buffer: ArrayBuffer): Record { + const text = new TextDecoder('utf-8', { fatal: true }).decode(new Uint8Array(buffer, 1)); + const value: unknown = JSON.parse(text); + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid server message payload'); + } + return value as Record; +} + +function readInteger(value: unknown, field: string, minimum: number): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) { + throw new Error(`Invalid ${field}`); } - throw new Error('Unknown message type'); + return value; } diff --git a/webapp/packages/shadow-player/src/sourceBuffer.ts b/webapp/packages/shadow-player/src/sourceBuffer.ts index 8f115c7bc..3daa39f5a 100644 --- a/webapp/packages/shadow-player/src/sourceBuffer.ts +++ b/webapp/packages/shadow-player/src/sourceBuffer.ts @@ -1,105 +1,58 @@ export class ReactiveSourceBuffer { - sourceBuffer: SourceBuffer; - bufferQueue: Uint8Array[] = []; - isAppending = false; - next = () => {}; - allBuffers: Blob[] = []; // Store all buffers for file creation - debug = false; + private readonly sourceBuffer: SourceBuffer; + private readonly allBuffers: Blob[] = []; + private debug = false; - private readonly onUpdateEnd: () => void; - - constructor( - mediaSource: MediaSource, - codec: string, - next: () => void, - onUpdateEnd?: () => void - ) { + constructor(mediaSource: MediaSource, codec: string) { this.sourceBuffer = mediaSource.addSourceBuffer(`video/webm; codecs="${codec}"`); - this.next = next; - this.onUpdateEnd = onUpdateEnd ?? (() => {}); - - this.sourceBuffer.addEventListener('updateend', () => { - try { - this.onUpdateEnd(); - } finally { - this.tryAppendBuffer(); - } - }); - - // Handle errors and trigger download of the file - this.sourceBuffer.addEventListener('error', (event) => { - this.logErrorDetails(event); - this.downloadBufferedFile(); - }); } - setDebug(debug: boolean) { + setDebug(debug: boolean): void { this.debug = debug; } - appendBuffer(buffer: Uint8Array) { - this.bufferQueue.push(buffer); + async appendBuffer(buffer: Uint8Array): Promise { + if (this.sourceBuffer.updating) { + throw new Error('SourceBuffer is already updating'); + } + if (this.debug) { - this.allBuffers.push(new Blob([buffer], { type: 'video/webm' })); // Save each buffer - console.log( - `[sourceBuffer] appendBuffer: size=${buffer.length} queueLen=${this.bufferQueue.length} bufferedRanges=${this.getBufferedRanges() || '(empty)'}` - ); + this.allBuffers.push(new Blob([buffer], { type: 'video/webm' })); } - this.tryAppendBuffer(); - } - private tryAppendBuffer() { - if (!this.isAppending && !this.sourceBuffer.updating && this.bufferQueue.length > 0) { - this.isAppending = true; + await new Promise((resolve, reject) => { + const cleanup = () => { + this.sourceBuffer.removeEventListener('updateend', onUpdateEnd); + this.sourceBuffer.removeEventListener('error', onError); + }; + const onUpdateEnd = () => { + cleanup(); + resolve(); + }; + const onError = () => { + cleanup(); + reject(new Error('SourceBuffer append failed')); + }; + + this.sourceBuffer.addEventListener('updateend', onUpdateEnd); + this.sourceBuffer.addEventListener('error', onError); try { - const buffer = this.bufferQueue.shift() as Uint8Array; this.sourceBuffer.appendBuffer(buffer); } catch (error) { - this.logErrorDetails(error); - } finally { - this.next(); - this.isAppending = false; + cleanup(); + reject(error); } - } + }); } - public downloadBufferedFile() { - const completeBlob = new Blob(this.allBuffers, { type: 'video/webm' }); - const url = URL.createObjectURL(completeBlob); - - // Create a download link + downloadBufferedFile(): void { + const url = URL.createObjectURL(new Blob(this.allBuffers, { type: 'video/webm' })); const link = document.createElement('a'); link.href = url; link.download = 'buffered-video.webm'; document.body.appendChild(link); link.click(); - - // Cleanup - document.body.removeChild(link); + link.remove(); URL.revokeObjectURL(url); - console.log('Buffered file downloaded.'); - } - - private logErrorDetails(error: unknown) { - console.error('Error encountered in ReactiveSourceBuffer:'); - - // Log the error object with stack trace - console.error('Error object:', error); - - // Log the state of the bufferQueue - console.log('Current bufferQueue length:', this.bufferQueue.length); - - // Log the sourceBuffer state - console.log('SourceBuffer updating:', this.sourceBuffer.updating); - console.log('SourceBuffer buffered ranges:', this.getBufferedRanges()); - } - - private getBufferedRanges(): string { - const ranges = this.sourceBuffer.buffered; - let rangeStr = ''; - for (let i = 0; i < ranges.length; i++) { - rangeStr += `[${ranges.start(i)} - ${ranges.end(i)}] `; - } - return rangeStr.trim(); } } diff --git a/webapp/packages/shadow-player/src/streamer.css b/webapp/packages/shadow-player/src/streamer.css index 1712cc44e..41f50ca9b 100644 --- a/webapp/packages/shadow-player/src/streamer.css +++ b/webapp/packages/shadow-player/src/streamer.css @@ -1,16 +1,32 @@ +:host { + display: block; + background: #000; +} + .container { position: relative; width: 100%; height: 100%; + overflow: hidden; + background: #000; } video { + position: absolute; + inset: 0; + display: none; width: 100%; height: 100%; + object-fit: contain; +} + +video.active { + display: block; } .replay-button { position: absolute; + z-index: 2; top: 50%; left: 50%; transform: translate(-50%, -50%); @@ -22,7 +38,9 @@ video { border-radius: 50%; cursor: pointer; display: none; - transition: transform 0.2s, background-color 0.2s; + transition: + transform 0.2s, + background-color 0.2s; } .replay-button:hover { @@ -39,3 +57,9 @@ video { .replay-button.visible { display: block; } + +@media (prefers-reduced-motion: reduce) { + .replay-button { + transition: none; + } +} diff --git a/webapp/packages/shadow-player/src/streamer.ts b/webapp/packages/shadow-player/src/streamer.ts index 9bfd29da8..ac5488dd9 100644 --- a/webapp/packages/shadow-player/src/streamer.ts +++ b/webapp/packages/shadow-player/src/streamer.ts @@ -1,8 +1,16 @@ -import { ErrorMessage } from './protocol'; -import { ReactiveSourceBuffer } from './sourceBuffer'; +import { PlaybackClip } from './playbackClip'; +import { + defaultPlaybackControlLabels, + type PlaybackControlLabels, + PlaybackControls, + type PlaybackControlsAction, +} from './playbackControls'; +import type { ErrorMessage, SegmentStartedMessage, ServerMessage } from './protocol'; import styles from './streamer.css?inline'; import { ServerWebSocket } from './websocket'; +export type { PlaybackControlLabels } from './playbackControls'; + export type ShadowPlayerError = | { type: 'websocket'; @@ -15,324 +23,574 @@ export type ShadowPlayerError = | { type: 'session-not-found'; message: string; + } + | { + type: 'player'; + inner: Error; }; type ShadowPlayerErrorCallback = (error: ShadowPlayerError) => void; -const LIVE_EDGE_THRESHOLD_SECONDS = 5; -const LIVE_EDGE_SAFETY_MARGIN_SECONDS = 0.25; - export class ShadowPlayer extends HTMLElement { - shadowRoot: ShadowRoot | null = null; _videoElement: HTMLVideoElement | null = null; _src: string | null = null; - _buffer: ReactiveSourceBuffer | null = null; onErrorCallback: ShadowPlayerErrorCallback | null = null; onEndCallback: (() => void) | null = null; debug = false; _container: HTMLDivElement | null = null; _replayButton: HTMLButtonElement | null = null; + private root: ShadowRoot | null = null; private websocket: ServerWebSocket | null = null; - private isDisconnecting = false; - - static get observedAttributes() { - return ['src', 'autoplay', 'loop', 'muted', 'poster', 'preload', 'style', 'width', 'height']; + private readonly clips: PlaybackClip[] = []; + private readonly playableClips = new Set(); + private receivingClip: PlaybackClip | null = null; + private activeClip: PlaybackClip | null = null; + private awaitingResponse = false; + private shouldPlay = false; + private streamEnded = false; + private muted = true; + private volume = 1; + private controls: PlaybackControls | null = null; + private controlLabels = defaultPlaybackControlLabels; + private readonly segmentStartTimes = new Map(); + private readonly onFullscreenChange = () => this.renderPlayerControls(); + + static get observedAttributes(): string[] { + return ['src', 'autoplay', 'controls', 'loop', 'muted', 'poster', 'preload', 'style', 'width', 'height']; } - setDebug(debug: boolean) { + setDebug(debug: boolean): void { this.debug = debug; - if (this._buffer) { - this._buffer.setDebug(debug); + for (const clip of this.clips) { + clip.setDebug(debug); } } - onError(callback: ShadowPlayerErrorCallback) { + onError(callback: ShadowPlayerErrorCallback): void { this.onErrorCallback = callback; } - onEnd(callback: () => void) { - if (this._videoElement) { - this._videoElement.controls = true; - } + onEnd(callback: () => void): void { this.onEndCallback = callback; } - attributeChangedCallback(name: string, _oldValue: string, newValue: string) { + setControlLabels(labels: Partial): void { + this.controlLabels = { ...this.controlLabels, ...labels }; + this.controls?.render({ type: 'labels', labels: this.controlLabels }); + } + + attributeChangedCallback(name: string, _oldValue: string | null, newValue: string | null): void { if (name === 'src') { - this.srcChange(newValue); + if (newValue === null) { + this.disconnect(); + this._src = null; + } else if (this._container) { + this.srcChange(newValue); + } else { + this._src = newValue; + } return; } - if (this._videoElement && Object.prototype.hasOwnProperty.call(this._videoElement, name)) { - this._videoElement.setAttribute(name, newValue !== null ? newValue : ''); + if (name === 'autoplay' && newValue !== null) { + this.shouldPlay = true; + } + if (name === 'controls') { + return; + } + if (name === 'muted') { + this.setMuted(newValue !== null); + return; + } + for (const clip of this.clips) { + this.applyVideoAttribute(clip.video, name, newValue); } } - connectedCallback() { + connectedCallback(): void { this.init(); + document.addEventListener('fullscreenchange', this.onFullscreenChange); + const src = this.getAttribute('src'); + if (src !== null && !this.websocket) { + this.srcChange(src); + } } - init() { - this.shadowRoot = this.attachShadow({ mode: 'open' }); - - // Add styles - const style = document.createElement('style'); - style.textContent = styles; - this.shadowRoot.appendChild(style); + disconnectedCallback(): void { + document.removeEventListener('fullscreenchange', this.onFullscreenChange); + this.disconnect(); + this.controls?.dispose(); + this.controls = null; + } - this._container = document.createElement('div'); - this._container.className = 'container'; + init(): void { + if (!this.root) { + this.root = this.attachShadow({ mode: 'open' }); + const style = document.createElement('style'); + style.textContent = styles; + this.root.appendChild(style); - this.videoElement = document.createElement('video'); - // Set muted to true so that the browser security policy will allow autoplay. - this.videoElement.muted = true; - this._container.appendChild(this.videoElement); + this._container = document.createElement('div'); + this._container.className = 'container'; - this._replayButton = document.createElement('button'); - this._replayButton.className = 'replay-button'; - this._replayButton.innerHTML = ` + this._replayButton = document.createElement('button'); + this._replayButton.className = 'replay-button'; + this._replayButton.innerHTML = ` `; - this._replayButton.onclick = () => this.replay(); - this._container.appendChild(this._replayButton); + this._replayButton.onclick = () => this.replay(); + this._container.appendChild(this._replayButton); + this.root.appendChild(this._container); + } - this.shadowRoot.appendChild(this._container); - this.syncAttributes(); + if (!this.controls && this._container) { + this.controls = new PlaybackControls(this._container); + this.controls.onAction((action) => this.handleControlsAction(action)); + this.controls.render({ type: 'labels', labels: this.controlLabels }); + } + this.shouldPlay = this.hasAttribute('autoplay'); + this.renderPlayerControls(); } - syncAttributes() { - for (const attr of ShadowPlayer.observedAttributes) { - const value = this.getAttribute(attr); - if (attr === 'src' && value !== null) { - this.srcChange(value); + private handleControlsAction(action: PlaybackControlsAction): void { + if (action.type === 'toggle-playback') { + if (this.shouldPlay) { + this.pause(); + } else { + this.play(); } - if (value !== null && this._videoElement) { - this._videoElement.setAttribute(attr, value); + return; + } + if (action.type === 'toggle-muted') { + if (this.volume === 0) { + this.setVolume(1); } + this.setMuted(!this.muted); + return; } + if (action.type === 'set-volume') { + this.setVolume(action.volume); + this.setMuted(this.volume === 0); + return; + } + if (action.type === 'seek') { + const clip = this.clips[action.sequence]; + if (clip?.metadata.sequence === action.sequence) { + this.seekToClip(clip, action.percentage); + } + return; + } + void this.toggleFullscreen().catch((error: unknown) => this.reportPlayerError(error)); } - private get videoElement() { - return this._videoElement as HTMLVideoElement; + private setMuted(muted: boolean): void { + this.muted = muted; + for (const clip of this.clips) { + clip.video.muted = muted; + } + this.renderPlayerControls(); } - private set videoElement(value: HTMLVideoElement) { - this._videoElement = value; + private setVolume(volume: number): void { + this.volume = Math.max(0, Math.min(1, volume)); + for (const clip of this.clips) { + clip.video.volume = this.volume; + } + this.renderPlayerControls(); } - public play() { - if (this._videoElement) { - this._videoElement.play(); + private async toggleFullscreen(): Promise { + if (document.fullscreenElement === this) { + await document.exitFullscreen(); + } else { + await this.requestFullscreen(); } } - private replay() { - if (this._replayButton) { - this._replayButton.classList.remove('visible'); + public play(): void { + this.shouldPlay = true; + this.renderPlayerControls(); + if (this.activeClip && !this.activeClip.video.ended) { + void this.activeClip.video.play(); + return; } - this._videoElement?.play(); + if (this.activateNextClip()) { + return; + } + if (this.streamEnded && this.activeClip?.video.ended) { + this.replay(); + } + } + + public pause(): void { + this.shouldPlay = false; + this.activeClip?.video.pause(); + this.renderPlayerControls(); } - public srcChange(value: string) { - if (!this._videoElement) { + private replay(): void { + this._replayButton?.classList.remove('visible'); + const firstClip = this.clips[0]; + if (!firstClip) { return; } - this.isDisconnecting = false; - const mediaSource = new MediaSource(); - this._src = value; - this._videoElement.src = URL.createObjectURL(mediaSource); - mediaSource.addEventListener('sourceopen', () => { - this.handleSourceOpen(mediaSource); - }); + for (const clip of this.clips) { + clip.video.currentTime = 0; + } + this.shouldPlay = true; + this.activateClip(firstClip); + this.renderAllSegments(); + this.renderPlayerControls(); } - private async handleSourceOpen(mediaSource: MediaSource) { - this.websocket = new ServerWebSocket(this._src as string); - let reactiveSourceBuffer: ReactiveSourceBuffer | null = null; + public srcChange(value: string): void { + this.closeSession(); + this._src = value; + if (!this._container) { + return; + } - this.websocket.onopen(() => { - this.websocket!.send({ type: 'start' }); - this.websocket!.send({ type: 'pull' }); + this.streamEnded = false; + this._replayButton?.classList.remove('visible'); + this.renderPlayerControls(); + const websocket = new ServerWebSocket(value); + this.websocket = websocket; - this._videoElement?.addEventListener('ended', () => { - this.showReplayButton(); - }); + websocket.onopen(() => { + if (this.websocket === websocket) { + this.sendRequest(websocket, 'start'); + } }); + websocket.onmessage( + async (message) => this.handleServerMessage(websocket, message), + (error) => this.handlePlayerFailure(websocket, error), + ); + websocket.onclose((event) => this.handleSocketClose(websocket, event)); + websocket.onerror((event) => this.handleSocketError(websocket, event)); + } - this.websocket.onmessage((ev) => { - if (mediaSource.readyState === 'closed') { - return; - } - if (ev.type === 'metadata') { - const codec = ev.codec; - reactiveSourceBuffer = new ReactiveSourceBuffer( - mediaSource, - codec, - () => { - this.websocket?.send({ type: 'pull' }); - }, - () => this.catchUpToLiveEdge() - ); - this._buffer = reactiveSourceBuffer; - } + private async handleServerMessage(websocket: ServerWebSocket, message: ServerMessage): Promise { + if (this.websocket !== websocket) { + return; + } + if (!this.awaitingResponse) { + throw new Error('Received a server message without a pending request'); + } + this.awaitingResponse = false; - if (ev.type === 'chunk') { - if (!reactiveSourceBuffer) { - return; - } - - reactiveSourceBuffer.appendBuffer(ev.data); - - if (!this._videoElement) { - return; - } - - if (this.debug) { - const v = this._videoElement; - const buffered = v.buffered.length > 0 - ? `[${v.buffered.start(0).toFixed(2)}-${v.buffered.end(0).toFixed(2)}]` - : '(empty)'; - console.log( - `[shadow-player] chunk appended: duration=${v.duration.toFixed(2)} currentTime=${v.currentTime.toFixed(2)} buffered=${buffered} readyState=${v.readyState}` - ); - } + if (message.type === 'segment-started') { + this.sendRequest(websocket, 'pull'); + await this.startSegment(message); + return; + } + if (message.type === 'chunk') { + if (!this.receivingClip) { + throw new Error('Received a chunk before a segment started'); } + this.sendRequest(websocket, 'pull'); + await this.receivingClip.append(message.data); + return; + } + if (message.type === 'error') { + this.onErrorCallback?.({ type: 'protocol', inner: message }); + return; + } - if (ev.type === 'error') { - this.onErrorCallback?.({ - type: 'protocol', - inner: ev, - }); - } + this.finishReceivingClip(); + this.streamEnded = true; + this.renderPlayerControls(); + if (this.activeClip?.video.ended) { + this.showReplayButton(); + } + this.onEndCallback?.(); + } + + private async startSegment(metadata: SegmentStartedMessage): Promise { + if (metadata.sequence !== this.clips.length) { + throw new Error(`Expected segment ${this.clips.length}, received ${metadata.sequence}`); + } + + this.finishReceivingClip(); + const clip = new PlaybackClip(metadata); + clip.setDebug(this.debug); + this.configureVideo(clip); + this.clips.push(clip); + this.receivingClip = clip; + this._container?.insertBefore(clip.video, this._replayButton); + this.renderAllSegments(); + await clip.open(); + } + + private finishReceivingClip(): void { + const clip = this.receivingClip; + if (!clip) { + return; + } + clip.finish(); + this.renderAllSegments(); + } - if (ev.type === 'end') { - this.onEndCallback?.(); + private configureVideo(clip: PlaybackClip): void { + const video = clip.video; + video.className = 'clip'; + video.muted = this.muted; + video.volume = this.volume; + for (const attribute of ShadowPlayer.observedAttributes) { + if (attribute !== 'src' && attribute !== 'controls' && attribute !== 'muted') { + this.applyVideoAttribute(video, attribute, this.getAttribute(attribute)); + } + } + video.addEventListener( + 'loadeddata', + () => { + this.playableClips.add(clip); + this.activateNextClip(); + this.renderAllSegments(); + }, + { once: true }, + ); + video.addEventListener('play', () => { + if (this.activeClip === clip) { + this.shouldPlay = true; + this.renderPlayerControls(); } }); - - this.websocket.onclose((ev) => { - if (this.isDisconnecting) { - this.websocket = null; - return; + video.addEventListener('pause', () => { + if (this.activeClip === clip && !video.ended) { + this.shouldPlay = false; + this.renderPlayerControls(); } - - if (ev.code === 4001) { - this.onErrorCallback?.({ - type: 'session-not-found', - message: 'Recording session is no longer active', - }); + }); + video.addEventListener('ended', () => { + if (this.activeClip !== clip) { + return; } - - this.videoElement.controls = true; - if (reactiveSourceBuffer && mediaSource.readyState === 'open') { - try { - if (this.debug && this._videoElement) { - const v = this._videoElement; - const buffered = v.buffered.length > 0 - ? `[${v.buffered.start(0).toFixed(2)}-${v.buffered.end(0).toFixed(2)}]` - : '(empty)'; - console.log( - `[shadow-player] BEFORE endOfStream: duration=${v.duration} currentTime=${v.currentTime.toFixed(2)} buffered=${buffered} mediaSource.readyState=${mediaSource.readyState}` - ); - } - mediaSource.endOfStream(); - if (this.debug && this._videoElement) { - const v = this._videoElement; - const buffered = v.buffered.length > 0 - ? `[${v.buffered.start(0).toFixed(2)}-${v.buffered.end(0).toFixed(2)}]` - : '(empty)'; - console.log( - `[shadow-player] AFTER endOfStream: duration=${v.duration} currentTime=${v.currentTime.toFixed(2)} buffered=${buffered} mediaSource.readyState=${mediaSource.readyState}` - ); - } - } catch (error) { - if (this.debug) { - console.error('[shadow-player] endOfStream error:', error); - } - } + if (!this.activateNextClip() && this.streamEnded) { + this.showReplayButton(); } - this.websocket = null; + this.renderClipControls(clip); + this.renderPlayerControls(); }); + video.addEventListener('timeupdate', () => this.renderClipControls(clip)); + video.addEventListener('durationchange', () => this.renderAllSegments()); + video.addEventListener('progress', () => this.renderClipControls(clip)); + video.addEventListener('click', () => this.handleControlsAction({ type: 'toggle-playback' })); + } - this.websocket.onerror((ev) => { - if (this.isDisconnecting) { - return; + private activateNextClip(): boolean { + const sequence = this.activeClip ? this.activeClip.metadata.sequence + 1 : 0; + const next = this.clips[sequence]; + if (!next || !this.playableClips.has(next)) { + return false; + } + if (this.activeClip && !this.activeClip.video.ended) { + return false; + } + this.activateClip(next); + return true; + } + + private activateClip(clip: PlaybackClip): void { + if (this.activeClip === clip) { + if (this.shouldPlay) { + void clip.video.play(); } + this.renderClipControls(clip); + this.renderPlayerControls(); + return; + } + const previous = this.activeClip; + this.activeClip = clip; + if (previous) { + previous.video.pause(); + previous.video.classList.remove('active'); + } + this._videoElement = clip.video; + clip.video.classList.add('active'); + if (this.shouldPlay) { + void clip.video.play(); + } + if (previous) { + this.renderClipControls(previous); + } + this.renderClipControls(clip); + this.renderPlayerControls(); + } - this.onErrorCallback?.({ - type: 'websocket', - inner: ev as unknown as ErrorEvent, - }); + private seekToClip(clip: PlaybackClip, percentage: number): void { + if (!this.playableClips.has(clip)) { + return; + } + const duration = this.clipDuration(clip); + if (duration <= 0) { + return; + } - if (reactiveSourceBuffer && mediaSource.readyState === 'open') { - try { - mediaSource.endOfStream(); - } catch (error) { - console.error('endOfStream error:', error); - } + for (const laterClip of this.clips) { + if (laterClip.metadata.sequence > clip.metadata.sequence && this.playableClips.has(laterClip)) { + laterClip.video.currentTime = 0; } + } + clip.video.currentTime = duration * Math.max(0, Math.min(1, percentage)); + this._replayButton?.classList.remove('visible'); + this.activateClip(clip); + this.renderAllSegments(); + } + + private clipDuration(clip: PlaybackClip): number { + if (Number.isFinite(clip.video.duration) && clip.video.duration > 0) { + return clip.video.duration; + } + const buffered = clip.video.buffered; + return buffered.length > 0 ? buffered.end(buffered.length - 1) : 0; + } + + private clipProgress(clip: PlaybackClip): number { + if (!this.activeClip) { + return 0; + } + if (clip.metadata.sequence < this.activeClip.metadata.sequence) { + return 1; + } + if (clip !== this.activeClip) { + return 0; + } + const duration = this.clipDuration(clip); + return duration > 0 ? Math.max(0, Math.min(1, clip.video.currentTime / duration)) : 0; + } + + private renderPlayerControls(): void { + this.controls?.render({ + type: 'player', + playing: this.shouldPlay, + muted: this.muted, + volume: this.volume, + fullscreen: document.fullscreenElement === this, + }); + } + + private renderClipControls(clip: PlaybackClip): void { + const startTime = this.segmentStartTimes.get(clip); + if (startTime === undefined) { + return; + } + const duration = this.clipDuration(clip); + const progress = this.clipProgress(clip); + this.controls?.render({ + type: 'segment', + sequence: clip.metadata.sequence, + startTime, + duration, + currentTime: duration * progress, + progress, + playable: this.playableClips.has(clip), }); } - public downloadBUfferAsFile() { - if (this._buffer && this.debug) { - this._buffer.downloadBufferedFile(); + private renderAllSegments(): void { + let startTime = 0; + for (const clip of this.clips) { + this.segmentStartTimes.set(clip, startTime); + this.renderClipControls(clip); + startTime += this.clipDuration(clip); } } - private showReplayButton() { - if (this._replayButton) { - this._replayButton.classList.add('visible'); + private applyVideoAttribute(video: HTMLVideoElement, name: string, value: string | null): void { + if (value === null) { + video.removeAttribute(name); + } else { + video.setAttribute(name, value); } } - private catchUpToLiveEdge() { - const video = this._videoElement; - if (!video || video.buffered.length === 0) { + private sendRequest(websocket: ServerWebSocket, type: 'start' | 'pull'): void { + if (this.websocket !== websocket) { return; } + if (this.awaitingResponse) { + throw new Error('A stream request is already pending'); + } + this.awaitingResponse = true; + websocket.send({ type }); + } - const latestRangeIndex = video.buffered.length - 1; - const latestRangeStart = video.buffered.start(latestRangeIndex); - const latestRangeEnd = video.buffered.end(latestRangeIndex); - const isOutsideLatestRange = - video.currentTime < latestRangeStart || video.currentTime >= latestRangeEnd; + private handleSocketClose(websocket: ServerWebSocket, event: CloseEvent): void { + if (this.websocket !== websocket) { + return; + } + this.awaitingResponse = false; + this.websocket = null; + if (event.code === 4001) { + this.onErrorCallback?.({ + type: 'session-not-found', + message: 'Recording session is no longer active', + }); + } + this.renderPlayerControls(); + } - if ( - isOutsideLatestRange || - latestRangeEnd - video.currentTime > LIVE_EDGE_THRESHOLD_SECONDS - ) { - video.currentTime = Math.max( - latestRangeStart, - latestRangeEnd - LIVE_EDGE_SAFETY_MARGIN_SECONDS - ); + private handleSocketError(websocket: ServerWebSocket, event: Event): void { + if (this.websocket !== websocket) { + return; } + this.onErrorCallback?.({ + type: 'websocket', + inner: event as ErrorEvent, + }); } - public disconnect(): void { - this.isDisconnecting = true; + private handlePlayerFailure(websocket: ServerWebSocket, value: unknown): void { + if (this.websocket !== websocket) { + return; + } + const error = value instanceof Error ? value : new Error(String(value)); + this.awaitingResponse = false; + this.onErrorCallback?.({ type: 'player', inner: error }); + websocket.close(1000, 'Player failure'); + this.websocket = null; + this.renderPlayerControls(); + } - if (this.websocket) { - try { - this.websocket.ws.close(1000, 'Component cleanup'); - } catch (error) { - // Intentionally ignored: WebSocket may already be closed - } - this.websocket = null; + private reportPlayerError(value: unknown): void { + const error = value instanceof Error ? value : new Error(String(value)); + this.onErrorCallback?.({ type: 'player', inner: error }); + } + + public downloadBUfferAsFile(): void { + if (this.debug) { + (this.receivingClip ?? this.activeClip)?.downloadBufferedFile(); } + } - if (this._videoElement) { - try { - this._videoElement.pause(); - this._videoElement.src = ''; - this._videoElement.load(); - } catch (error) { - // Intentionally ignored: Video element may already be in an invalid state - } + private showReplayButton(): void { + this._replayButton?.classList.add('visible'); + this.renderPlayerControls(); + } + + public disconnect(): void { + this.closeSession(); + } + + private closeSession(): void { + const websocket = this.websocket; + this.websocket = null; + this.awaitingResponse = false; + websocket?.close(1000, 'Component cleanup'); + for (const clip of this.clips) { + clip.dispose(); } + this.clips.length = 0; + this.playableClips.clear(); + this.segmentStartTimes.clear(); + this.receivingClip = null; + this.activeClip = null; + this._videoElement = null; + this.controls?.render({ type: 'reset' }); + this.renderPlayerControls(); } } diff --git a/webapp/packages/shadow-player/src/websocket.ts b/webapp/packages/shadow-player/src/websocket.ts index 0d690b26c..99211c712 100644 --- a/webapp/packages/shadow-player/src/websocket.ts +++ b/webapp/packages/shadow-player/src/websocket.ts @@ -1,41 +1,46 @@ -import { ClientMessage, ServerMessage, parseClientMessage, parseServerMessage } from './protocol'; +import { ClientMessage, parseClientMessage, parseServerMessage, ServerMessage } from './protocol'; export class ServerWebSocket { - ws: WebSocket; + private readonly socket: WebSocket; + constructor(url: string) { - this.ws = new WebSocket(url); + this.socket = new WebSocket(url); + this.socket.binaryType = 'arraybuffer'; } - onopen(callback: (ev: Event) => unknown) { - this.ws.onopen = callback; + onopen(callback: (event: Event) => void): void { + this.socket.onopen = callback; } - onmessage(callback: (ev: ServerMessage) => unknown) { - this.ws.onmessage = (ev) => { - const reader = new FileReader(); - reader.onload = () => { - const arrayBuffer = reader.result as ArrayBuffer; - const serverResponse = parseServerMessage(arrayBuffer); - callback(serverResponse); - }; - - reader.readAsArrayBuffer(ev.data); + onmessage(callback: (message: ServerMessage) => Promise | void, onFailure: (error: unknown) => void): void { + this.socket.onmessage = (event) => { + try { + if (!(event.data instanceof ArrayBuffer)) { + throw new Error('Server sent a non-binary message'); + } + Promise.resolve(callback(parseServerMessage(event.data))).catch(onFailure); + } catch (error) { + onFailure(error); + } }; } - onclose(callback: (ev: CloseEvent) => unknown) { - this.ws.onclose = callback; + onclose(callback: (event: CloseEvent) => void): void { + this.socket.onclose = callback; } - onerror(callback: (ev: Event) => unknown) { - this.ws.onerror = callback; + onerror(callback: (event: Event) => void): void { + this.socket.onerror = callback; } - send(data: T) { - this.ws.send(parseClientMessage(data)); + send(message: ClientMessage): void { + if (this.socket.readyState !== WebSocket.OPEN) { + throw new Error('WebSocket is not open'); + } + this.socket.send(parseClientMessage(message)); } - isClosed() { - return this.ws.readyState === WebSocket.CLOSED; + close(code: number, reason: string): void { + this.socket.close(code, reason); } } From 2521ffabf9207f67916df5279d7c8e4917b2e3e7 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Mon, 24 Aug 2026 20:58:01 -0400 Subject: [PATCH 9/9] fix(webapp): play every live stream segment Queue metadata-delimited WebM clips as independent playback sources. Serialize SourceBuffer completion and terminal events so delayed segments cannot be skipped. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- webapp/packages/shadow-player/package.json | 7 +- .../shadow-player/src/playbackClip.test.ts | 93 ++++ .../shadow-player/src/playbackClip.ts | 23 +- .../shadow-player/src/protocol.test.ts | 39 ++ .../shadow-player/src/sourceBuffer.test.ts | 53 ++ .../shadow-player/src/sourceBuffer.ts | 13 +- .../shadow-player/src/streamer.test.ts | 512 ++++++++++++++++++ webapp/packages/shadow-player/src/streamer.ts | 144 ++++- .../shadow-player/src/websocket.test.ts | 142 +++++ .../packages/shadow-player/src/websocket.ts | 36 +- webapp/packages/shadow-player/vite.config.ts | 4 +- webapp/pnpm-lock.yaml | 6 + 12 files changed, 1022 insertions(+), 50 deletions(-) create mode 100644 webapp/packages/shadow-player/src/playbackClip.test.ts create mode 100644 webapp/packages/shadow-player/src/protocol.test.ts create mode 100644 webapp/packages/shadow-player/src/sourceBuffer.test.ts create mode 100644 webapp/packages/shadow-player/src/streamer.test.ts create mode 100644 webapp/packages/shadow-player/src/websocket.test.ts diff --git a/webapp/packages/shadow-player/package.json b/webapp/packages/shadow-player/package.json index fddd1d8a6..d8ee79f78 100644 --- a/webapp/packages/shadow-player/package.json +++ b/webapp/packages/shadow-player/package.json @@ -8,13 +8,16 @@ "scripts": { "dev": "vite", "build": "tsc && vite build", - "preview": "vite preview" + "preview": "vite preview", + "test": "vitest run" }, "devDependencies": { + "jsdom": "^20.0.3", "ts-node": "^10.9.2", "typescript": "~5.6.2", "vite": "^5.4.9", "vite-plugin-dts": "^4.3.0", - "vite-plugin-static-copy": "^2.3.0" + "vite-plugin-static-copy": "^2.3.0", + "vitest": "^3.1.1" } } diff --git a/webapp/packages/shadow-player/src/playbackClip.test.ts b/webapp/packages/shadow-player/src/playbackClip.test.ts new file mode 100644 index 000000000..d636d9893 --- /dev/null +++ b/webapp/packages/shadow-player/src/playbackClip.test.ts @@ -0,0 +1,93 @@ +// @vitest-environment jsdom + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { PlaybackClip } from './playbackClip'; + +class FakeSourceBuffer extends EventTarget { + updating = false; + + appendBuffer(): void { + if (this.updating) { + throw new Error('concurrent append'); + } + this.updating = true; + } + + completeAppend(): void { + this.updating = false; + this.dispatchEvent(new Event('updateend')); + } +} + +class FakeMediaSource extends EventTarget { + static latest: FakeMediaSource | null = null; + + readyState: ReadyState = 'closed'; + readonly sourceBuffer = new FakeSourceBuffer(); + endOfStreamCalls = 0; + + constructor() { + super(); + FakeMediaSource.latest = this; + } + + addSourceBuffer(): SourceBuffer { + return this.sourceBuffer as unknown as SourceBuffer; + } + + open(): void { + this.readyState = 'open'; + this.dispatchEvent(new Event('sourceopen')); + } + + endOfStream(): void { + if (this.sourceBuffer.updating) { + throw new Error('endOfStream during append'); + } + this.endOfStreamCalls += 1; + this.readyState = 'ended'; + } +} + +describe('PlaybackClip', () => { + beforeEach(() => { + vi.stubGlobal('MediaSource', FakeMediaSource); + vi.stubGlobal('URL', { + createObjectURL: vi.fn(() => 'blob:test'), + revokeObjectURL: vi.fn(), + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + FakeMediaSource.latest = null; + }); + + it('waits for pending SourceBuffer work before ending the MediaSource', async () => { + const clip = new PlaybackClip({ + type: 'segment-started', + codec: 'vp8', + sequence: 0, + width: 640, + height: 480, + }); + const mediaSource = FakeMediaSource.latest; + expect(mediaSource).not.toBeNull(); + mediaSource?.open(); + await clip.open(); + + const append = clip.append(new Uint8Array([1])); + const finish = Promise.resolve().then(() => clip.finish()); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + expect(mediaSource?.sourceBuffer.updating).toBe(true); + expect(mediaSource?.endOfStreamCalls).toBe(0); + + mediaSource?.sourceBuffer.completeAppend(); + await append; + await finish; + expect(mediaSource?.endOfStreamCalls).toBe(1); + }); +}); diff --git a/webapp/packages/shadow-player/src/playbackClip.ts b/webapp/packages/shadow-player/src/playbackClip.ts index 3844bae72..5b08b5328 100644 --- a/webapp/packages/shadow-player/src/playbackClip.ts +++ b/webapp/packages/shadow-player/src/playbackClip.ts @@ -10,6 +10,7 @@ export class PlaybackClip { private sourceBuffer: ReactiveSourceBuffer | null = null; private debug = false; private complete = false; + private finishing: Promise | null = null; constructor(readonly metadata: SegmentStartedMessage) { this.video.src = this.objectUrl; @@ -50,15 +51,25 @@ export class PlaybackClip { await this.sourceBuffer.appendBuffer(data); } - finish(): void { - if (this.complete) { - return; + async finish(): Promise { + await this.opened; + if (this.finishing) { + return this.finishing; } - if (this.mediaSource.readyState !== 'open') { - throw new Error('Cannot finish a MediaSource that is not open'); + const sourceBuffer = this.sourceBuffer; + if (this.complete || !sourceBuffer) { + return; } - this.mediaSource.endOfStream(); + this.complete = true; + this.finishing = (async () => { + await sourceBuffer.whenIdle(); + if (this.mediaSource.readyState !== 'open') { + throw new Error('Cannot finish a MediaSource that is not open'); + } + this.mediaSource.endOfStream(); + })(); + return this.finishing; } setDebug(debug: boolean): void { diff --git a/webapp/packages/shadow-player/src/protocol.test.ts b/webapp/packages/shadow-player/src/protocol.test.ts new file mode 100644 index 000000000..92fba7065 --- /dev/null +++ b/webapp/packages/shadow-player/src/protocol.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { parseServerMessage } from './protocol'; + +function encodedMessage(type: number, payload = ''): ArrayBuffer { + const encodedPayload = new TextEncoder().encode(payload); + const message = new Uint8Array(1 + encodedPayload.length); + message[0] = type; + message.set(encodedPayload, 1); + return message.buffer; +} + +describe('parseServerMessage', () => { + it('accepts legacy one-segment metadata', () => { + expect(parseServerMessage(encodedMessage(1, '{"codec":"vp9"}'))).toEqual({ + type: 'segment-started', + codec: 'vp9', + sequence: 0, + }); + }); + + it('parses independent segment metadata', () => { + expect(parseServerMessage(encodedMessage(1, '{"codec":"vp8","sequence":2,"width":1280,"height":720}'))).toEqual({ + type: 'segment-started', + codec: 'vp8', + sequence: 2, + width: 1280, + height: 720, + }); + }); + + it('requires stream-ended to have no payload', () => { + expect(parseServerMessage(encodedMessage(3))).toEqual({ type: 'stream-ended' }); + expect(() => parseServerMessage(encodedMessage(3, 'unexpected'))).toThrow('Invalid stream-ended message'); + }); + + it('rejects partially extended metadata', () => { + expect(() => parseServerMessage(encodedMessage(1, '{"codec":"vp8","sequence":0}'))).toThrow('Invalid width'); + }); +}); diff --git a/webapp/packages/shadow-player/src/sourceBuffer.test.ts b/webapp/packages/shadow-player/src/sourceBuffer.test.ts new file mode 100644 index 000000000..a98cd4c04 --- /dev/null +++ b/webapp/packages/shadow-player/src/sourceBuffer.test.ts @@ -0,0 +1,53 @@ +// @vitest-environment jsdom + +import { describe, expect, it } from 'vitest'; +import { ReactiveSourceBuffer } from './sourceBuffer'; + +class FakeSourceBuffer extends EventTarget { + updating = false; + readonly appended: Uint8Array[] = []; + + appendBuffer(buffer: BufferSource): void { + if (this.updating) { + throw new Error('concurrent append'); + } + this.updating = true; + const bytes = + buffer instanceof ArrayBuffer + ? new Uint8Array(buffer) + : new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + this.appended.push(Uint8Array.from(bytes)); + } + + completeAppend(): void { + this.updating = false; + this.dispatchEvent(new Event('updateend')); + } +} + +describe('ReactiveSourceBuffer', () => { + it('serializes append operations', async () => { + const sourceBuffer = new FakeSourceBuffer(); + const mediaSource = { + addSourceBuffer: () => sourceBuffer, + } as unknown as MediaSource; + const reactive = new ReactiveSourceBuffer(mediaSource, 'vp8'); + + const first = reactive.appendBuffer(new Uint8Array([1])); + const secondResult = reactive.appendBuffer(new Uint8Array([2])).then( + () => null, + (error: unknown) => error, + ); + + await Promise.resolve(); + expect(sourceBuffer.appended).toEqual([new Uint8Array([1])]); + + sourceBuffer.completeAppend(); + await first; + await Promise.resolve(); + expect(sourceBuffer.appended).toEqual([new Uint8Array([1]), new Uint8Array([2])]); + + sourceBuffer.completeAppend(); + expect(await secondResult).toBeNull(); + }); +}); diff --git a/webapp/packages/shadow-player/src/sourceBuffer.ts b/webapp/packages/shadow-player/src/sourceBuffer.ts index 3daa39f5a..c86cfcd66 100644 --- a/webapp/packages/shadow-player/src/sourceBuffer.ts +++ b/webapp/packages/shadow-player/src/sourceBuffer.ts @@ -1,6 +1,7 @@ export class ReactiveSourceBuffer { private readonly sourceBuffer: SourceBuffer; private readonly allBuffers: Blob[] = []; + private pendingOperation = Promise.resolve(); private debug = false; constructor(mediaSource: MediaSource, codec: string) { @@ -11,7 +12,17 @@ export class ReactiveSourceBuffer { this.debug = debug; } - async appendBuffer(buffer: Uint8Array): Promise { + appendBuffer(buffer: Uint8Array): Promise { + const operation = this.pendingOperation.then(() => this.append(buffer)); + this.pendingOperation = operation; + return operation; + } + + whenIdle(): Promise { + return this.pendingOperation; + } + + private async append(buffer: Uint8Array): Promise { if (this.sourceBuffer.updating) { throw new Error('SourceBuffer is already updating'); } diff --git a/webapp/packages/shadow-player/src/streamer.test.ts b/webapp/packages/shadow-player/src/streamer.test.ts new file mode 100644 index 000000000..5a077aa53 --- /dev/null +++ b/webapp/packages/shadow-player/src/streamer.test.ts @@ -0,0 +1,512 @@ +// @vitest-environment jsdom + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { ClientMessage, SegmentStartedMessage, ServerMessage } from './protocol'; + +interface MockServerWebSocket { + sent: ClientMessage[]; + emitOpen: () => void; + emitMessage: (message: ServerMessage) => Promise; + emitClose: (code?: number, reason?: string) => void; + emitError: () => void; +} + +interface MockPlaybackClip { + metadata: SegmentStartedMessage; + video: HTMLVideoElement; + play: ReturnType; + pause: ReturnType; + open: ReturnType; + append: ReturnType; + finish: ReturnType; + resolveOpen: () => void; + resolveAppend: () => void; + resolveFinish: () => void; + setDuration: (duration: number) => void; + loaded: () => void; + end: () => void; +} + +const mocks = vi.hoisted(() => ({ + sockets: [] as MockServerWebSocket[], + clips: [] as MockPlaybackClip[], +})); + +vi.mock('./websocket', () => ({ + ServerWebSocket: class { + readonly sent: ClientMessage[] = []; + private openCallback: (() => void) | null = null; + private messageCallback: ((message: ServerMessage) => Promise | void) | null = null; + private closeCallback: ((event: CloseEvent) => void) | null = null; + private errorCallback: ((event: Event) => void) | null = null; + private failureCallback: ((error: unknown) => void) | null = null; + + constructor(_url: string) { + mocks.sockets.push(this); + } + + onopen(callback: () => void): void { + this.openCallback = callback; + } + + onmessage(callback: (message: ServerMessage) => Promise | void, onFailure: (error: unknown) => void): void { + this.messageCallback = callback; + this.failureCallback = onFailure; + } + + onclose(callback: (event: CloseEvent) => void): void { + this.closeCallback = callback; + } + + onerror(callback: (event: Event) => void): void { + this.errorCallback = callback; + } + + send(message: ClientMessage): void { + this.sent.push(message); + } + + isOpen(): boolean { + return true; + } + + close(): void {} + + emitOpen(): void { + this.openCallback?.(); + } + + async emitMessage(message: ServerMessage): Promise { + try { + await this.messageCallback?.(message); + } catch (error) { + this.failureCallback?.(error); + throw error; + } + } + + emitClose(code = 1006, reason = ''): void { + this.closeCallback?.(new CloseEvent('close', { code, reason, wasClean: false })); + } + + emitError(): void { + this.errorCallback?.(new Event('error')); + } + }, +})); + +vi.mock('./playbackClip', () => ({ + PlaybackClip: class { + readonly video = document.createElement('video'); + readonly play = vi.fn(async () => undefined); + readonly pause = vi.fn(); + readonly open: ReturnType; + readonly append: ReturnType; + readonly finish: ReturnType; + private readonly openPromise: Promise; + private readonly appendPromise: Promise; + private readonly finishPromise: Promise; + private openResolver!: () => void; + private appendResolver!: () => void; + private finishResolver!: () => void; + private duration = 0; + private ended = false; + + constructor(readonly metadata: SegmentStartedMessage) { + this.openPromise = new Promise((resolve) => { + this.openResolver = resolve; + }); + this.appendPromise = new Promise((resolve) => { + this.appendResolver = resolve; + }); + this.finishPromise = new Promise((resolve) => { + this.finishResolver = resolve; + }); + this.open = vi.fn(() => this.openPromise); + this.append = vi.fn(() => this.appendPromise); + this.finish = vi.fn(() => this.finishPromise); + Object.defineProperties(this.video, { + play: { configurable: true, value: this.play }, + pause: { configurable: true, value: this.pause }, + load: { configurable: true, value: vi.fn() }, + duration: { configurable: true, get: () => this.duration }, + ended: { configurable: true, get: () => this.ended }, + }); + mocks.clips.push(this); + } + + resolveOpen(): void { + this.openResolver(); + } + + resolveAppend(): void { + this.appendResolver(); + } + + resolveFinish(): void { + this.finishResolver(); + } + + setDuration(duration: number): void { + this.duration = duration; + this.video.dispatchEvent(new Event('durationchange')); + } + + loaded(): void { + this.video.dispatchEvent(new Event('loadeddata')); + } + + end(): void { + this.ended = true; + this.video.dispatchEvent(new Event('ended')); + } + + setDebug(): void {} + + downloadBufferedFile(): void {} + + dispose(): void { + this.video.remove(); + } + }, +})); + +import { ShadowPlayer } from './streamer'; + +async function flushMicrotasks(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +function createPlayer(attributes: string[] = []): { player: ShadowPlayer; socket: MockServerWebSocket } { + const player = new ShadowPlayer(); + for (const attribute of attributes) { + player.setAttribute(attribute, ''); + } + player.setAttribute('src', 'ws://example.test'); + document.body.appendChild(player); + const socket = mocks.sockets.at(-1); + if (!socket) { + throw new Error('ShadowPlayer did not create a websocket'); + } + socket.emitOpen(); + return { player, socket }; +} + +const firstMetadata: SegmentStartedMessage = { + type: 'segment-started', + codec: 'vp8', + sequence: 0, + width: 640, + height: 480, +}; + +const secondMetadata: SegmentStartedMessage = { + type: 'segment-started', + codec: 'vp8', + sequence: 1, + width: 1280, + height: 720, +}; + +describe('ShadowPlayer', () => { + beforeEach(() => { + mocks.sockets.length = 0; + mocks.clips.length = 0; + }); + + afterEach(() => { + document.body.replaceChildren(); + }); + + it('pulls only after segment and append work completes', async () => { + const { player, socket } = createPlayer(); + const onEnd = vi.fn(); + player.onEnd(onEnd); + expect(socket.sent).toEqual([{ type: 'start' }]); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + expect(firstClip).toBeDefined(); + expect(socket.sent).toEqual([{ type: 'start' }]); + + firstClip.resolveOpen(); + await firstStart; + expect(socket.sent).toEqual([{ type: 'start' }, { type: 'pull' }]); + + const chunk = socket.emitMessage({ type: 'chunk', data: new Uint8Array([1]) }); + await flushMicrotasks(); + expect(socket.sent).toHaveLength(2); + firstClip.resolveAppend(); + await chunk; + expect(socket.sent).toHaveLength(3); + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + expect(firstClip.finish).toHaveBeenCalledOnce(); + expect(mocks.clips).toHaveLength(1); + + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + expect(secondClip).toBeDefined(); + secondClip.resolveOpen(); + await secondStart; + expect(socket.sent).toHaveLength(4); + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + expect(secondClip.finish).toHaveBeenCalledOnce(); + expect(onEnd).not.toHaveBeenCalled(); + + secondClip.resolveFinish(); + await streamEnd; + expect(onEnd).toHaveBeenCalledOnce(); + expect(socket.sent).toHaveLength(4); + }); + + it('does not turn an abrupt close into a clean stream end', async () => { + const { player, socket } = createPlayer(); + const onEnd = vi.fn(); + player.onEnd(onEnd); + + const start = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const clip = mocks.clips[0]; + clip.resolveOpen(); + await start; + + socket.emitClose(); + expect(clip.finish).not.toHaveBeenCalled(); + expect(onEnd).not.toHaveBeenCalled(); + }); + + it.each([4002, 4003, 1011])('surfaces unexpected close code %i', (code) => { + const { player, socket } = createPlayer(); + const onError = vi.fn(); + player.onError(onError); + + socket.emitClose(code, `close ${code}`); + + expect(onError).toHaveBeenCalledOnce(); + expect(onError).toHaveBeenCalledWith({ + type: 'websocket-close', + code, + reason: `close ${code}`, + wasClean: false, + }); + }); + + it('reports only the clean End when a socket error follows it', async () => { + const { player, socket } = createPlayer(); + const onEnd = vi.fn(); + const onError = vi.fn(); + player.onEnd(onEnd); + player.onError(onError); + + const start = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const clip = mocks.clips[0]; + clip.resolveOpen(); + await start; + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + clip.resolveFinish(); + await streamEnd; + socket.emitError(); + + expect(onEnd).toHaveBeenCalledOnce(); + expect(onError).not.toHaveBeenCalled(); + }); + + it('rejects a noncontiguous segment sequence', async () => { + const { socket } = createPlayer(); + + await expect(socket.emitMessage({ ...firstMetadata, sequence: 1 })).rejects.toThrow( + 'Expected segment 0, received 1', + ); + expect(mocks.clips).toHaveLength(0); + }); + + it('does not loop from stream completion while the next segment is not yet playable', async () => { + const { player, socket } = createPlayer(['autoplay', 'loop']); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + firstClip.resolveOpen(); + await firstStart; + firstClip.loaded(); + expect(firstClip.play).toHaveBeenCalledOnce(); + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + secondClip.resolveOpen(); + await secondStart; + + firstClip.end(); + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + secondClip.resolveFinish(); + await streamEnd; + + expect(player._videoElement).toBe(firstClip.video); + expect(firstClip.play).toHaveBeenCalledOnce(); + expect(secondClip.play).not.toHaveBeenCalled(); + + secondClip.loaded(); + expect(player._videoElement).toBe(secondClip.video); + expect(firstClip.play).toHaveBeenCalledOnce(); + expect(secondClip.play).toHaveBeenCalledOnce(); + }); + + it('does not show replay from the ended handler while the next segment is not yet playable', async () => { + const { player, socket } = createPlayer(['autoplay']); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + firstClip.resolveOpen(); + await firstStart; + firstClip.loaded(); + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + secondClip.resolveOpen(); + await secondStart; + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + secondClip.resolveFinish(); + await streamEnd; + firstClip.end(); + + const replayButton = player.shadowRoot?.querySelector('.replay-button'); + expect(player._videoElement).toBe(firstClip.video); + expect(replayButton?.classList.contains('visible')).toBe(false); + expect(secondClip.play).not.toHaveBeenCalled(); + + secondClip.loaded(); + expect(player._videoElement).toBe(secondClip.video); + expect(secondClip.play).toHaveBeenCalledOnce(); + expect(replayButton?.classList.contains('visible')).toBe(false); + }); + + it('coordinates autoplay and loop across the full segment sequence', async () => { + const { player, socket } = createPlayer(['autoplay', 'loop']); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + firstClip.resolveOpen(); + await firstStart; + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + secondClip.resolveOpen(); + await secondStart; + + expect(firstClip.video.hasAttribute('autoplay')).toBe(false); + expect(firstClip.video.hasAttribute('loop')).toBe(false); + expect(secondClip.video.hasAttribute('autoplay')).toBe(false); + expect(secondClip.video.hasAttribute('loop')).toBe(false); + + secondClip.loaded(); + expect(secondClip.play).not.toHaveBeenCalled(); + firstClip.loaded(); + expect(firstClip.play).toHaveBeenCalledOnce(); + + firstClip.end(); + expect(player._videoElement).toBe(secondClip.video); + expect(secondClip.play).toHaveBeenCalledOnce(); + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + secondClip.resolveFinish(); + await streamEnd; + secondClip.end(); + + expect(player._videoElement).toBe(firstClip.video); + expect(firstClip.play).toHaveBeenCalledTimes(2); + expect(player.shadowRoot?.querySelector('.replay-button')?.classList.contains('visible')).toBe(false); + }); + + it('keeps segment playback chronological while preserving pause, seek, and replay intent', async () => { + const { player, socket } = createPlayer(); + + const firstStart = socket.emitMessage(firstMetadata); + await flushMicrotasks(); + const firstClip = mocks.clips[0]; + firstClip.resolveOpen(); + await firstStart; + + const secondStart = socket.emitMessage(secondMetadata); + await flushMicrotasks(); + firstClip.resolveFinish(); + await flushMicrotasks(); + const secondClip = mocks.clips[1]; + secondClip.resolveOpen(); + await secondStart; + + firstClip.setDuration(10); + secondClip.setDuration(20); + secondClip.loaded(); + expect(player._videoElement).toBeNull(); + firstClip.loaded(); + expect(player._videoElement).toBe(firstClip.video); + + player.play(); + expect(firstClip.play).toHaveBeenCalledOnce(); + player.pause(); + firstClip.end(); + expect(player._videoElement).toBe(secondClip.video); + expect(secondClip.play).not.toHaveBeenCalled(); + + player.play(); + expect(secondClip.play).toHaveBeenCalledOnce(); + + const firstTimelineSegment = player.shadowRoot?.querySelector('.timeline-segment'); + expect(firstTimelineSegment).not.toBeNull(); + vi.spyOn(firstTimelineSegment as HTMLElement, 'getBoundingClientRect').mockReturnValue({ + x: 0, + y: 0, + width: 100, + height: 10, + top: 0, + right: 100, + bottom: 10, + left: 0, + toJSON: () => ({}), + }); + firstTimelineSegment?.dispatchEvent(new MouseEvent('click', { clientX: 25 })); + expect(player._videoElement).toBe(firstClip.video); + expect(firstClip.video.currentTime).toBe(2.5); + expect(secondClip.video.currentTime).toBe(0); + + const streamEnd = socket.emitMessage({ type: 'stream-ended' }); + await flushMicrotasks(); + secondClip.resolveFinish(); + await streamEnd; + + firstClip.end(); + secondClip.end(); + const replayButton = player.shadowRoot?.querySelector('.replay-button'); + expect(replayButton?.classList.contains('visible')).toBe(true); + replayButton?.click(); + expect(player._videoElement).toBe(firstClip.video); + expect(firstClip.video.currentTime).toBe(0); + expect(secondClip.video.currentTime).toBe(0); + }); +}); diff --git a/webapp/packages/shadow-player/src/streamer.ts b/webapp/packages/shadow-player/src/streamer.ts index ac5488dd9..046b366c5 100644 --- a/webapp/packages/shadow-player/src/streamer.ts +++ b/webapp/packages/shadow-player/src/streamer.ts @@ -16,6 +16,12 @@ export type ShadowPlayerError = type: 'websocket'; inner: ErrorEvent; } + | { + type: 'websocket-close'; + code: number; + reason: string; + wasClean: boolean; + } | { type: 'protocol'; inner: ErrorMessage; @@ -30,6 +36,7 @@ export type ShadowPlayerError = }; type ShadowPlayerErrorCallback = (error: ShadowPlayerError) => void; +type TerminalOutcome = 'none' | 'end' | 'error' | 'closed'; export class ShadowPlayer extends HTMLElement { _videoElement: HTMLVideoElement | null = null; @@ -49,6 +56,7 @@ export class ShadowPlayer extends HTMLElement { private awaitingResponse = false; private shouldPlay = false; private streamEnded = false; + private terminalOutcome: TerminalOutcome = 'closed'; private muted = true; private volume = 1; private controls: PlaybackControls | null = null; @@ -93,10 +101,13 @@ export class ShadowPlayer extends HTMLElement { return; } - if (name === 'autoplay' && newValue !== null) { - this.shouldPlay = true; + if (name === 'autoplay') { + if (newValue !== null) { + this.shouldPlay = true; + } + return; } - if (name === 'controls') { + if (name === 'controls' || name === 'loop') { return; } if (name === 'muted') { @@ -220,7 +231,7 @@ export class ShadowPlayer extends HTMLElement { if (this.activateNextClip()) { return; } - if (this.streamEnded && this.activeClip?.video.ended) { + if (this.isSequencePlaybackComplete()) { this.replay(); } } @@ -253,6 +264,7 @@ export class ShadowPlayer extends HTMLElement { return; } + this.terminalOutcome = 'none'; this.streamEnded = false; this._replayButton?.classList.remove('visible'); this.renderPlayerControls(); @@ -273,7 +285,7 @@ export class ShadowPlayer extends HTMLElement { } private async handleServerMessage(websocket: ServerWebSocket, message: ServerMessage): Promise { - if (this.websocket !== websocket) { + if (this.websocket !== websocket || this.terminalOutcome !== 'none') { return; } if (!this.awaitingResponse) { @@ -282,38 +294,40 @@ export class ShadowPlayer extends HTMLElement { this.awaitingResponse = false; if (message.type === 'segment-started') { + await this.startSegment(websocket, message); this.sendRequest(websocket, 'pull'); - await this.startSegment(message); return; } if (message.type === 'chunk') { - if (!this.receivingClip) { + const clip = this.receivingClip; + if (!clip) { throw new Error('Received a chunk before a segment started'); } + await clip.append(message.data); this.sendRequest(websocket, 'pull'); - await this.receivingClip.append(message.data); return; } if (message.type === 'error') { - this.onErrorCallback?.({ type: 'protocol', inner: message }); + this.reportTerminalError({ type: 'protocol', inner: message }); return; } - this.finishReceivingClip(); - this.streamEnded = true; - this.renderPlayerControls(); - if (this.activeClip?.video.ended) { - this.showReplayButton(); + await this.finishReceivingClip(); + if (this.websocket !== websocket || this.terminalOutcome !== 'none') { + return; } - this.onEndCallback?.(); + this.completeStream(); } - private async startSegment(metadata: SegmentStartedMessage): Promise { + private async startSegment(websocket: ServerWebSocket, metadata: SegmentStartedMessage): Promise { if (metadata.sequence !== this.clips.length) { throw new Error(`Expected segment ${this.clips.length}, received ${metadata.sequence}`); } - this.finishReceivingClip(); + await this.finishReceivingClip(); + if (this.websocket !== websocket || this.terminalOutcome !== 'none') { + return; + } const clip = new PlaybackClip(metadata); clip.setDebug(this.debug); this.configureVideo(clip); @@ -324,12 +338,13 @@ export class ShadowPlayer extends HTMLElement { await clip.open(); } - private finishReceivingClip(): void { + private async finishReceivingClip(): Promise { const clip = this.receivingClip; if (!clip) { return; } - clip.finish(); + this.receivingClip = null; + await clip.finish(); this.renderAllSegments(); } @@ -339,7 +354,13 @@ export class ShadowPlayer extends HTMLElement { video.muted = this.muted; video.volume = this.volume; for (const attribute of ShadowPlayer.observedAttributes) { - if (attribute !== 'src' && attribute !== 'controls' && attribute !== 'muted') { + if ( + attribute !== 'src' && + attribute !== 'autoplay' && + attribute !== 'controls' && + attribute !== 'loop' && + attribute !== 'muted' + ) { this.applyVideoAttribute(video, attribute, this.getAttribute(attribute)); } } @@ -368,8 +389,8 @@ export class ShadowPlayer extends HTMLElement { if (this.activeClip !== clip) { return; } - if (!this.activateNextClip() && this.streamEnded) { - this.showReplayButton(); + if (!this.activateNextClip()) { + this.handleSequencePlaybackEnd(); } this.renderClipControls(clip); this.renderPlayerControls(); @@ -508,7 +529,10 @@ export class ShadowPlayer extends HTMLElement { } private sendRequest(websocket: ServerWebSocket, type: 'start' | 'pull'): void { - if (this.websocket !== websocket) { + if (this.websocket !== websocket || this.terminalOutcome !== 'none') { + return; + } + if (!websocket.isOpen()) { return; } if (this.awaitingResponse) { @@ -524,11 +548,20 @@ export class ShadowPlayer extends HTMLElement { } this.awaitingResponse = false; this.websocket = null; - if (event.code === 4001) { - this.onErrorCallback?.({ - type: 'session-not-found', - message: 'Recording session is no longer active', - }); + if (this.terminalOutcome === 'none') { + const error: ShadowPlayerError = + event.code === 4001 + ? { + type: 'session-not-found', + message: 'Recording session is no longer active', + } + : { + type: 'websocket-close', + code: event.code, + reason: event.reason, + wasClean: event.wasClean, + }; + this.reportTerminalError(error); } this.renderPlayerControls(); } @@ -537,7 +570,7 @@ export class ShadowPlayer extends HTMLElement { if (this.websocket !== websocket) { return; } - this.onErrorCallback?.({ + this.reportTerminalError({ type: 'websocket', inner: event as ErrorEvent, }); @@ -548,13 +581,62 @@ export class ShadowPlayer extends HTMLElement { return; } const error = value instanceof Error ? value : new Error(String(value)); + if ( + !this.reportTerminalError({ + type: 'player', + inner: error, + }) + ) { + return; + } this.awaitingResponse = false; - this.onErrorCallback?.({ type: 'player', inner: error }); websocket.close(1000, 'Player failure'); this.websocket = null; this.renderPlayerControls(); } + private completeStream(): void { + if (this.terminalOutcome !== 'none') { + return; + } + this.terminalOutcome = 'end'; + this.streamEnded = true; + this.renderPlayerControls(); + this.handleSequencePlaybackEnd(); + this.onEndCallback?.(); + } + + private reportTerminalError(error: ShadowPlayerError): boolean { + if (this.terminalOutcome !== 'none') { + return false; + } + this.terminalOutcome = 'error'; + this.onErrorCallback?.(error); + this.renderPlayerControls(); + return true; + } + + private handleSequencePlaybackEnd(): void { + if (!this.isSequencePlaybackComplete()) { + return; + } + if (this.hasAttribute('loop') && this.shouldPlay) { + this.replay(); + } else { + this.showReplayButton(); + } + } + + private isSequencePlaybackComplete(): boolean { + const activeClip = this.activeClip; + return ( + this.streamEnded && + activeClip !== null && + activeClip.video.ended && + activeClip.metadata.sequence === this.clips.length - 1 + ); + } + private reportPlayerError(value: unknown): void { const error = value instanceof Error ? value : new Error(String(value)); this.onErrorCallback?.({ type: 'player', inner: error }); @@ -579,6 +661,8 @@ export class ShadowPlayer extends HTMLElement { const websocket = this.websocket; this.websocket = null; this.awaitingResponse = false; + this.terminalOutcome = 'closed'; + this.streamEnded = false; websocket?.close(1000, 'Component cleanup'); for (const clip of this.clips) { clip.dispose(); diff --git a/webapp/packages/shadow-player/src/websocket.test.ts b/webapp/packages/shadow-player/src/websocket.test.ts new file mode 100644 index 000000000..f72372c4d --- /dev/null +++ b/webapp/packages/shadow-player/src/websocket.test.ts @@ -0,0 +1,142 @@ +// @vitest-environment jsdom + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ServerWebSocket } from './websocket'; + +interface Deferred { + promise: Promise; + resolve: (value: T | PromiseLike) => void; +} + +function deferred(): Deferred { + let resolve!: Deferred['resolve']; + const promise = new Promise((promiseResolve) => { + resolve = promiseResolve; + }); + return { promise, resolve }; +} + +function encodedMessage(type: number, payload = ''): ArrayBuffer { + const encodedPayload = new TextEncoder().encode(payload); + const message = new Uint8Array(1 + encodedPayload.length); + message[0] = type; + message.set(encodedPayload, 1); + return message.buffer; +} + +class FakeWebSocket { + static readonly OPEN = 1; + static latest: FakeWebSocket | null = null; + + binaryType: BinaryType = 'blob'; + readyState = FakeWebSocket.OPEN; + onopen: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + onclose: ((event: CloseEvent) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + + constructor(readonly url: string) { + FakeWebSocket.latest = this; + } + + send(): void {} + + close(): void {} + + emitMessage(data: ArrayBuffer): void { + this.onmessage?.(new MessageEvent('message', { data })); + } + + emitClose(): void { + this.onclose?.(new CloseEvent('close', { code: 1006 })); + } + + emitError(): void { + this.onerror?.(new Event('error')); + } +} + +describe('ServerWebSocket', () => { + beforeEach(() => { + vi.stubGlobal('WebSocket', FakeWebSocket); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + FakeWebSocket.latest = null; + }); + + it('serializes messages and dispatches close after pending message work', async () => { + const websocket = new ServerWebSocket('ws://example.test'); + const socket = FakeWebSocket.latest; + expect(socket).not.toBeNull(); + + const firstStarted = deferred(); + const releaseFirst = deferred(); + const secondStarted = deferred(); + const closed = deferred(); + const calls: string[] = []; + + websocket.onmessage(async (message) => { + calls.push(message.type); + if (message.type === 'segment-started') { + firstStarted.resolve(); + await releaseFirst.promise; + } else { + secondStarted.resolve(); + } + }, vi.fn()); + websocket.onclose(() => closed.resolve()); + + socket?.emitMessage(encodedMessage(1, '{"codec":"vp8","sequence":0,"width":640,"height":480}')); + socket?.emitMessage(encodedMessage(0, 'chunk')); + socket?.emitClose(); + + await firstStarted.promise; + await Promise.resolve(); + expect(calls).toEqual(['segment-started']); + + let closeDispatched = false; + void closed.promise.then(() => { + closeDispatched = true; + }); + await Promise.resolve(); + expect(closeDispatched).toBe(false); + + releaseFirst.resolve(); + await secondStarted.promise; + await closed.promise; + expect(calls).toEqual(['segment-started', 'chunk']); + }); + + it('serializes an error after a queued stream end', async () => { + const websocket = new ServerWebSocket('ws://example.test'); + const socket = FakeWebSocket.latest; + expect(socket).not.toBeNull(); + + const endStarted = deferred(); + const releaseEnd = deferred(); + const errorDispatched = deferred(); + + websocket.onmessage(async (message) => { + expect(message).toEqual({ type: 'stream-ended' }); + endStarted.resolve(); + await releaseEnd.promise; + }, vi.fn()); + websocket.onerror(() => errorDispatched.resolve()); + + socket?.emitMessage(encodedMessage(3)); + socket?.emitError(); + + await endStarted.promise; + let errorObserved = false; + void errorDispatched.promise.then(() => { + errorObserved = true; + }); + await Promise.resolve(); + expect(errorObserved).toBe(false); + + releaseEnd.resolve(); + await errorDispatched.promise; + }); +}); diff --git a/webapp/packages/shadow-player/src/websocket.ts b/webapp/packages/shadow-player/src/websocket.ts index 99211c712..91e1d698f 100644 --- a/webapp/packages/shadow-player/src/websocket.ts +++ b/webapp/packages/shadow-player/src/websocket.ts @@ -2,6 +2,8 @@ import { ClientMessage, parseClientMessage, parseServerMessage, ServerMessage } export class ServerWebSocket { private readonly socket: WebSocket; + private pendingEvent = Promise.resolve(); + private closed = false; constructor(url: string) { this.socket = new WebSocket(url); @@ -14,33 +16,47 @@ export class ServerWebSocket { onmessage(callback: (message: ServerMessage) => Promise | void, onFailure: (error: unknown) => void): void { this.socket.onmessage = (event) => { - try { - if (!(event.data instanceof ArrayBuffer)) { - throw new Error('Server sent a non-binary message'); + this.enqueueEvent(async () => { + try { + if (!(event.data instanceof ArrayBuffer)) { + throw new Error('Server sent a non-binary message'); + } + await callback(parseServerMessage(event.data)); + } catch (error) { + onFailure(error); } - Promise.resolve(callback(parseServerMessage(event.data))).catch(onFailure); - } catch (error) { - onFailure(error); - } + }); }; } onclose(callback: (event: CloseEvent) => void): void { - this.socket.onclose = callback; + this.socket.onclose = (event) => { + this.closed = true; + this.enqueueEvent(() => callback(event)); + }; } onerror(callback: (event: Event) => void): void { - this.socket.onerror = callback; + this.socket.onerror = (event) => this.enqueueEvent(() => callback(event)); } send(message: ClientMessage): void { - if (this.socket.readyState !== WebSocket.OPEN) { + if (!this.isOpen()) { throw new Error('WebSocket is not open'); } this.socket.send(parseClientMessage(message)); } + isOpen(): boolean { + return !this.closed && this.socket.readyState === WebSocket.OPEN; + } + close(code: number, reason: string): void { this.socket.close(code, reason); } + + private enqueueEvent(callback: () => Promise | void): void { + const event = this.pendingEvent.then(callback); + this.pendingEvent = event.catch(() => undefined); + } } diff --git a/webapp/packages/shadow-player/vite.config.ts b/webapp/packages/shadow-player/vite.config.ts index 632a03336..10e685ef8 100644 --- a/webapp/packages/shadow-player/vite.config.ts +++ b/webapp/packages/shadow-player/vite.config.ts @@ -1,5 +1,5 @@ import path from 'node:path'; -import { UserConfig, defineConfig } from 'vite'; +import { defineConfig, UserConfig } from 'vite'; import dts from 'vite-plugin-dts'; import { viteStaticCopy } from 'vite-plugin-static-copy'; @@ -51,12 +51,14 @@ const staticCopyPlugin = viteStaticCopy({ const Plugins = { debug: [ dts({ + exclude: ['src/**/*.test.ts'], insertTypesEntry: true, }), staticCopyPlugin, ], release: [ dts({ + exclude: ['src/**/*.test.ts'], insertTypesEntry: true, }), staticCopyPlugin, diff --git a/webapp/pnpm-lock.yaml b/webapp/pnpm-lock.yaml index 295ff1c75..28cd4e092 100644 --- a/webapp/pnpm-lock.yaml +++ b/webapp/pnpm-lock.yaml @@ -268,6 +268,9 @@ importers: packages/shadow-player: devDependencies: + jsdom: + specifier: ^20.0.3 + version: 20.0.3 ts-node: specifier: ^10.9.2 version: 10.9.2(@types/node@22.19.3)(typescript@5.6.3) @@ -283,6 +286,9 @@ importers: vite-plugin-static-copy: specifier: ^2.3.0 version: 2.3.2(vite@5.4.21(@types/node@22.19.3)(less@4.4.0)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.43.1)) + vitest: + specifier: ^3.1.1 + version: 3.2.7(@types/node@20.19.27)(jiti@2.6.1)(jsdom@20.0.3)(less@4.4.0)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.43.1) packages/web-recorder: devDependencies: