From 09b64e2f8676e5c7a11ad37da57533bfd38929d8 Mon Sep 17 00:00:00 2001 From: ANDROIDHASSAN <84630851+ANDROIDHASSAN@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:29:38 +0530 Subject: [PATCH] fix(rtc): leave the bytes past an AudioFrame's samples out of its data AudioFrame accepts a buffer larger than num_channels * samples_per_channel samples, but data, to_wav_bytes and AudioProcessingModule used the whole buffer. combine_audio_frames then shifted every later frame by the extra bytes and dropped its tail, and the APM was told the frame was longer than 10 ms. --- livekit-rtc/livekit/rtc/apm.py | 4 +- livekit-rtc/livekit/rtc/audio_frame.py | 11 ++-- tests/rtc/test_audio_frame.py | 78 ++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 tests/rtc/test_audio_frame.py diff --git a/livekit-rtc/livekit/rtc/apm.py b/livekit-rtc/livekit/rtc/apm.py index 9fda2ce9..94656855 100644 --- a/livekit-rtc/livekit/rtc/apm.py +++ b/livekit-rtc/livekit/rtc/apm.py @@ -56,7 +56,7 @@ def process_stream(self, data: AudioFrame) -> None: req = proto_ffi.FfiRequest() req.apm_process_stream.apm_handle = self._ffi_handle.handle req.apm_process_stream.data_ptr = get_address(data._data) - req.apm_process_stream.size = len(data._data) + req.apm_process_stream.size = data.data.nbytes req.apm_process_stream.sample_rate = data.sample_rate req.apm_process_stream.num_channels = data.num_channels @@ -84,7 +84,7 @@ def process_reverse_stream(self, data: AudioFrame) -> None: req = proto_ffi.FfiRequest() req.apm_process_reverse_stream.apm_handle = self._ffi_handle.handle req.apm_process_reverse_stream.data_ptr = get_address(data._data) - req.apm_process_reverse_stream.size = len(data._data) + req.apm_process_reverse_stream.size = data.data.nbytes req.apm_process_reverse_stream.sample_rate = data.sample_rate req.apm_process_reverse_stream.num_channels = data.num_channels diff --git a/livekit-rtc/livekit/rtc/audio_frame.py b/livekit-rtc/livekit/rtc/audio_frame.py index 0a776adc..84e74766 100644 --- a/livekit-rtc/livekit/rtc/audio_frame.py +++ b/livekit-rtc/livekit/rtc/audio_frame.py @@ -41,7 +41,8 @@ def __init__( Args: data (Union[bytes, bytearray, memoryview]): The raw audio data, which must be at least - `num_channels * samples_per_channel * sizeof(int16)` bytes long. + `num_channels * samples_per_channel * sizeof(int16)` bytes long. Bytes past that + size are not part of the frame. sample_rate (int): The sample rate of the audio in Hz. num_channels (int): The number of audio channels (e.g., 1 for mono, 2 for stereo). samples_per_channel (int): The number of samples per channel. @@ -121,9 +122,11 @@ def data(self) -> memoryview: Returns a memory view of the audio data as 16-bit signed integers. Returns: - memoryview: A memory view of the audio data. + memoryview: A memory view of the frame's `num_channels * samples_per_channel` + samples. Bytes of the buffer past them are left out. """ - return memoryview(self._data).cast("B").cast("h") + size = self.num_channels * self.samples_per_channel * ctypes.sizeof(ctypes.c_int16) + return memoryview(self._data).cast("B")[:size].cast("h") @property def sample_rate(self) -> int: @@ -180,7 +183,7 @@ def to_wav_bytes(self) -> bytes: wav.setnchannels(self.num_channels) wav.setsampwidth(2) wav.setframerate(self.sample_rate) - wav.writeframes(self._data) + wav.writeframes(self.data) return wav_file.getvalue() diff --git a/tests/rtc/test_audio_frame.py b/tests/rtc/test_audio_frame.py new file mode 100644 index 00000000..d6afee72 --- /dev/null +++ b/tests/rtc/test_audio_frame.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import array +import io +import types +import wave + +import pytest + +from livekit import rtc + + +def _frame(samples: list[int], samples_per_channel: int, num_channels: int = 1) -> rtc.AudioFrame: + return rtc.AudioFrame( + array.array("h", samples).tobytes(), 16000, num_channels, samples_per_channel + ) + + +def test_data_holds_only_the_frame_samples() -> None: + # the buffer may be larger than the frame; the samples past it are not part of the frame + frame = _frame([1, 2, 3, 4, 99, 99], samples_per_channel=2, num_channels=2) + + assert list(frame.data) == [1, 2, 3, 4] + + +def test_combine_audio_frames_keeps_every_frame_sample() -> None: + combined = rtc.combine_audio_frames([_frame([1, 2, 99, 99], 2), _frame([3, 4], 2)]) + + assert combined.samples_per_channel == 4 + assert list(combined.data) == [1, 2, 3, 4] + + +def test_to_wav_bytes_writes_only_the_frame_samples() -> None: + frame = _frame([1, 2, 99, 99], samples_per_channel=2) + + with wave.open(io.BytesIO(frame.to_wav_bytes())) as wav: + assert wav.getnframes() == 2 + assert array.array("h", wav.readframes(wav.getnframes())).tolist() == [1, 2] + + +def test_frame_over_a_larger_bytearray_shares_its_memory() -> None: + # AudioProcessingModule processes frames in place, so the frame must not copy the buffer + buffer = bytearray(array.array("h", [1, 2, 99, 99]).tobytes()) + frame = rtc.AudioFrame(buffer, 16000, 1, 2) + + frame.data[0] = 7 + assert array.array("h", bytes(buffer[:2])).tolist() == [7] + + +@pytest.mark.parametrize("method", ["process_stream", "process_reverse_stream"]) +def test_apm_sends_only_the_frame_bytes(monkeypatch: pytest.MonkeyPatch, method: str) -> None: + from livekit.rtc import apm as apm_module + + requests = [] + + class _FakeClient: + def request(self, req): # type: ignore[no-untyped-def] + requests.append(req) + return apm_module.proto_ffi.FfiResponse() + + monkeypatch.setattr(apm_module, "FfiClient", types.SimpleNamespace(instance=_FakeClient())) + module = object.__new__(apm_module.AudioProcessingModule) + module._ffi_handle = types.SimpleNamespace(handle=1) # type: ignore[assignment] + + # a 10 ms mono frame at 16 kHz over a buffer with 40 extra samples + getattr(module, method)(rtc.AudioFrame(bytearray(400), 16000, 1, 160)) + + sent = getattr(requests[0], "apm_" + method) + assert sent.size == 320 + + +def test_serialized_frame_holds_only_the_frame_samples() -> None: + pydantic = pytest.importorskip("pydantic") + adapter = pydantic.TypeAdapter(rtc.AudioFrame) + + data = adapter.dump_python(_frame([1, 2, 99, 99], samples_per_channel=2)) + + assert list(adapter.validate_python(data).data) == [1, 2]