From 48ffba60c06329fa68dcde67829a2660ebd75192 Mon Sep 17 00:00:00 2001 From: Chrizz Gelbach Date: Thu, 15 Oct 2026 00:03:09 +0200 Subject: [PATCH 1/4] Drop encoder priming and padding for gapless playback Decoders output the encoder delay (priming) and padding (remainder) of formats like AAC and MP3 as audio. The packet table info was only used for the duration, so consecutive entries were separated by a short silence (2112 + remainder frames, about 50 ms, for AAC). - Keep only the decoded frames in [priming, priming + valid) while the decoded position is exact (decoding from the start of the stream) - After a seek, drop the remainder at the end of the stream instead - Frame counts are converted to the output sample rate --- .../Streaming/Audio Entry/AudioEntry.swift | 1 + .../Audio Entry/Models/EntryFrames.swift | 10 +++ .../Streaming/AudioPlayer/AudioPlayer.swift | 2 + .../Processors/AudioFileStreamProcessor.swift | 81 ++++++++++++++++++- 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift index c948801..0ce2e01 100644 --- a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift +++ b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift @@ -100,6 +100,7 @@ class AudioEntry { framesState.played = 0 framesState.queued = 0 framesState.lastFrameQueued = -1 + framesState.decoded = 0 } func has(same source: CoreAudioStreamSource) -> Bool { diff --git a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift index 1b4d377..bf6b07c 100644 --- a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift +++ b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift @@ -9,4 +9,14 @@ final class EntryFramesState { var queued: Int = 0 var played: Int = 0 var lastFrameQueued: Int = -1 + + /// Gapless info from the packet table, in source frames: encoder delay (priming), audio frames + /// (valid) and padding (remainder). Decoders output priming and remainder like audio; both are + /// dropped. `validFrames` is nil when the stream has no packet table info. + var primingFrames: Int = 0 + var validFrames: Int? + var remainderFrames: Int = 0 + /// Output frames decoded since the start of the stream, including dropped ones. + /// nil after a seek, where the decoded position is not exact. + var decoded: Int? = 0 } diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift index 13fba34..358344d 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift @@ -1091,6 +1091,8 @@ extension AudioPlayer: AudioStreamSourceDelegate { Logger.debug("Failed to flush remaining packets at EOF: \(flushStatus)", category: .generic) } + fileStreamProcessor.trimTrailingGaplessPadding() + readingEntry.lock.lock() readingEntry.framesState.lastFrameQueued = readingEntry.framesState.queued readingEntry.lock.unlock() diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift index 77d560e..8944cad 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift @@ -210,6 +210,9 @@ final class AudioFileStreamProcessor { } readingEntry.reset() + readingEntry.lock.lock() + readingEntry.framesState.decoded = nil + readingEntry.lock.unlock() readingEntry.seek(at: Int(seekByteOffset)) rendererContext.waitingForDataAfterSeekFrameCount.write { $0 = 0 } playerContext.setInternalState(to: .waitingForDataAfterSeek) @@ -431,6 +434,9 @@ final class AudioFileStreamProcessor { entry.lock.lock(); defer { entry.lock.unlock() } if pti.mNumberValidFrames > 0 { entry.audioStreamState.dataPacketCount = Double(pti.mNumberValidFrames) / Double(max(1, entry.audioStreamFormat.mFramesPerPacket)) + entry.framesState.primingFrames = Int(pti.mPrimingFrames) + entry.framesState.validFrames = Int(pti.mNumberValidFrames) + entry.framesState.remainderFrames = Int(pti.mRemainderFrames) } } @@ -581,7 +587,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded = framesToDecode + framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) @@ -591,6 +597,12 @@ final class AudioFileStreamProcessor { return } + if framesAdded < framesToDecode { + // dropped frames left a hole before the end of the buffer, continue right after the kept ones + fillUsedFrames(framesCount: framesAdded) + continue packetProcess + } + framesToDecode = start if framesToDecode == 0 { fillUsedFrames(framesCount: framesAdded) @@ -611,7 +623,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded += framesToDecode + framesAdded += trimGaplessPadding(bufferIndex: 0, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) @@ -643,7 +655,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded = framesToDecode + framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) return @@ -676,6 +688,69 @@ final class AudioFileStreamProcessor { bufferList[0].mNumberChannels = rendererContext.audioBuffer.mNumberChannels } + /// Drops the encoder delay (priming) and padding (remainder) that decoders output for formats like + /// AAC and MP3, using the packet table info of the stream. Without this, consecutive entries are + /// separated by a short silence (about 50 ms for AAC). + /// Keeps the decoded frames in [priming, priming + valid) while the decoded position is exact; + /// after a seek, `trimTrailingGaplessPadding()` drops the padding at the end of the stream instead. + /// + /// - parameter bufferIndex: The frame index in the renderer buffer where the decoded frames start + /// - parameter framesCount: The number of frames decoded there + /// - Returns: The number of frames to keep; they are moved to `bufferIndex` + @inline(__always) + private func trimGaplessPadding(bufferIndex: UInt32, framesCount: UInt32) -> UInt32 { + guard framesCount > 0, let entry = playerContext.audioReadingEntry else { return framesCount } + entry.lock.lock() + let framesState = entry.framesState + let decodedStart = framesState.decoded + if let decodedStart { + framesState.decoded = decodedStart + Int(framesCount) + } + let primingFrames = framesState.primingFrames + let validFrames = framesState.validFrames + let sourceSampleRate = entry.audioStreamFormat.mSampleRate + entry.lock.unlock() + + guard let decodedStart, let validFrames, sourceSampleRate > 0 else { return framesCount } + let ratio = outputAudioFormat.mSampleRate / sourceSampleRate + let keepFrom = Int((Double(primingFrames) * ratio).rounded()) + let keepTo = keepFrom + Int((Double(validFrames) * ratio).rounded()) + let first = max(decodedStart, keepFrom) + let last = min(decodedStart + Int(framesCount), keepTo) + guard last > first else { return 0 } + + let leadingFrames = first - decodedStart + let keptFrames = last - first + if leadingFrames > 0, let mData = rendererContext.audioBuffer.mData { + let frameSize = Int(rendererContext.bufferContext.sizeInBytes) + let destination = mData + Int(bufferIndex) * frameSize + memmove(destination, destination + leadingFrames * frameSize, keptFrames * frameSize) + } + return UInt32(keptFrames) + } + + /// Drops the encoder padding (remainder) at the end of the stream when `trimGaplessPadding` could not, + /// because the decoded position is not exact after a seek. + /// Must be called once all packets of the reading entry have been decoded: the padding is the + /// last part written to the renderer buffer then. + func trimTrailingGaplessPadding() { + guard let entry = playerContext.audioReadingEntry else { return } + // same lock order as the renderer: buffer first, then entry + rendererContext.lock.lock(); defer { rendererContext.lock.unlock() } + entry.lock.lock(); defer { entry.lock.unlock() } + + let framesState = entry.framesState + let sourceSampleRate = entry.audioStreamFormat.mSampleRate + guard framesState.decoded == nil, framesState.remainderFrames > 0, sourceSampleRate > 0 else { return } + let outputRemainderFrames = Int((Double(framesState.remainderFrames) * outputAudioFormat.mSampleRate / sourceSampleRate).rounded()) + let notPlayedFrames = framesState.queued - framesState.played + let droppedFrames = min(outputRemainderFrames, notPlayedFrames, Int(rendererContext.bufferContext.frameUsedCount)) + guard droppedFrames > 0 else { return } + + rendererContext.bufferContext.frameUsedCount -= UInt32(droppedFrames) + framesState.queued -= droppedFrames + } + /// Advances the processed frames for buffer and reading entry /// /// - parameter frameCount: An `UInt32` value to be added to the used count of the buffers. From e2e51ffa13cac5ec56348f8ddd562bf65fd80aa8 Mon Sep 17 00:00:00 2001 From: Chrizz Gelbach Date: Thu, 15 Oct 2026 10:19:22 +0200 Subject: [PATCH 2/4] Address review findings for gapless trimming - Mark the decoded position as inexact when re-reading from a seek offset (reset(decodedPositionExact:)), so mid-stream data isn't trimmed as the start - Fall back to trailing-only trimming when packet table info arrives late - Account for frames after a fully filled decode so they aren't overwritten - Pass the EOF entry to trimTrailingGaplessPadding instead of looking it up - Extract the trim math into GaplessTrim and add unit tests --- .../Streaming/Audio Entry/AudioEntry.swift | 6 +- .../Audio Entry/Models/EntryFrames.swift | 23 ++++++ .../Streaming/AudioPlayer/AudioPlayer.swift | 4 +- .../Processors/AudioFileStreamProcessor.swift | 41 +++++----- .../Audio Entry/AudioEntryTests.swift | 8 ++ .../Audio Entry/GaplessTrimTests.swift | 74 +++++++++++++++++++ 6 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift diff --git a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift index 0ce2e01..baeba15 100644 --- a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift +++ b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift @@ -95,12 +95,14 @@ class AudioEntry { source.seek(at: offset) } - func reset() { + /// - parameter decodedPositionExact: Whether the entry is re-read from the start of the stream. + /// Pass `false` when re-reading from a seek offset, so gapless trimming doesn't treat mid-stream data as the start. + func reset(decodedPositionExact: Bool = true) { lock.lock(); defer { lock.unlock() } framesState.played = 0 framesState.queued = 0 framesState.lastFrameQueued = -1 - framesState.decoded = 0 + framesState.decoded = decodedPositionExact ? 0 : nil } func has(same source: CoreAudioStreamSource) -> Bool { diff --git a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift index bf6b07c..db2d2c3 100644 --- a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift +++ b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift @@ -20,3 +20,26 @@ final class EntryFramesState { /// nil after a seek, where the decoded position is not exact. var decoded: Int? = 0 } + +/// Pure gapless trimming math, in output frames. +enum GaplessTrim { + /// The part of a decoded chunk to keep. + /// - parameter decodedStart: Output frames decoded before this chunk + /// - parameter framesCount: Frames in this chunk + /// - parameter ratio: Output sample rate / source sample rate + /// - Returns: The frames to skip at the start of the chunk and the frames to keep after them + static func keptRange(decodedStart: Int, framesCount: Int, primingFrames: Int, validFrames: Int, ratio: Double) -> (leading: Int, kept: Int) { + let keepFrom = Int((Double(primingFrames) * ratio).rounded()) + let keepTo = keepFrom + Int((Double(validFrames) * ratio).rounded()) + let first = max(decodedStart, keepFrom) + let last = min(decodedStart + framesCount, keepTo) + guard last > first else { return (0, 0) } + return (first - decodedStart, last - first) + } + + /// The output frames of end padding to drop, limited to what is still unplayed and buffered. + static func trailingFramesToDrop(remainderFrames: Int, ratio: Double, notPlayedFrames: Int, usedFrames: Int) -> Int { + let outputRemainder = Int((Double(remainderFrames) * ratio).rounded()) + return max(0, min(outputRemainder, notPlayedFrames, usedFrames)) + } +} diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift index 358344d..b5b5025 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift @@ -730,7 +730,7 @@ open class AudioPlayer { playingEntry != snapshot.reading { playingEntry.audioStreamState.processedDataFormat = false - playingEntry.reset() + playingEntry.reset(decodedPositionExact: false) if let readingEntry = snapshot.reading { readingEntry.delegate = nil readingEntry.close() @@ -1091,7 +1091,7 @@ extension AudioPlayer: AudioStreamSourceDelegate { Logger.debug("Failed to flush remaining packets at EOF: \(flushStatus)", category: .generic) } - fileStreamProcessor.trimTrailingGaplessPadding() + fileStreamProcessor.trimTrailingGaplessPadding(for: readingEntry) readingEntry.lock.lock() readingEntry.framesState.lastFrameQueued = readingEntry.framesState.queued diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift index 8944cad..d516bfa 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift @@ -209,10 +209,7 @@ final class AudioFileStreamProcessor { AudioConverterReset(converted) } - readingEntry.reset() - readingEntry.lock.lock() - readingEntry.framesState.decoded = nil - readingEntry.lock.unlock() + readingEntry.reset(decodedPositionExact: false) readingEntry.seek(at: Int(seekByteOffset)) rendererContext.waitingForDataAfterSeekFrameCount.write { $0 = 0 } playerContext.setInternalState(to: .waitingForDataAfterSeek) @@ -437,6 +434,11 @@ final class AudioFileStreamProcessor { entry.framesState.primingFrames = Int(pti.mPrimingFrames) entry.framesState.validFrames = Int(pti.mNumberValidFrames) entry.framesState.remainderFrames = Int(pti.mRemainderFrames) + // Frames decoded before this info arrived were kept untrimmed, so the decoded position can't + // be used to trim the start; fall back to dropping only the padding at the end of the stream. + if let decoded = entry.framesState.decoded, decoded > 0 { + entry.framesState.decoded = nil + } } } @@ -635,6 +637,8 @@ final class AudioFileStreamProcessor { fileStreamCallback?(.raiseError(.codecError)) return } + // the requested frames were all filled: account for them before decoding more + fillUsedFrames(framesCount: framesAdded) } else { var framesAdded: UInt32 = 0 var framesToDecode: UInt32 = start - end @@ -666,6 +670,8 @@ final class AudioFileStreamProcessor { fileStreamCallback?(.raiseError(.codecError)) return } + // the requested frames were all filled: account for them before decoding more + fillUsedFrames(framesCount: framesAdded) } } } @@ -713,14 +719,11 @@ final class AudioFileStreamProcessor { guard let decodedStart, let validFrames, sourceSampleRate > 0 else { return framesCount } let ratio = outputAudioFormat.mSampleRate / sourceSampleRate - let keepFrom = Int((Double(primingFrames) * ratio).rounded()) - let keepTo = keepFrom + Int((Double(validFrames) * ratio).rounded()) - let first = max(decodedStart, keepFrom) - let last = min(decodedStart + Int(framesCount), keepTo) - guard last > first else { return 0 } - - let leadingFrames = first - decodedStart - let keptFrames = last - first + let (leadingFrames, keptFrames) = GaplessTrim.keptRange( + decodedStart: decodedStart, framesCount: Int(framesCount), + primingFrames: primingFrames, validFrames: validFrames, ratio: ratio + ) + guard keptFrames > 0 else { return 0 } if leadingFrames > 0, let mData = rendererContext.audioBuffer.mData { let frameSize = Int(rendererContext.bufferContext.sizeInBytes) let destination = mData + Int(bufferIndex) * frameSize @@ -733,8 +736,9 @@ final class AudioFileStreamProcessor { /// because the decoded position is not exact after a seek. /// Must be called once all packets of the reading entry have been decoded: the padding is the /// last part written to the renderer buffer then. - func trimTrailingGaplessPadding() { - guard let entry = playerContext.audioReadingEntry else { return } + /// + /// - parameter entry: The entry whose stream just ended + func trimTrailingGaplessPadding(for entry: AudioEntry) { // same lock order as the renderer: buffer first, then entry rendererContext.lock.lock(); defer { rendererContext.lock.unlock() } entry.lock.lock(); defer { entry.lock.unlock() } @@ -742,9 +746,12 @@ final class AudioFileStreamProcessor { let framesState = entry.framesState let sourceSampleRate = entry.audioStreamFormat.mSampleRate guard framesState.decoded == nil, framesState.remainderFrames > 0, sourceSampleRate > 0 else { return } - let outputRemainderFrames = Int((Double(framesState.remainderFrames) * outputAudioFormat.mSampleRate / sourceSampleRate).rounded()) - let notPlayedFrames = framesState.queued - framesState.played - let droppedFrames = min(outputRemainderFrames, notPlayedFrames, Int(rendererContext.bufferContext.frameUsedCount)) + let droppedFrames = GaplessTrim.trailingFramesToDrop( + remainderFrames: framesState.remainderFrames, + ratio: outputAudioFormat.mSampleRate / sourceSampleRate, + notPlayedFrames: framesState.queued - framesState.played, + usedFrames: Int(rendererContext.bufferContext.frameUsedCount) + ) guard droppedFrames > 0 else { return } rendererContext.bufferContext.frameUsedCount -= UInt32(droppedFrames) diff --git a/AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift b/AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift index 126b94e..4667e4d 100644 --- a/AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift +++ b/AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift @@ -8,6 +8,14 @@ import XCTest @testable import AudioStreaming class AudioEntryTests: XCTestCase { + func test_Reset_Marks_Decoded_Position_Exact_Only_When_Asked() { + let entry = audioEntry(length: 100) + entry.reset() + XCTAssertEqual(entry.framesState.decoded, 0) + entry.reset(decodedPositionExact: false) + XCTAssertNil(entry.framesState.decoded) + } + func test_AudioDataLengthBytes_Is_Zero_When_DataOffset_Exceeds_Length() { // a live stream can report a length smaller than the parsed header offset let entry = audioEntry(length: 100) diff --git a/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift b/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift new file mode 100644 index 0000000..6392387 --- /dev/null +++ b/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift @@ -0,0 +1,74 @@ +// +// GaplessTrimTests.swift +// AudioStreamingTests +// + +import XCTest +@testable import AudioStreaming + +class GaplessTrimTests: XCTestCase { + private func range(_ start: Int, _ count: Int, priming: Int = 2112, valid: Int = 10000, ratio: Double = 1) -> (leading: Int, kept: Int) { + GaplessTrim.keptRange(decodedStart: start, framesCount: count, primingFrames: priming, validFrames: valid, ratio: ratio) + } + + func test_Chunk_Inside_Priming_Is_Dropped() { + let r = range(0, 1024) + XCTAssertEqual(r.kept, 0) + } + + func test_Chunk_Straddling_Priming_Drops_Leading_Frames() { + let r = range(2048, 1024) + XCTAssertEqual(r.leading, 64) + XCTAssertEqual(r.kept, 960) + } + + func test_Chunk_Inside_Valid_Range_Is_Kept() { + let r = range(4096, 1024) + XCTAssertEqual(r.leading, 0) + XCTAssertEqual(r.kept, 1024) + } + + func test_Chunk_Straddling_End_Drops_Trailing_Frames() { + // valid range ends at 2112 + 10000 = 12112 + let r = range(11776, 1024) + XCTAssertEqual(r.leading, 0) + XCTAssertEqual(r.kept, 336) + } + + func test_Chunk_After_Valid_Range_Is_Dropped() { + XCTAssertEqual(range(12112, 1024).kept, 0) + } + + func test_Chunk_Containing_Whole_Valid_Range() { + let r = range(0, 20000, priming: 100, valid: 500) + XCTAssertEqual(r.leading, 100) + XCTAssertEqual(r.kept, 500) + } + + func test_Kept_Frames_Over_Chunks_Equal_Valid_Frames() { + var decoded = 0 + var total = 0 + for count in [1024, 1024, 777, 4096, 1, 9000, 1024] { + total += range(decoded, count).kept + decoded += count + } + XCTAssertEqual(total, 10000) + } + + func test_Ratio_Scales_Priming_And_Valid() { + let r = range(0, 20000, priming: 1000, valid: 4000, ratio: 48000.0 / 44100.0) + XCTAssertEqual(r.leading, 1088) + XCTAssertEqual(r.kept, 4354) + } + + func test_Trailing_Drop_Is_Limited_By_Unplayed_And_Used_Frames() { + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 10000, usedFrames: 10000), 500) + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 200, usedFrames: 10000), 200) + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 10000, usedFrames: 100), 100) + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: -5, usedFrames: 100), 0) + } + + func test_Trailing_Drop_Scales_With_Ratio() { + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 1000, ratio: 2, notPlayedFrames: 5000, usedFrames: 5000), 2000) + } +} From e13d3b9aae59b828d5e6738993e482a4519bf2cd Mon Sep 17 00:00:00 2001 From: Chrizz Gelbach Date: Thu, 15 Oct 2026 10:37:01 +0200 Subject: [PATCH 3/4] Keep gapless start trimming on seek to start and stabilise trim values - Keep the decoded position exact when seeking to the start of the data - Ignore repeated packet table info once the gapless values are set - Round the end of the kept range once to avoid an off-by-one with resampling --- .../Streaming/Audio Entry/Models/EntryFrames.swift | 2 +- .../Processors/AudioFileStreamProcessor.swift | 7 +++++-- .../Streaming/Audio Entry/GaplessTrimTests.swift | 13 +++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift index db2d2c3..617a641 100644 --- a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift +++ b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift @@ -30,7 +30,7 @@ enum GaplessTrim { /// - Returns: The frames to skip at the start of the chunk and the frames to keep after them static func keptRange(decodedStart: Int, framesCount: Int, primingFrames: Int, validFrames: Int, ratio: Double) -> (leading: Int, kept: Int) { let keepFrom = Int((Double(primingFrames) * ratio).rounded()) - let keepTo = keepFrom + Int((Double(validFrames) * ratio).rounded()) + let keepTo = Int((Double(primingFrames + validFrames) * ratio).rounded()) let first = max(decodedStart, keepFrom) let last = min(decodedStart + framesCount, keepTo) guard last > first else { return (0, 0) } diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift index d516bfa..c1689d6 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift @@ -209,7 +209,9 @@ final class AudioFileStreamProcessor { AudioConverterReset(converted) } - readingEntry.reset(decodedPositionExact: false) + // seeking to the start of the data re-decodes the encoder delay, so the decoded position stays exact + let seeksToStart = seekByteOffset <= Int64(readingEntry.audioStreamState.dataOffset) + readingEntry.reset(decodedPositionExact: seeksToStart) readingEntry.seek(at: Int(seekByteOffset)) rendererContext.waitingForDataAfterSeekFrameCount.write { $0 = 0 } playerContext.setInternalState(to: .waitingForDataAfterSeek) @@ -429,7 +431,8 @@ final class AudioFileStreamProcessor { guard status == noErr else { return } // Use valid frames to refine duration if present entry.lock.lock(); defer { entry.lock.unlock() } - if pti.mNumberValidFrames > 0 { + // the property can be reported again after a seek; keep the values from the first report + if pti.mNumberValidFrames > 0, entry.framesState.validFrames == nil { entry.audioStreamState.dataPacketCount = Double(pti.mNumberValidFrames) / Double(max(1, entry.audioStreamFormat.mFramesPerPacket)) entry.framesState.primingFrames = Int(pti.mPrimingFrames) entry.framesState.validFrames = Int(pti.mNumberValidFrames) diff --git a/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift b/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift index 6392387..ff4915d 100644 --- a/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift +++ b/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift @@ -61,6 +61,19 @@ class GaplessTrimTests: XCTestCase { XCTAssertEqual(r.kept, 4354) } + func test_Kept_Frames_Over_Chunks_Match_Rounded_End_With_Ratio() { + let ratio = 48000.0 / 44100.0 + var decoded = 0 + var total = 0 + for count in [1000, 1024, 777, 4096, 1, 9000, 1024] { + let r = range(decoded, count, priming: 1105, valid: 7003, ratio: ratio) + total += r.kept + decoded += count + } + // keepFrom = round(1105 * ratio), keepTo = round(8108 * ratio) + XCTAssertEqual(total, Int((8108 * ratio).rounded()) - Int((1105 * ratio).rounded())) + } + func test_Trailing_Drop_Is_Limited_By_Unplayed_And_Used_Frames() { XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 10000, usedFrames: 10000), 500) XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 200, usedFrames: 10000), 200) From 59e1e7ea81a0af303993916f939e096ef5cf1bf9 Mon Sep 17 00:00:00 2001 From: Chrizz Gelbach Date: Wed, 28 Oct 2026 20:28:56 +0100 Subject: [PATCH 4/4] Trim the encoder delay and padding of MP3 streams without gapless info An MP3 without Xing/Info header reports no packet table info: a transcode of a server (live or from its cache), or a file of a simple encoder. The decoder then outputs the encoder delay at the start and the padding at the end as audio, so two entries are separated by a gap of about 35 ms, which is a tick in a continuous mix. The trimming added for AAC and tagged MP3 does not apply. - Assume the encoder delay of LAME (576 frames, the decoder delay is already compensated by AudioToolbox) for an MP3 that reports no packet table info when it is ready to produce packets. The packet table info of a file with header is reported before and takes precedence. - The padding is unknown. At the end of the stream drop the frames at the end of the renderer buffer that are below 20 percent of the peak level of the music before them (at most one frame plus the encoder delay, and only frames that are not played yet). The codec rings for a few dozen frames after music that ends abruptly, so a fixed silence threshold is not enough. Measured with the end of one track and the start of the next one of a DJ mix (M4A transcoded to MP3 by Navidrome, with and without its transcoding cache), played back to back and compared sample by sample with the lossless source: gap at the transition 1535 frames (cached) / 659 frames (live) -> 0 frames, on macOS and on an iPhone 15 Pro. A sine cut in two parts, MP3 files with and without Xing header, and RAW: no gap, no phase jump. --- .../Audio Entry/Models/EntryFrames.swift | 42 ++++++++++++ .../Processors/AudioFileStreamProcessor.swift | 68 ++++++++++++++++--- .../Audio Entry/GaplessTrimTests.swift | 51 ++++++++++++++ 3 files changed, 153 insertions(+), 8 deletions(-) diff --git a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift index 617a641..ea0675c 100644 --- a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift +++ b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift @@ -3,6 +3,7 @@ // Copyright © 2020 Decimal. All rights reserved. // +import AudioToolbox import Foundation final class EntryFramesState { @@ -37,6 +38,47 @@ enum GaplessTrim { return (first - decodedStart, last - first) } + /// Encoder delay of LAME, also used by ffmpeg's libmp3lame. The decoder delay (529) is already + /// compensated by the AudioToolbox MP3 decoder. + static let lameMp3PrimingFrames = 576 + + /// The priming frames to assume for a stream without packet table info. + /// An MP3 without Xing/Info header (a live transcode, a cached transcode of a server, a file of a + /// simple encoder) gives no gapless info, so the decoder outputs the encoder delay as silence. + static func assumedPrimingFrames(formatID: AudioFormatID, hasPacketTableInfo: Bool) -> Int { + guard !hasPacketTableInfo, formatID == kAudioFormatMPEGLayer3 else { return 0 } + return lameMp3PrimingFrames + } + + /// The most end padding an MP3 encoder adds (one frame plus the encoder delay), in source frames. + static let maxAssumedMp3PaddingFrames = 1152 + 576 + + /// A frame at the end of the stream counts as padding below this fraction (-14 dB) of the peak level + /// of the music before it: the encoder padding is silence plus the ringing of the codec, which is + /// about 20 dB below a music that ends abruptly and far below the music that follows. At least + /// `paddingSilenceMinimum`. + static let paddingRelativeThreshold: Float = 0.2 + static let paddingSilenceMinimum: Float = 0.001 + + /// The frames at the end of a stream without padding info that are silent or decaying + /// compared to the music before them, to be dropped as padding. + /// - parameter limit: The most frames to drop + /// - parameter referenceFrames: The number of frames before the dropped ones to take the music level from + /// - parameter peakFromEnd: The peak level of the frame `n` frames before the end (0 is the last frame) + static func trailingSilentFrames(limit: Int, referenceFrames: Int, peakFromEnd: (Int) -> Float) -> Int { + guard limit > 0 else { return 0 } + var reference: Float = 0 + for n in limit ..< limit + max(0, referenceFrames) { + reference = max(reference, peakFromEnd(n)) + } + let threshold = max(paddingSilenceMinimum, reference * paddingRelativeThreshold) + var count = 0 + while count < limit, peakFromEnd(count) < threshold { + count += 1 + } + return count + } + /// The output frames of end padding to drop, limited to what is still unplayed and buffered. static func trailingFramesToDrop(remainderFrames: Int, ratio: Double, notPlayedFrames: Int, usedFrames: Int) -> Int { let outputRemainder = Int((Double(remainderFrames) * ratio).rounded()) diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift index c1689d6..7920cb2 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift @@ -308,6 +308,7 @@ final class AudioFileStreamProcessor { assignMagicCookieToConverterIfNeeded() processPacketUpperBoundAndMaxPacketSize(entry: entry, fileStream: fileStream) processReadyToProducePackets(entry: entry, fileStream: fileStream) + applyAssumedPrimingIfNeeded(entry: entry) case kAudioFileStreamProperty_FormatList: processFormatList(entry: entry, fileStream: fileStream) case kAudioFileStreamProperty_PacketTableInfo: @@ -423,6 +424,17 @@ final class AudioFileStreamProcessor { entry.audioStreamState.bitRate = Double(bitRate) } + /// An MP3 without packet table info (no Xing/Info header, e.g. a transcode of a server or a file + /// of a simple encoder) has the encoder delay of LAME at its start. The packet table info of a file + /// with header is reported before the stream is ready to produce packets and takes precedence. + private func applyAssumedPrimingIfNeeded(entry: AudioEntry) { + entry.lock.lock(); defer { entry.lock.unlock() } + guard entry.framesState.validFrames == nil, entry.framesState.primingFrames == 0 else { return } + entry.framesState.primingFrames = GaplessTrim.assumedPrimingFrames( + formatID: entry.audioStreamFormat.mFormatID, hasPacketTableInfo: false + ) + } + private func processPacketTableInfo(entry: AudioEntry, fileStream: AudioFileStreamID) { var pti = AudioFilePacketTableInfo(mNumberValidFrames: 0, mPrimingFrames: 0, @@ -716,7 +728,8 @@ final class AudioFileStreamProcessor { framesState.decoded = decodedStart + Int(framesCount) } let primingFrames = framesState.primingFrames - let validFrames = framesState.validFrames + // without packet table info only the assumed priming at the start is dropped + let validFrames = framesState.validFrames ?? (primingFrames > 0 ? Int(Int32.max) : nil) let sourceSampleRate = entry.audioStreamFormat.mSampleRate entry.lock.unlock() @@ -748,19 +761,58 @@ final class AudioFileStreamProcessor { let framesState = entry.framesState let sourceSampleRate = entry.audioStreamFormat.mSampleRate - guard framesState.decoded == nil, framesState.remainderFrames > 0, sourceSampleRate > 0 else { return } - let droppedFrames = GaplessTrim.trailingFramesToDrop( - remainderFrames: framesState.remainderFrames, - ratio: outputAudioFormat.mSampleRate / sourceSampleRate, - notPlayedFrames: framesState.queued - framesState.played, - usedFrames: Int(rendererContext.bufferContext.frameUsedCount) - ) + guard sourceSampleRate > 0 else { return } + let ratio = outputAudioFormat.mSampleRate / sourceSampleRate + let notPlayedFrames = framesState.queued - framesState.played + let usedFrames = Int(rendererContext.bufferContext.frameUsedCount) + let droppedFrames: Int + if framesState.remainderFrames > 0 { + guard framesState.decoded == nil else { return } + droppedFrames = GaplessTrim.trailingFramesToDrop( + remainderFrames: framesState.remainderFrames, + ratio: ratio, + notPlayedFrames: notPlayedFrames, + usedFrames: usedFrames + ) + } else if framesState.validFrames == nil, framesState.primingFrames > 0 { + // padding of an MP3 without packet table info is unknown: drop the silence at the end + let limit = max(0, min( + Int((Double(GaplessTrim.maxAssumedMp3PaddingFrames) * ratio).rounded()), + notPlayedFrames, + usedFrames + )) + droppedFrames = trailingSilentFrames(limit: limit, usedFrames: usedFrames) + } else { + return + } guard droppedFrames > 0 else { return } rendererContext.bufferContext.frameUsedCount -= UInt32(droppedFrames) framesState.queued -= droppedFrames } + /// Counts the silent frames at the end of the used renderer buffer. + /// The renderer buffer holds interleaved 32 bit float frames (the canonical output format of the player). + /// Must be called with the renderer lock held. + private func trailingSilentFrames(limit: Int, usedFrames: Int) -> Int { + guard limit > 0, let mData = rendererContext.audioBuffer.mData else { return 0 } + let bufferContext = rendererContext.bufferContext + let total = Int(bufferContext.totalFrameCount) + let end = Int(bufferContext.end) + let channels = max(1, Int(outputAudioFormat.mChannelsPerFrame)) + let samples = mData.assumingMemoryBound(to: Float.self) + // the music level comes from the frames before the ones that may be dropped + let referenceFrames = max(0, min(4096, usedFrames - limit)) + return GaplessTrim.trailingSilentFrames(limit: limit, referenceFrames: referenceFrames) { n in + let frame = (end - 1 - n + total) % total + var peak: Float = 0 + for channel in 0 ..< channels { + peak = max(peak, abs(samples[frame * channels + channel])) + } + return peak + } + } + /// Advances the processed frames for buffer and reading entry /// /// - parameter frameCount: An `UInt32` value to be added to the used count of the buffers. diff --git a/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift b/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift index ff4915d..fe0bf3f 100644 --- a/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift +++ b/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift @@ -3,6 +3,7 @@ // AudioStreamingTests // +import AudioToolbox import XCTest @testable import AudioStreaming @@ -84,4 +85,54 @@ class GaplessTrimTests: XCTestCase { func test_Trailing_Drop_Scales_With_Ratio() { XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 1000, ratio: 2, notPlayedFrames: 5000, usedFrames: 5000), 2000) } + + func test_Assumed_Priming_For_Mp3_Without_Packet_Table_Info() { + XCTAssertEqual(GaplessTrim.assumedPrimingFrames(formatID: kAudioFormatMPEGLayer3, hasPacketTableInfo: false), 576) + } + + func test_No_Assumed_Priming_For_Mp3_With_Packet_Table_Info() { + XCTAssertEqual(GaplessTrim.assumedPrimingFrames(formatID: kAudioFormatMPEGLayer3, hasPacketTableInfo: true), 0) + } + + func test_No_Assumed_Priming_For_Other_Formats() { + XCTAssertEqual(GaplessTrim.assumedPrimingFrames(formatID: kAudioFormatMPEG4AAC, hasPacketTableInfo: false), 0) + } + + func test_Trailing_Silent_Frames_Counts_Until_Signal() { + let samples: [Float] = [0.5, 0.4, 0, 0.0005, 0, 0] // oldest to newest + let count = GaplessTrim.trailingSilentFrames(limit: 100, referenceFrames: 0) { samples[samples.count - 1 - $0] } + XCTAssertEqual(count, 4) + } + + func test_Trailing_Silent_Frames_Is_Limited() { + let count = GaplessTrim.trailingSilentFrames(limit: 3, referenceFrames: 0) { _ in 0 } + XCTAssertEqual(count, 3) + } + + func test_Trailing_Silent_Frames_Zero_For_Signal_At_End() { + let count = GaplessTrim.trailingSilentFrames(limit: 100, referenceFrames: 0) { _ in 0.2 } + XCTAssertEqual(count, 0) + } + + func test_Trailing_Decay_Is_Dropped_Relative_To_Loud_Music() { + // loud music (0.5), then a decaying tail below 2 % of it + let samples: [Float] = Array(repeating: 0.5, count: 50) + [0.008, 0.006, 0.004, 0.003] + let count = GaplessTrim.trailingSilentFrames(limit: 4, referenceFrames: 40) { samples[samples.count - 1 - $0] } + XCTAssertEqual(count, 4) + } + + func test_Trailing_Quiet_Music_Is_Kept() { + // quiet music (0.02): 2 % of it is below the minimum, so only really silent frames are dropped + let samples: [Float] = Array(repeating: 0.02, count: 50) + [0.004, 0.004] + let count = GaplessTrim.trailingSilentFrames(limit: 2, referenceFrames: 40) { samples[samples.count - 1 - $0] } + XCTAssertEqual(count, 0) + } + + func test_Ringing_After_An_Abrupt_Music_End_Is_Dropped() { + // music at 0.4, then the ringing of the codec at about 0.04 and a noise floor, as measured on a live stream + let tail: [Float] = [0.044, 0.010, 0.041, 0.010, 0.036, 0.024, 0.006, 0.004] + let samples: [Float] = Array(repeating: 0.4, count: 50) + tail + let count = GaplessTrim.trailingSilentFrames(limit: 20, referenceFrames: 30) { samples[samples.count - 1 - $0] } + XCTAssertEqual(count, tail.count) + } }