Drop encoder priming and padding for gapless playback

AudioStreaming/Streaming/Audio Entry/AudioEntry.swift

@@ -95,11 +95,14 @@ class AudioEntry {
         source.seek(at: offset)
     }
 
-    func reset() {
+    /// - parameter decodedPositionExact: Whether the entry is re-read from the start of the stream.
+    ///   Pass `false` when re-reading from a seek offset, so gapless trimming doesn't treat mid-stream data as the start.
+    func reset(decodedPositionExact: Bool = true) {
         lock.lock(); defer { lock.unlock() }
         framesState.played = 0
         framesState.queued = 0
         framesState.lastFrameQueued = -1
+        framesState.decoded = decodedPositionExact ? 0 : nil
     }
 
     func has(same source: CoreAudioStreamSource) -> Bool {

AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift

@@ -9,4 +9,37 @@ final class EntryFramesState {
     var queued: Int = 0
     var played: Int = 0
     var lastFrameQueued: Int = -1
+
+    /// Gapless info from the packet table, in source frames: encoder delay (priming), audio frames
+    /// (valid) and padding (remainder). Decoders output priming and remainder like audio; both are
+    /// dropped. `validFrames` is nil when the stream has no packet table info.
+    var primingFrames: Int = 0
+    var validFrames: Int?
+    var remainderFrames: Int = 0
+    /// Output frames decoded since the start of the stream, including dropped ones.
+    /// nil after a seek, where the decoded position is not exact.
+    var decoded: Int? = 0
+}
+
+/// Pure gapless trimming math, in output frames.
+enum GaplessTrim {
+    /// The part of a decoded chunk to keep.
+    /// - parameter decodedStart: Output frames decoded before this chunk
+    /// - parameter framesCount: Frames in this chunk
+    /// - parameter ratio: Output sample rate / source sample rate
+    /// - Returns: The frames to skip at the start of the chunk and the frames to keep after them
+    static func keptRange(decodedStart: Int, framesCount: Int, primingFrames: Int, validFrames: Int, ratio: Double) -> (leading: Int, kept: Int) {
+        let keepFrom = Int((Double(primingFrames) * ratio).rounded())
+        let keepTo = Int((Double(primingFrames + validFrames) * ratio).rounded())
+        let first = max(decodedStart, keepFrom)
+        let last = min(decodedStart + framesCount, keepTo)
+        guard last > first else { return (0, 0) }
+        return (first - decodedStart, last - first)
+    }
+
+    /// The output frames of end padding to drop, limited to what is still unplayed and buffered.
+    static func trailingFramesToDrop(remainderFrames: Int, ratio: Double, notPlayedFrames: Int, usedFrames: Int) -> Int {
+        let outputRemainder = Int((Double(remainderFrames) * ratio).rounded())
+        return max(0, min(outputRemainder, notPlayedFrames, usedFrames))
+    }
 }

AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift

@@ -730,7 +730,7 @@ open class AudioPlayer {
                   playingEntry != snapshot.reading
         {
             playingEntry.audioStreamState.processedDataFormat = false
-            playingEntry.reset()
+            playingEntry.reset(decodedPositionExact: false)
             if let readingEntry = snapshot.reading {
                 readingEntry.delegate = nil
                 readingEntry.close()
@@ -1091,6 +1091,8 @@ extension AudioPlayer: AudioStreamSourceDelegate {
             Logger.debug("Failed to flush remaining packets at EOF: \(flushStatus)", category: .generic)
         }
 
+        fileStreamProcessor.trimTrailingGaplessPadding(for: readingEntry)
+
         readingEntry.lock.lock()
         readingEntry.framesState.lastFrameQueued = readingEntry.framesState.queued
         readingEntry.lock.unlock()

AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift

@@ -209,7 +209,9 @@ final class AudioFileStreamProcessor {
             AudioConverterReset(converted)
         }
 
-        readingEntry.reset()
+        // seeking to the start of the data re-decodes the encoder delay, so the decoded position stays exact
+        let seeksToStart = seekByteOffset <= Int64(readingEntry.audioStreamState.dataOffset)
+        readingEntry.reset(decodedPositionExact: seeksToStart)
         readingEntry.seek(at: Int(seekByteOffset))
         rendererContext.waitingForDataAfterSeekFrameCount.write { $0 = 0 }
         playerContext.setInternalState(to: .waitingForDataAfterSeek)
@@ -429,8 +431,17 @@ final class AudioFileStreamProcessor {
         guard status == noErr else { return }
         // Use valid frames to refine duration if present
         entry.lock.lock(); defer { entry.lock.unlock() }
-        if pti.mNumberValidFrames > 0 {
+        // the property can be reported again after a seek; keep the values from the first report
+        if pti.mNumberValidFrames > 0, entry.framesState.validFrames == nil {
             entry.audioStreamState.dataPacketCount = Double(pti.mNumberValidFrames) / Double(max(1, entry.audioStreamFormat.mFramesPerPacket))
+            entry.framesState.primingFrames = Int(pti.mPrimingFrames)
+            entry.framesState.validFrames = Int(pti.mNumberValidFrames)
+            entry.framesState.remainderFrames = Int(pti.mRemainderFrames)
+            // Frames decoded before this info arrived were kept untrimmed, so the decoded position can't
+            // be used to trim the start; fall back to dropping only the padding at the end of the stream.
+            if let decoded = entry.framesState.decoded, decoded > 0 {
+                entry.framesState.decoded = nil
+            }
         }
     }
 
@@ -581,7 +592,7 @@ final class AudioFileStreamProcessor {
                     nil
                 )
 
-                framesAdded = framesToDecode
+                framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode)
 
                 if status == AudioConvertStatus.done.rawValue {
                     fillUsedFrames(framesCount: framesAdded)
@@ -591,6 +602,12 @@ final class AudioFileStreamProcessor {
                     return
                 }
 
+                if framesAdded < framesToDecode {
+                    // dropped frames left a hole before the end of the buffer, continue right after the kept ones
+                    fillUsedFrames(framesCount: framesAdded)
+                    continue packetProcess
+                }
+
                 framesToDecode = start
                 if framesToDecode == 0 {
                     fillUsedFrames(framesCount: framesAdded)
@@ -611,7 +628,7 @@ final class AudioFileStreamProcessor {
                     nil
                 )
 
-                framesAdded += framesToDecode
+                framesAdded += trimGaplessPadding(bufferIndex: 0, framesCount: framesToDecode)
 
                 if status == AudioConvertStatus.done.rawValue {
                     fillUsedFrames(framesCount: framesAdded)
@@ -623,6 +640,8 @@ final class AudioFileStreamProcessor {
                     fileStreamCallback?(.raiseError(.codecError))
                     return
                 }
+                // the requested frames were all filled: account for them before decoding more
+                fillUsedFrames(framesCount: framesAdded)
             } else {
                 var framesAdded: UInt32 = 0
                 var framesToDecode: UInt32 = start - end
@@ -643,7 +662,7 @@ final class AudioFileStreamProcessor {
                     nil
                 )
 
-                framesAdded = framesToDecode
+                framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode)
                 if status == AudioConvertStatus.done.rawValue {
                     fillUsedFrames(framesCount: framesAdded)
                     return
@@ -654,6 +673,8 @@ final class AudioFileStreamProcessor {
                     fileStreamCallback?(.raiseError(.codecError))
                     return
                 }
+                // the requested frames were all filled: account for them before decoding more
+                fillUsedFrames(framesCount: framesAdded)
             }
         }
     }
@@ -676,6 +697,70 @@ final class AudioFileStreamProcessor {
         bufferList[0].mNumberChannels = rendererContext.audioBuffer.mNumberChannels
     }
 
+    /// Drops the encoder delay (priming) and padding (remainder) that decoders output for formats like
+    /// AAC and MP3, using the packet table info of the stream. Without this, consecutive entries are
+    /// separated by a short silence (about 50 ms for AAC).
+    /// Keeps the decoded frames in [priming, priming + valid) while the decoded position is exact;
+    /// after a seek, `trimTrailingGaplessPadding()` drops the padding at the end of the stream instead.
+    ///
+    /// - parameter bufferIndex: The frame index in the renderer buffer where the decoded frames start
+    /// - parameter framesCount: The number of frames decoded there
+    /// - Returns: The number of frames to keep; they are moved to `bufferIndex`
+    @inline(__always)
+    private func trimGaplessPadding(bufferIndex: UInt32, framesCount: UInt32) -> UInt32 {
+        guard framesCount > 0, let entry = playerContext.audioReadingEntry else { return framesCount }
+        entry.lock.lock()
+        let framesState = entry.framesState
+        let decodedStart = framesState.decoded
+        if let decodedStart {
+            framesState.decoded = decodedStart + Int(framesCount)
+        }
+        let primingFrames = framesState.primingFrames
+        let validFrames = framesState.validFrames
+        let sourceSampleRate = entry.audioStreamFormat.mSampleRate
+        entry.lock.unlock()
+
+        guard let decodedStart, let validFrames, sourceSampleRate > 0 else { return framesCount }
+        let ratio = outputAudioFormat.mSampleRate / sourceSampleRate
+        let (leadingFrames, keptFrames) = GaplessTrim.keptRange(
+            decodedStart: decodedStart, framesCount: Int(framesCount),
+            primingFrames: primingFrames, validFrames: validFrames, ratio: ratio
+        )
+        guard keptFrames > 0 else { return 0 }
+        if leadingFrames > 0, let mData = rendererContext.audioBuffer.mData {
+            let frameSize = Int(rendererContext.bufferContext.sizeInBytes)
+            let destination = mData + Int(bufferIndex) * frameSize
+            memmove(destination, destination + leadingFrames * frameSize, keptFrames * frameSize)
+        }
+        return UInt32(keptFrames)
+    }
+
+    /// Drops the encoder padding (remainder) at the end of the stream when `trimGaplessPadding` could not,
+    /// because the decoded position is not exact after a seek.
+    /// Must be called once all packets of the reading entry have been decoded: the padding is the
+    /// last part written to the renderer buffer then.
+    ///
+    /// - parameter entry: The entry whose stream just ended
+    func trimTrailingGaplessPadding(for entry: AudioEntry) {
+        // same lock order as the renderer: buffer first, then entry
+        rendererContext.lock.lock(); defer { rendererContext.lock.unlock() }
+        entry.lock.lock(); defer { entry.lock.unlock() }
+
+        let framesState = entry.framesState
+        let sourceSampleRate = entry.audioStreamFormat.mSampleRate
+        guard framesState.decoded == nil, framesState.remainderFrames > 0, sourceSampleRate > 0 else { return }
+        let droppedFrames = GaplessTrim.trailingFramesToDrop(
+            remainderFrames: framesState.remainderFrames,
+            ratio: outputAudioFormat.mSampleRate / sourceSampleRate,
+            notPlayedFrames: framesState.queued - framesState.played,
+            usedFrames: Int(rendererContext.bufferContext.frameUsedCount)
+        )
+        guard droppedFrames > 0 else { return }
+
+        rendererContext.bufferContext.frameUsedCount -= UInt32(droppedFrames)
+        framesState.queued -= droppedFrames
+    }
+
     /// Advances the processed frames for buffer and reading entry
     ///
     /// - parameter frameCount: An `UInt32` value to be added to the used count of the buffers.

AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift

@@ -8,6 +8,14 @@ import XCTest
 @testable import AudioStreaming
 
 class AudioEntryTests: XCTestCase {
+    func test_Reset_Marks_Decoded_Position_Exact_Only_When_Asked() {
+        let entry = audioEntry(length: 100)
+        entry.reset()
+        XCTAssertEqual(entry.framesState.decoded, 0)
+        entry.reset(decodedPositionExact: false)
+        XCTAssertNil(entry.framesState.decoded)
+    }
+
     func test_AudioDataLengthBytes_Is_Zero_When_DataOffset_Exceeds_Length() {
         // a live stream can report a length smaller than the parsed header offset
         let entry = audioEntry(length: 100)

AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift

@@ -0,0 +1,87 @@
+//
+//  GaplessTrimTests.swift
+//  AudioStreamingTests
+//
+
+import XCTest
+@testable import AudioStreaming
+
+class GaplessTrimTests: XCTestCase {
+    private func range(_ start: Int, _ count: Int, priming: Int = 2112, valid: Int = 10000, ratio: Double = 1) -> (leading: Int, kept: Int) {
+        GaplessTrim.keptRange(decodedStart: start, framesCount: count, primingFrames: priming, validFrames: valid, ratio: ratio)
+    }
+
+    func test_Chunk_Inside_Priming_Is_Dropped() {
+        let r = range(0, 1024)
+        XCTAssertEqual(r.kept, 0)
+    }
+
+    func test_Chunk_Straddling_Priming_Drops_Leading_Frames() {
+        let r = range(2048, 1024)
+        XCTAssertEqual(r.leading, 64)
+        XCTAssertEqual(r.kept, 960)
+    }
+
+    func test_Chunk_Inside_Valid_Range_Is_Kept() {
+        let r = range(4096, 1024)
+        XCTAssertEqual(r.leading, 0)
+        XCTAssertEqual(r.kept, 1024)
+    }
+
+    func test_Chunk_Straddling_End_Drops_Trailing_Frames() {
+        // valid range ends at 2112 + 10000 = 12112
+        let r = range(11776, 1024)
+        XCTAssertEqual(r.leading, 0)
+        XCTAssertEqual(r.kept, 336)
+    }
+
+    func test_Chunk_After_Valid_Range_Is_Dropped() {
+        XCTAssertEqual(range(12112, 1024).kept, 0)
+    }
+
+    func test_Chunk_Containing_Whole_Valid_Range() {
+        let r = range(0, 20000, priming: 100, valid: 500)
+        XCTAssertEqual(r.leading, 100)
+        XCTAssertEqual(r.kept, 500)
+    }
+
+    func test_Kept_Frames_Over_Chunks_Equal_Valid_Frames() {
+        var decoded = 0
+        var total = 0
+        for count in [1024, 1024, 777, 4096, 1, 9000, 1024] {
+            total += range(decoded, count).kept
+            decoded += count
+        }
+        XCTAssertEqual(total, 10000)
+    }
+
+    func test_Ratio_Scales_Priming_And_Valid() {
+        let r = range(0, 20000, priming: 1000, valid: 4000, ratio: 48000.0 / 44100.0)
+        XCTAssertEqual(r.leading, 1088)
+        XCTAssertEqual(r.kept, 4354)
+    }
+
+    func test_Kept_Frames_Over_Chunks_Match_Rounded_End_With_Ratio() {
+        let ratio = 48000.0 / 44100.0
+        var decoded = 0
+        var total = 0
+        for count in [1000, 1024, 777, 4096, 1, 9000, 1024] {
+            let r = range(decoded, count, priming: 1105, valid: 7003, ratio: ratio)
+            total += r.kept
+            decoded += count
+        }
+        // keepFrom = round(1105 * ratio), keepTo = round(8108 * ratio)
+        XCTAssertEqual(total, Int((8108 * ratio).rounded()) - Int((1105 * ratio).rounded()))
+    }
+
+    func test_Trailing_Drop_Is_Limited_By_Unplayed_And_Used_Frames() {
+        XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 10000, usedFrames: 10000), 500)
+        XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 200, usedFrames: 10000), 200)
+        XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 10000, usedFrames: 100), 100)
+        XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: -5, usedFrames: 100), 0)
+    }
+
+    func test_Trailing_Drop_Scales_With_Ratio() {
+        XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 1000, ratio: 2, notPlayedFrames: 5000, usedFrames: 5000), 2000)
+    }
+}