Trim the encoder delay and padding of MP3 streams without gapless info

AudioStreaming/Streaming/Audio Entry/AudioEntry.swift

@@ -95,11 +95,14 @@ class AudioEntry {
         source.seek(at: offset)
     }
 
-    func reset() {
+    /// - parameter decodedPositionExact: Whether the entry is re-read from the start of the stream.
+    ///   Pass `false` when re-reading from a seek offset, so gapless trimming doesn't treat mid-stream data as the start.
+    func reset(decodedPositionExact: Bool = true) {
         lock.lock(); defer { lock.unlock() }
         framesState.played = 0
         framesState.queued = 0
         framesState.lastFrameQueued = -1
+        framesState.decoded = decodedPositionExact ? 0 : nil
     }
 
     func has(same source: CoreAudioStreamSource) -> Bool {

AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift

@@ -3,10 +3,85 @@
 //  Copyright © 2020 Decimal. All rights reserved.
 //
 
+import AudioToolbox
 import Foundation
 
 final class EntryFramesState {
     var queued: Int = 0
     var played: Int = 0
     var lastFrameQueued: Int = -1
+
+    /// Gapless info from the packet table, in source frames: encoder delay (priming), audio frames
+    /// (valid) and padding (remainder). Decoders output priming and remainder like audio; both are
+    /// dropped. `validFrames` is nil when the stream has no packet table info.
+    var primingFrames: Int = 0
+    var validFrames: Int?
+    var remainderFrames: Int = 0
+    /// Output frames decoded since the start of the stream, including dropped ones.
+    /// nil after a seek, where the decoded position is not exact.
+    var decoded: Int? = 0
+}
+
+/// Pure gapless trimming math, in output frames.
+enum GaplessTrim {
+    /// The part of a decoded chunk to keep.
+    /// - parameter decodedStart: Output frames decoded before this chunk
+    /// - parameter framesCount: Frames in this chunk
+    /// - parameter ratio: Output sample rate / source sample rate
+    /// - Returns: The frames to skip at the start of the chunk and the frames to keep after them
+    static func keptRange(decodedStart: Int, framesCount: Int, primingFrames: Int, validFrames: Int, ratio: Double) -> (leading: Int, kept: Int) {
+        let keepFrom = Int((Double(primingFrames) * ratio).rounded())
+        let keepTo = Int((Double(primingFrames + validFrames) * ratio).rounded())
+        let first = max(decodedStart, keepFrom)
+        let last = min(decodedStart + framesCount, keepTo)
+        guard last > first else { return (0, 0) }
+        return (first - decodedStart, last - first)
+    }
+
+    /// Encoder delay of LAME, also used by ffmpeg's libmp3lame. The decoder delay (529) is already
+    /// compensated by the AudioToolbox MP3 decoder.
+    static let lameMp3PrimingFrames = 576
+
+    /// The priming frames to assume for a stream without packet table info.
+    /// An MP3 without Xing/Info header (a live transcode, a cached transcode of a server, a file of a
+    /// simple encoder) gives no gapless info, so the decoder outputs the encoder delay as silence.
+    static func assumedPrimingFrames(formatID: AudioFormatID, hasPacketTableInfo: Bool) -> Int {
+        guard !hasPacketTableInfo, formatID == kAudioFormatMPEGLayer3 else { return 0 }
+        return lameMp3PrimingFrames
+    }
+
+    /// The most end padding an MP3 encoder adds (one frame plus the encoder delay), in source frames.
+    static let maxAssumedMp3PaddingFrames = 1152 + 576
+
+    /// A frame at the end of the stream counts as padding below this fraction (-14 dB) of the peak level
+    /// of the music before it: the encoder padding is silence plus the ringing of the codec, which is
+    /// about 20 dB below a music that ends abruptly and far below the music that follows. At least
+    /// `paddingSilenceMinimum`.
+    static let paddingRelativeThreshold: Float = 0.2
+    static let paddingSilenceMinimum: Float = 0.001
+
+    /// The frames at the end of a stream without padding info that are silent or decaying
+    /// compared to the music before them, to be dropped as padding.
+    /// - parameter limit: The most frames to drop
+    /// - parameter referenceFrames: The number of frames before the dropped ones to take the music level from
+    /// - parameter peakFromEnd: The peak level of the frame `n` frames before the end (0 is the last frame)
+    static func trailingSilentFrames(limit: Int, referenceFrames: Int, peakFromEnd: (Int) -> Float) -> Int {
+        guard limit > 0 else { return 0 }
+        var reference: Float = 0
+        for n in limit ..< limit + max(0, referenceFrames) {
+            reference = max(reference, peakFromEnd(n))
+        }
+        let threshold = max(paddingSilenceMinimum, reference * paddingRelativeThreshold)
+        var count = 0
+        while count < limit, peakFromEnd(count) < threshold {
+            count += 1
+        }
+        return count
+    }
+
+    /// The output frames of end padding to drop, limited to what is still unplayed and buffered.
+    static func trailingFramesToDrop(remainderFrames: Int, ratio: Double, notPlayedFrames: Int, usedFrames: Int) -> Int {
+        let outputRemainder = Int((Double(remainderFrames) * ratio).rounded())
+        return max(0, min(outputRemainder, notPlayedFrames, usedFrames))
+    }
 }

AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift

@@ -730,7 +730,7 @@ open class AudioPlayer {
                   playingEntry != snapshot.reading
         {
             playingEntry.audioStreamState.processedDataFormat = false
-            playingEntry.reset()
+            playingEntry.reset(decodedPositionExact: false)
             if let readingEntry = snapshot.reading {
                 readingEntry.delegate = nil
                 readingEntry.close()
@@ -1091,6 +1091,8 @@ extension AudioPlayer: AudioStreamSourceDelegate {
             Logger.debug("Failed to flush remaining packets at EOF: \(flushStatus)", category: .generic)
         }
 
+        fileStreamProcessor.trimTrailingGaplessPadding(for: readingEntry)
+
         readingEntry.lock.lock()
         readingEntry.framesState.lastFrameQueued = readingEntry.framesState.queued
         readingEntry.lock.unlock()

AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift

@@ -209,7 +209,9 @@ final class AudioFileStreamProcessor {
             AudioConverterReset(converted)
         }
 
-        readingEntry.reset()
+        // seeking to the start of the data re-decodes the encoder delay, so the decoded position stays exact
+        let seeksToStart = seekByteOffset <= Int64(readingEntry.audioStreamState.dataOffset)
+        readingEntry.reset(decodedPositionExact: seeksToStart)
         readingEntry.seek(at: Int(seekByteOffset))
         rendererContext.waitingForDataAfterSeekFrameCount.write { $0 = 0 }
         playerContext.setInternalState(to: .waitingForDataAfterSeek)
@@ -306,6 +308,7 @@ final class AudioFileStreamProcessor {
             assignMagicCookieToConverterIfNeeded()
             processPacketUpperBoundAndMaxPacketSize(entry: entry, fileStream: fileStream)
             processReadyToProducePackets(entry: entry, fileStream: fileStream)
+            applyAssumedPrimingIfNeeded(entry: entry)
         case kAudioFileStreamProperty_FormatList:
             processFormatList(entry: entry, fileStream: fileStream)
         case kAudioFileStreamProperty_PacketTableInfo:
@@ -421,6 +424,17 @@ final class AudioFileStreamProcessor {
         entry.audioStreamState.bitRate = Double(bitRate)
     }
 
+    /// An MP3 without packet table info (no Xing/Info header, e.g. a transcode of a server or a file
+    /// of a simple encoder) has the encoder delay of LAME at its start. The packet table info of a file
+    /// with header is reported before the stream is ready to produce packets and takes precedence.
+    private func applyAssumedPrimingIfNeeded(entry: AudioEntry) {
+        entry.lock.lock(); defer { entry.lock.unlock() }
+        guard entry.framesState.validFrames == nil, entry.framesState.primingFrames == 0 else { return }
+        entry.framesState.primingFrames = GaplessTrim.assumedPrimingFrames(
+            formatID: entry.audioStreamFormat.mFormatID, hasPacketTableInfo: false
+        )
+    }
+
     private func processPacketTableInfo(entry: AudioEntry, fileStream: AudioFileStreamID) {
         var pti = AudioFilePacketTableInfo(mNumberValidFrames: 0,
                                            mPrimingFrames: 0,
@@ -429,8 +443,17 @@ final class AudioFileStreamProcessor {
         guard status == noErr else { return }
         // Use valid frames to refine duration if present
         entry.lock.lock(); defer { entry.lock.unlock() }
-        if pti.mNumberValidFrames > 0 {
+        // the property can be reported again after a seek; keep the values from the first report
+        if pti.mNumberValidFrames > 0, entry.framesState.validFrames == nil {
             entry.audioStreamState.dataPacketCount = Double(pti.mNumberValidFrames) / Double(max(1, entry.audioStreamFormat.mFramesPerPacket))
+            entry.framesState.primingFrames = Int(pti.mPrimingFrames)
+            entry.framesState.validFrames = Int(pti.mNumberValidFrames)
+            entry.framesState.remainderFrames = Int(pti.mRemainderFrames)
+            // Frames decoded before this info arrived were kept untrimmed, so the decoded position can't
+            // be used to trim the start; fall back to dropping only the padding at the end of the stream.
+            if let decoded = entry.framesState.decoded, decoded > 0 {
+                entry.framesState.decoded = nil
+            }
         }
     }
 
@@ -581,7 +604,7 @@ final class AudioFileStreamProcessor {
                     nil
                 )
 
-                framesAdded = framesToDecode
+                framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode)
 
                 if status == AudioConvertStatus.done.rawValue {
                     fillUsedFrames(framesCount: framesAdded)
@@ -591,6 +614,12 @@ final class AudioFileStreamProcessor {
                     return
                 }
 
+                if framesAdded < framesToDecode {
+                    // dropped frames left a hole before the end of the buffer, continue right after the kept ones
+                    fillUsedFrames(framesCount: framesAdded)
+                    continue packetProcess
+                }
+
                 framesToDecode = start
                 if framesToDecode == 0 {
                     fillUsedFrames(framesCount: framesAdded)
@@ -611,7 +640,7 @@ final class AudioFileStreamProcessor {
                     nil
                 )
 
-                framesAdded += framesToDecode
+                framesAdded += trimGaplessPadding(bufferIndex: 0, framesCount: framesToDecode)
 
                 if status == AudioConvertStatus.done.rawValue {
                     fillUsedFrames(framesCount: framesAdded)
@@ -623,6 +652,8 @@ final class AudioFileStreamProcessor {
                     fileStreamCallback?(.raiseError(.codecError))
                     return
                 }
+                // the requested frames were all filled: account for them before decoding more
+                fillUsedFrames(framesCount: framesAdded)
             } else {
                 var framesAdded: UInt32 = 0
                 var framesToDecode: UInt32 = start - end
@@ -643,7 +674,7 @@ final class AudioFileStreamProcessor {
                     nil
                 )
 
-                framesAdded = framesToDecode
+                framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode)
                 if status == AudioConvertStatus.done.rawValue {
                     fillUsedFrames(framesCount: framesAdded)
                     return
@@ -654,6 +685,8 @@ final class AudioFileStreamProcessor {
                     fileStreamCallback?(.raiseError(.codecError))
                     return
                 }
+                // the requested frames were all filled: account for them before decoding more
+                fillUsedFrames(framesCount: framesAdded)
             }
         }
     }
@@ -676,6 +709,110 @@ final class AudioFileStreamProcessor {
         bufferList[0].mNumberChannels = rendererContext.audioBuffer.mNumberChannels
     }
 
+    /// Drops the encoder delay (priming) and padding (remainder) that decoders output for formats like
+    /// AAC and MP3, using the packet table info of the stream. Without this, consecutive entries are
+    /// separated by a short silence (about 50 ms for AAC).
+    /// Keeps the decoded frames in [priming, priming + valid) while the decoded position is exact;
+    /// after a seek, `trimTrailingGaplessPadding()` drops the padding at the end of the stream instead.
+    ///
+    /// - parameter bufferIndex: The frame index in the renderer buffer where the decoded frames start
+    /// - parameter framesCount: The number of frames decoded there
+    /// - Returns: The number of frames to keep; they are moved to `bufferIndex`
+    @inline(__always)
+    private func trimGaplessPadding(bufferIndex: UInt32, framesCount: UInt32) -> UInt32 {
+        guard framesCount > 0, let entry = playerContext.audioReadingEntry else { return framesCount }
+        entry.lock.lock()
+        let framesState = entry.framesState
+        let decodedStart = framesState.decoded
+        if let decodedStart {
+            framesState.decoded = decodedStart + Int(framesCount)
+        }
+        let primingFrames = framesState.primingFrames
+        // without packet table info only the assumed priming at the start is dropped
+        let validFrames = framesState.validFrames ?? (primingFrames > 0 ? Int(Int32.max) : nil)
+        let sourceSampleRate = entry.audioStreamFormat.mSampleRate
+        entry.lock.unlock()
+
+        guard let decodedStart, let validFrames, sourceSampleRate > 0 else { return framesCount }
+        let ratio = outputAudioFormat.mSampleRate / sourceSampleRate
+        let (leadingFrames, keptFrames) = GaplessTrim.keptRange(
+            decodedStart: decodedStart, framesCount: Int(framesCount),
+            primingFrames: primingFrames, validFrames: validFrames, ratio: ratio
+        )
+        guard keptFrames > 0 else { return 0 }
+        if leadingFrames > 0, let mData = rendererContext.audioBuffer.mData {
+            let frameSize = Int(rendererContext.bufferContext.sizeInBytes)
+            let destination = mData + Int(bufferIndex) * frameSize
+            memmove(destination, destination + leadingFrames * frameSize, keptFrames * frameSize)
+        }
+        return UInt32(keptFrames)
+    }
+
+    /// Drops the encoder padding (remainder) at the end of the stream when `trimGaplessPadding` could not,
+    /// because the decoded position is not exact after a seek.
+    /// Must be called once all packets of the reading entry have been decoded: the padding is the
+    /// last part written to the renderer buffer then.
+    ///
+    /// - parameter entry: The entry whose stream just ended
+    func trimTrailingGaplessPadding(for entry: AudioEntry) {
+        // same lock order as the renderer: buffer first, then entry
+        rendererContext.lock.lock(); defer { rendererContext.lock.unlock() }
+        entry.lock.lock(); defer { entry.lock.unlock() }
+
+        let framesState = entry.framesState
+        let sourceSampleRate = entry.audioStreamFormat.mSampleRate
+        guard sourceSampleRate > 0 else { return }
+        let ratio = outputAudioFormat.mSampleRate / sourceSampleRate
+        let notPlayedFrames = framesState.queued - framesState.played
+        let usedFrames = Int(rendererContext.bufferContext.frameUsedCount)
+        let droppedFrames: Int
+        if framesState.remainderFrames > 0 {
+            guard framesState.decoded == nil else { return }
+            droppedFrames = GaplessTrim.trailingFramesToDrop(
+                remainderFrames: framesState.remainderFrames,
+                ratio: ratio,
+                notPlayedFrames: notPlayedFrames,
+                usedFrames: usedFrames
+            )
+        } else if framesState.validFrames == nil, framesState.primingFrames > 0 {
+            // padding of an MP3 without packet table info is unknown: drop the silence at the end
+            let limit = max(0, min(
+                Int((Double(GaplessTrim.maxAssumedMp3PaddingFrames) * ratio).rounded()),
+                notPlayedFrames,
+                usedFrames
+            ))
+            droppedFrames = trailingSilentFrames(limit: limit, usedFrames: usedFrames)
+        } else {
+            return
+        }
+        guard droppedFrames > 0 else { return }
+
+        rendererContext.bufferContext.frameUsedCount -= UInt32(droppedFrames)
+        framesState.queued -= droppedFrames
+    }
+
+    /// Counts the silent frames at the end of the used renderer buffer.
+    /// The renderer buffer holds interleaved 32 bit float frames (the canonical output format of the player).
+    /// Must be called with the renderer lock held.
+    private func trailingSilentFrames(limit: Int, usedFrames: Int) -> Int {
+        guard limit > 0, let mData = rendererContext.audioBuffer.mData else { return 0 }
+        let bufferContext = rendererContext.bufferContext
+        let total = Int(bufferContext.totalFrameCount)
+        let end = Int(bufferContext.end)
+        let channels = max(1, Int(outputAudioFormat.mChannelsPerFrame))
+        let samples = mData.assumingMemoryBound(to: Float.self)
+        // the music level comes from the frames before the ones that may be dropped
+        let referenceFrames = max(0, min(4096, usedFrames - limit))
+        return GaplessTrim.trailingSilentFrames(limit: limit, referenceFrames: referenceFrames) { n in
+            let frame = (end - 1 - n + total) % total
+            var peak: Float = 0
+            for channel in 0 ..< channels {
+                peak = max(peak, abs(samples[frame * channels + channel]))
+            }
+            return peak
+        }
+    }
+
     /// Advances the processed frames for buffer and reading entry
     ///
     /// - parameter frameCount: An `UInt32` value to be added to the used count of the buffers.

AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift

@@ -8,6 +8,14 @@ import XCTest
 @testable import AudioStreaming
 
 class AudioEntryTests: XCTestCase {
+    func test_Reset_Marks_Decoded_Position_Exact_Only_When_Asked() {
+        let entry = audioEntry(length: 100)
+        entry.reset()
+        XCTAssertEqual(entry.framesState.decoded, 0)
+        entry.reset(decodedPositionExact: false)
+        XCTAssertNil(entry.framesState.decoded)
+    }
+
     func test_AudioDataLengthBytes_Is_Zero_When_DataOffset_Exceeds_Length() {
         // a live stream can report a length smaller than the parsed header offset
         let entry = audioEntry(length: 100)