diff --git a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift index c948801..baeba15 100644 --- a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift +++ b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift @@ -95,11 +95,14 @@ class AudioEntry { source.seek(at: offset) } - func reset() { + /// - parameter decodedPositionExact: Whether the entry is re-read from the start of the stream. + /// Pass `false` when re-reading from a seek offset, so gapless trimming doesn't treat mid-stream data as the start. + func reset(decodedPositionExact: Bool = true) { lock.lock(); defer { lock.unlock() } framesState.played = 0 framesState.queued = 0 framesState.lastFrameQueued = -1 + framesState.decoded = decodedPositionExact ? 0 : nil } func has(same source: CoreAudioStreamSource) -> Bool { diff --git a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift index 1b4d377..617a641 100644 --- a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift +++ b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift @@ -9,4 +9,37 @@ final class EntryFramesState { var queued: Int = 0 var played: Int = 0 var lastFrameQueued: Int = -1 + + /// Gapless info from the packet table, in source frames: encoder delay (priming), audio frames + /// (valid) and padding (remainder). Decoders output priming and remainder like audio; both are + /// dropped. `validFrames` is nil when the stream has no packet table info. + var primingFrames: Int = 0 + var validFrames: Int? + var remainderFrames: Int = 0 + /// Output frames decoded since the start of the stream, including dropped ones. + /// nil after a seek, where the decoded position is not exact. + var decoded: Int? = 0 +} + +/// Pure gapless trimming math, in output frames. +enum GaplessTrim { + /// The part of a decoded chunk to keep. + /// - parameter decodedStart: Output frames decoded before this chunk + /// - parameter framesCount: Frames in this chunk + /// - parameter ratio: Output sample rate / source sample rate + /// - Returns: The frames to skip at the start of the chunk and the frames to keep after them + static func keptRange(decodedStart: Int, framesCount: Int, primingFrames: Int, validFrames: Int, ratio: Double) -> (leading: Int, kept: Int) { + let keepFrom = Int((Double(primingFrames) * ratio).rounded()) + let keepTo = Int((Double(primingFrames + validFrames) * ratio).rounded()) + let first = max(decodedStart, keepFrom) + let last = min(decodedStart + framesCount, keepTo) + guard last > first else { return (0, 0) } + return (first - decodedStart, last - first) + } + + /// The output frames of end padding to drop, limited to what is still unplayed and buffered. + static func trailingFramesToDrop(remainderFrames: Int, ratio: Double, notPlayedFrames: Int, usedFrames: Int) -> Int { + let outputRemainder = Int((Double(remainderFrames) * ratio).rounded()) + return max(0, min(outputRemainder, notPlayedFrames, usedFrames)) + } } diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift index 13fba34..b5b5025 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift @@ -730,7 +730,7 @@ open class AudioPlayer { playingEntry != snapshot.reading { playingEntry.audioStreamState.processedDataFormat = false - playingEntry.reset() + playingEntry.reset(decodedPositionExact: false) if let readingEntry = snapshot.reading { readingEntry.delegate = nil readingEntry.close() @@ -1091,6 +1091,8 @@ extension AudioPlayer: AudioStreamSourceDelegate { Logger.debug("Failed to flush remaining packets at EOF: \(flushStatus)", category: .generic) } + fileStreamProcessor.trimTrailingGaplessPadding(for: readingEntry) + readingEntry.lock.lock() readingEntry.framesState.lastFrameQueued = readingEntry.framesState.queued readingEntry.lock.unlock() diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift index 77d560e..c1689d6 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift @@ -209,7 +209,9 @@ final class AudioFileStreamProcessor { AudioConverterReset(converted) } - readingEntry.reset() + // seeking to the start of the data re-decodes the encoder delay, so the decoded position stays exact + let seeksToStart = seekByteOffset <= Int64(readingEntry.audioStreamState.dataOffset) + readingEntry.reset(decodedPositionExact: seeksToStart) readingEntry.seek(at: Int(seekByteOffset)) rendererContext.waitingForDataAfterSeekFrameCount.write { $0 = 0 } playerContext.setInternalState(to: .waitingForDataAfterSeek) @@ -429,8 +431,17 @@ final class AudioFileStreamProcessor { guard status == noErr else { return } // Use valid frames to refine duration if present entry.lock.lock(); defer { entry.lock.unlock() } - if pti.mNumberValidFrames > 0 { + // the property can be reported again after a seek; keep the values from the first report + if pti.mNumberValidFrames > 0, entry.framesState.validFrames == nil { entry.audioStreamState.dataPacketCount = Double(pti.mNumberValidFrames) / Double(max(1, entry.audioStreamFormat.mFramesPerPacket)) + entry.framesState.primingFrames = Int(pti.mPrimingFrames) + entry.framesState.validFrames = Int(pti.mNumberValidFrames) + entry.framesState.remainderFrames = Int(pti.mRemainderFrames) + // Frames decoded before this info arrived were kept untrimmed, so the decoded position can't + // be used to trim the start; fall back to dropping only the padding at the end of the stream. + if let decoded = entry.framesState.decoded, decoded > 0 { + entry.framesState.decoded = nil + } } } @@ -581,7 +592,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded = framesToDecode + framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) @@ -591,6 +602,12 @@ final class AudioFileStreamProcessor { return } + if framesAdded < framesToDecode { + // dropped frames left a hole before the end of the buffer, continue right after the kept ones + fillUsedFrames(framesCount: framesAdded) + continue packetProcess + } + framesToDecode = start if framesToDecode == 0 { fillUsedFrames(framesCount: framesAdded) @@ -611,7 +628,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded += framesToDecode + framesAdded += trimGaplessPadding(bufferIndex: 0, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) @@ -623,6 +640,8 @@ final class AudioFileStreamProcessor { fileStreamCallback?(.raiseError(.codecError)) return } + // the requested frames were all filled: account for them before decoding more + fillUsedFrames(framesCount: framesAdded) } else { var framesAdded: UInt32 = 0 var framesToDecode: UInt32 = start - end @@ -643,7 +662,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded = framesToDecode + framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) return @@ -654,6 +673,8 @@ final class AudioFileStreamProcessor { fileStreamCallback?(.raiseError(.codecError)) return } + // the requested frames were all filled: account for them before decoding more + fillUsedFrames(framesCount: framesAdded) } } } @@ -676,6 +697,70 @@ final class AudioFileStreamProcessor { bufferList[0].mNumberChannels = rendererContext.audioBuffer.mNumberChannels } + /// Drops the encoder delay (priming) and padding (remainder) that decoders output for formats like + /// AAC and MP3, using the packet table info of the stream. Without this, consecutive entries are + /// separated by a short silence (about 50 ms for AAC). + /// Keeps the decoded frames in [priming, priming + valid) while the decoded position is exact; + /// after a seek, `trimTrailingGaplessPadding()` drops the padding at the end of the stream instead. + /// + /// - parameter bufferIndex: The frame index in the renderer buffer where the decoded frames start + /// - parameter framesCount: The number of frames decoded there + /// - Returns: The number of frames to keep; they are moved to `bufferIndex` + @inline(__always) + private func trimGaplessPadding(bufferIndex: UInt32, framesCount: UInt32) -> UInt32 { + guard framesCount > 0, let entry = playerContext.audioReadingEntry else { return framesCount } + entry.lock.lock() + let framesState = entry.framesState + let decodedStart = framesState.decoded + if let decodedStart { + framesState.decoded = decodedStart + Int(framesCount) + } + let primingFrames = framesState.primingFrames + let validFrames = framesState.validFrames + let sourceSampleRate = entry.audioStreamFormat.mSampleRate + entry.lock.unlock() + + guard let decodedStart, let validFrames, sourceSampleRate > 0 else { return framesCount } + let ratio = outputAudioFormat.mSampleRate / sourceSampleRate + let (leadingFrames, keptFrames) = GaplessTrim.keptRange( + decodedStart: decodedStart, framesCount: Int(framesCount), + primingFrames: primingFrames, validFrames: validFrames, ratio: ratio + ) + guard keptFrames > 0 else { return 0 } + if leadingFrames > 0, let mData = rendererContext.audioBuffer.mData { + let frameSize = Int(rendererContext.bufferContext.sizeInBytes) + let destination = mData + Int(bufferIndex) * frameSize + memmove(destination, destination + leadingFrames * frameSize, keptFrames * frameSize) + } + return UInt32(keptFrames) + } + + /// Drops the encoder padding (remainder) at the end of the stream when `trimGaplessPadding` could not, + /// because the decoded position is not exact after a seek. + /// Must be called once all packets of the reading entry have been decoded: the padding is the + /// last part written to the renderer buffer then. + /// + /// - parameter entry: The entry whose stream just ended + func trimTrailingGaplessPadding(for entry: AudioEntry) { + // same lock order as the renderer: buffer first, then entry + rendererContext.lock.lock(); defer { rendererContext.lock.unlock() } + entry.lock.lock(); defer { entry.lock.unlock() } + + let framesState = entry.framesState + let sourceSampleRate = entry.audioStreamFormat.mSampleRate + guard framesState.decoded == nil, framesState.remainderFrames > 0, sourceSampleRate > 0 else { return } + let droppedFrames = GaplessTrim.trailingFramesToDrop( + remainderFrames: framesState.remainderFrames, + ratio: outputAudioFormat.mSampleRate / sourceSampleRate, + notPlayedFrames: framesState.queued - framesState.played, + usedFrames: Int(rendererContext.bufferContext.frameUsedCount) + ) + guard droppedFrames > 0 else { return } + + rendererContext.bufferContext.frameUsedCount -= UInt32(droppedFrames) + framesState.queued -= droppedFrames + } + /// Advances the processed frames for buffer and reading entry /// /// - parameter frameCount: An `UInt32` value to be added to the used count of the buffers. diff --git a/AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift b/AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift index 126b94e..4667e4d 100644 --- a/AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift +++ b/AudioStreamingTests/Streaming/Audio Entry/AudioEntryTests.swift @@ -8,6 +8,14 @@ import XCTest @testable import AudioStreaming class AudioEntryTests: XCTestCase { + func test_Reset_Marks_Decoded_Position_Exact_Only_When_Asked() { + let entry = audioEntry(length: 100) + entry.reset() + XCTAssertEqual(entry.framesState.decoded, 0) + entry.reset(decodedPositionExact: false) + XCTAssertNil(entry.framesState.decoded) + } + func test_AudioDataLengthBytes_Is_Zero_When_DataOffset_Exceeds_Length() { // a live stream can report a length smaller than the parsed header offset let entry = audioEntry(length: 100) diff --git a/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift b/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift new file mode 100644 index 0000000..ff4915d --- /dev/null +++ b/AudioStreamingTests/Streaming/Audio Entry/GaplessTrimTests.swift @@ -0,0 +1,87 @@ +// +// GaplessTrimTests.swift +// AudioStreamingTests +// + +import XCTest +@testable import AudioStreaming + +class GaplessTrimTests: XCTestCase { + private func range(_ start: Int, _ count: Int, priming: Int = 2112, valid: Int = 10000, ratio: Double = 1) -> (leading: Int, kept: Int) { + GaplessTrim.keptRange(decodedStart: start, framesCount: count, primingFrames: priming, validFrames: valid, ratio: ratio) + } + + func test_Chunk_Inside_Priming_Is_Dropped() { + let r = range(0, 1024) + XCTAssertEqual(r.kept, 0) + } + + func test_Chunk_Straddling_Priming_Drops_Leading_Frames() { + let r = range(2048, 1024) + XCTAssertEqual(r.leading, 64) + XCTAssertEqual(r.kept, 960) + } + + func test_Chunk_Inside_Valid_Range_Is_Kept() { + let r = range(4096, 1024) + XCTAssertEqual(r.leading, 0) + XCTAssertEqual(r.kept, 1024) + } + + func test_Chunk_Straddling_End_Drops_Trailing_Frames() { + // valid range ends at 2112 + 10000 = 12112 + let r = range(11776, 1024) + XCTAssertEqual(r.leading, 0) + XCTAssertEqual(r.kept, 336) + } + + func test_Chunk_After_Valid_Range_Is_Dropped() { + XCTAssertEqual(range(12112, 1024).kept, 0) + } + + func test_Chunk_Containing_Whole_Valid_Range() { + let r = range(0, 20000, priming: 100, valid: 500) + XCTAssertEqual(r.leading, 100) + XCTAssertEqual(r.kept, 500) + } + + func test_Kept_Frames_Over_Chunks_Equal_Valid_Frames() { + var decoded = 0 + var total = 0 + for count in [1024, 1024, 777, 4096, 1, 9000, 1024] { + total += range(decoded, count).kept + decoded += count + } + XCTAssertEqual(total, 10000) + } + + func test_Ratio_Scales_Priming_And_Valid() { + let r = range(0, 20000, priming: 1000, valid: 4000, ratio: 48000.0 / 44100.0) + XCTAssertEqual(r.leading, 1088) + XCTAssertEqual(r.kept, 4354) + } + + func test_Kept_Frames_Over_Chunks_Match_Rounded_End_With_Ratio() { + let ratio = 48000.0 / 44100.0 + var decoded = 0 + var total = 0 + for count in [1000, 1024, 777, 4096, 1, 9000, 1024] { + let r = range(decoded, count, priming: 1105, valid: 7003, ratio: ratio) + total += r.kept + decoded += count + } + // keepFrom = round(1105 * ratio), keepTo = round(8108 * ratio) + XCTAssertEqual(total, Int((8108 * ratio).rounded()) - Int((1105 * ratio).rounded())) + } + + func test_Trailing_Drop_Is_Limited_By_Unplayed_And_Used_Frames() { + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 10000, usedFrames: 10000), 500) + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 200, usedFrames: 10000), 200) + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: 10000, usedFrames: 100), 100) + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 500, ratio: 1, notPlayedFrames: -5, usedFrames: 100), 0) + } + + func test_Trailing_Drop_Scales_With_Ratio() { + XCTAssertEqual(GaplessTrim.trailingFramesToDrop(remainderFrames: 1000, ratio: 2, notPlayedFrames: 5000, usedFrames: 5000), 2000) + } +}