From 25a7d99822756d04b0394fe8b45348eb7073af7d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 1 Jul 2026 08:57:15 +0200 Subject: [PATCH 01/12] feat(download): start SABR download support --- .../newpipe/download/DownloadDialog.java | 53 +++-- .../newpipe/util/StreamItemAdapter.java | 3 +- .../shandian/giga/get/DirectDownloader.java | 9 +- .../us/shandian/giga/get/DownloadMission.java | 30 ++- .../shandian/giga/get/MissionRecoveryInfo.kt | 9 +- .../giga/get/SabrDownloadFormatResolver.kt | 75 ++++++ .../giga/get/SabrDownloadStreamHelper.java | 69 ++++++ .../shandian/giga/get/SabrDownloadTarget.kt | 16 ++ .../us/shandian/giga/get/SabrDownloader.kt | 218 ++++++++++++++++++ .../us/shandian/giga/get/SabrFfmpegMuxer.kt | 99 ++++++++ .../us/shandian/giga/get/SabrSegmentWriter.kt | 177 ++++++++++++++ .../giga/service/DownloadManagerService.java | 5 +- 12 files changed, 737 insertions(+), 26 deletions(-) create mode 100644 app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt create mode 100644 app/src/main/java/us/shandian/giga/get/SabrDownloadStreamHelper.java create mode 100644 app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt create mode 100644 app/src/main/java/us/shandian/giga/get/SabrDownloader.kt create mode 100644 app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt create mode 100644 app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 8f1e98524..e1d56fab8 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -52,7 +52,6 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.SubtitlesStream; @@ -88,6 +87,7 @@ import io.reactivex.rxjava3.disposables.CompositeDisposable; import us.shandian.giga.get.HlsDownloadStreamHelper; import us.shandian.giga.get.MissionRecoveryInfo; +import us.shandian.giga.get.SabrDownloadStreamHelper; import us.shandian.giga.postprocessing.Postprocessing; import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; @@ -157,29 +157,20 @@ public static DownloadDialog newInstance(final StreamInfo info) { } public static DownloadDialog newInstance(final Context context, final StreamInfo info) { - final List videoStreams = info.getVideoStreams().stream() - .filter(stream -> stream.getDeliveryMethod() != DeliveryMethod.SABR) - .collect(Collectors.toList()); - final List videoOnlyStreams = info.getVideoOnlyStreams().stream() - .filter(stream -> stream.getDeliveryMethod() != DeliveryMethod.SABR) - .collect(Collectors.toList()); - final List audioStreams = info.getAudioStreams().stream() - .filter(stream -> stream.getDeliveryMethod() != DeliveryMethod.SABR) - .collect(Collectors.toList()); final ArrayList streamsList = new ArrayList<>(ListHelper - .getSortedStreamVideosList(context, videoStreams, - videoOnlyStreams, false, false)); + .getSortedStreamVideosList(context, info.getVideoStreams(), + info.getVideoOnlyStreams(), false, false)); final List filteredVideoStreams = ListHelper .filterVideoStreamsByPreferredLanguage(context, streamsList, - audioStreams); + info.getAudioStreams()); final int selectedStreamIndex = ListHelper.getDefaultResolutionIndex( context, filteredVideoStreams); HlsDownloadStreamHelper.addManifestFallbackIfNeeded(filteredVideoStreams, info); final List downloadableAudio = ListHelper - .filterDownloadableAudioStreams(audioStreams); + .filterDownloadableAudioStreams(info.getAudioStreams()); HlsDownloadStreamHelper.addAudioFallbackIfNeeded(downloadableAudio, info); final DownloadDialog instance = newInstance(info); @@ -278,9 +269,11 @@ public void onCreate(@Nullable final Bundle savedInstanceState) { continue; } final AudioStream audioStream = SecondaryStreamHelper - .getAudioStreamFor(getContext(), wrappedAudioStreams.getStreamsList(), videoStreams.get(i)); + .getAudioStreamFor(getContext(), SabrDownloadStreamHelper.audioStreamsForVideo( + wrappedAudioStreams.getStreamsList(), videoStreams.get(i)), videoStreams.get(i)); - if (audioStream != null) { + if (audioStream != null && SabrDownloadStreamHelper + .isCompatibleSecondaryStream(videoStreams.get(i), audioStream)) { secondaryStreams .append(i, new SecondaryStreamHelper<>(wrappedAudioStreams, audioStream)); } else if (DEBUG) { @@ -348,6 +341,7 @@ public void onViewCreated(@NonNull final View view, @Nullable final Bundle saved initToolbar(dialogBinding.toolbarLayout.toolbar); setupDownloadOptions(); + dumpDownloadStreamsForLocalTest(); prefs = PreferenceManager.getDefaultSharedPreferences(requireContext()); @@ -368,6 +362,30 @@ public void onProgressChanged(@NonNull final SeekBar seekbar, final int progress fetchStreamsSize(); } + private void dumpDownloadStreamsForLocalTest() { + for (int i = 0; i < wrappedVideoStreams.getStreamsList().size(); i++) { + final VideoStream stream = wrappedVideoStreams.getStreamsList().get(i); + Log.d(TAG, "local-download-stream video index=" + i + + " delivery=" + stream.getDeliveryMethod() + + " format=" + stream.getFormat() + + " codec=" + stream.getCodec() + + " resolution=" + stream.getResolution() + + " itag=" + stream.getItag() + + " videoOnly=" + stream.isVideoOnly() + + " audioTrackId=" + stream.getAudioTrackId()); + } + for (int i = 0; i < wrappedAudioStreams.getStreamsList().size(); i++) { + final AudioStream stream = wrappedAudioStreams.getStreamsList().get(i); + Log.d(TAG, "local-download-stream audio index=" + i + + " delivery=" + stream.getDeliveryMethod() + + " format=" + stream.getFormat() + + " codec=" + stream.getCodec() + + " quality=" + stream.getQuality() + + " itag=" + stream.getItag() + + " audioTrackId=" + stream.getAudioTrackId()); + } + } + private void initToolbar(final Toolbar toolbar) { if (DEBUG) { Log.d(TAG, "initToolbar() called with: toolbar = [" + toolbar + "]"); @@ -1154,7 +1172,8 @@ private void continueSelectedDownload(@NonNull final StoredFileHelper storage) { resourceIsUrls = HlsDownloadStreamHelper .buildResourceIsUrls(selectedStream, secondaryStream); if (HlsDownloadStreamHelper.containsHlsResource(resourceDeliveryMethods, - resourceManifestUrls, urls)) { + resourceManifestUrls, urls) + || SabrDownloadStreamHelper.containsSabrStream(selectedStream, secondaryStream)) { psName = null; psArgs = null; } diff --git a/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java b/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java index 04cca6bf6..ff2ccbf26 100644 --- a/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java +++ b/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java @@ -259,7 +259,8 @@ public static Single fetchSizeForWrapper( if (streamsWrapper.getSizeInBytes(stream) > -2) { continue; } - if (stream.getDeliveryMethod() == DeliveryMethod.HLS) { + if (stream.getDeliveryMethod() == DeliveryMethod.HLS + || stream.getDeliveryMethod() == DeliveryMethod.SABR) { streamsWrapper.setSize(stream, -1); hasChanged = true; continue; diff --git a/app/src/main/java/us/shandian/giga/get/DirectDownloader.java b/app/src/main/java/us/shandian/giga/get/DirectDownloader.java index f8631b2f2..637273bad 100644 --- a/app/src/main/java/us/shandian/giga/get/DirectDownloader.java +++ b/app/src/main/java/us/shandian/giga/get/DirectDownloader.java @@ -105,9 +105,11 @@ public void init(){ continue; } final AudioStream audioStream = SecondaryStreamHelper - .getAudioStreamFor(context, wrappedAudioStreams.getStreamsList(), videoStreams.get(i)); + .getAudioStreamFor(context, SabrDownloadStreamHelper.audioStreamsForVideo( + wrappedAudioStreams.getStreamsList(), videoStreams.get(i)), videoStreams.get(i)); - if (audioStream != null) { + if (audioStream != null && SabrDownloadStreamHelper + .isCompatibleSecondaryStream(videoStreams.get(i), audioStream)) { secondaryStreams .append(i, new SecondaryStreamHelper<>(wrappedAudioStreams, audioStream)); } @@ -270,7 +272,8 @@ public void startDownload(StoredFileHelper storage) { resourceIsUrls = HlsDownloadStreamHelper .buildResourceIsUrls(selectedStream, secondaryStream); if (HlsDownloadStreamHelper.containsHlsResource(resourceDeliveryMethods, - resourceManifestUrls, urls)) { + resourceManifestUrls, urls) + || SabrDownloadStreamHelper.containsSabrStream(selectedStream, secondaryStream)) { psName = null; psArgs = null; } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 359c17b35..02ab94bb3 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -128,12 +128,14 @@ public class DownloadMission extends Mission { public MissionRecoveryInfo[] recoveryInfo; /** - * Optional typed metadata for resources. Used only to route HLS resources to the HLS downloader. + * Optional typed metadata for resources. Used to route session/manifest resources to their + * dedicated downloaders instead of the direct HTTP range downloader. */ public String[] resourceDeliveryMethods; public String[] resourceManifestUrls; public boolean[] resourceIsUrls; public HlsDownloadCheckpoint hlsCheckpoint; + public boolean sabrStarted; private transient int finishCount; public transient volatile boolean running; @@ -436,6 +438,13 @@ private void notifyPostProcessing(int state) { * Start downloading with multiple threads. */ public void start() { + Log.d(TAG, "local-download-start running=" + running + + " finished=" + isFinished() + + " urls=" + urls.length + + " current=" + current + + " hasSabr=" + hasSabrResource() + + " hasHls=" + hasHlsResource() + + " sabrStarted=" + sabrStarted); if (running || isFinished() || urls.length < 1) return; // ensure that the previous state is completely paused. @@ -461,7 +470,14 @@ public void start() { return; } + if (hasSabrResource()) { + Log.d(TAG, "local-download-route SABR"); + init = runAsync(DownloadInitializer.mId, new SabrDownloader(this)); + return; + } + if (hasHlsResource()) { + Log.d(TAG, "local-download-route HLS"); init = runAsync(DownloadInitializer.mId, new HlsDownloader(this)); return; } @@ -540,6 +556,7 @@ private void pauseThreads() { public boolean delete() { if (psAlgorithm != null) psAlgorithm.cleanupTemporalDir(); HlsDownloader.cleanup(this); + SabrDownloader.cleanup(this); notify(DownloadManagerService.MESSAGE_DELETED); @@ -565,7 +582,10 @@ public void resetState(boolean rollback, boolean persistChanges, int errorCode) fallbackResumeOffset = 0; blocks = null; blockAcquired = null; - if (rollback) hlsCheckpoint = null; + if (rollback) { + hlsCheckpoint = null; + sabrStarted = false; + } if (rollback) current = 0; if (persistChanges) writeThisToFile(); @@ -630,7 +650,7 @@ public boolean isPsRunning() { * @return true, otherwise, false */ public boolean isInitialized() { - return blocks != null || hlsCheckpoint != null; // DownloadMissionInitializer or HLS downloader was executed + return blocks != null || hlsCheckpoint != null || sabrStarted; } boolean hasHlsResource() { @@ -638,6 +658,10 @@ boolean hasHlsResource() { resourceManifestUrls, urls); } + boolean hasSabrResource() { + return SabrDownloadStreamHelper.containsSabrResource(resourceDeliveryMethods, recoveryInfo); + } + /** * Gets the approximated final length of the file * diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.kt b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.kt index e2f1f6e63..4995e8219 100644 --- a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.kt +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.kt @@ -19,22 +19,29 @@ class MissionRecoveryInfo( var kind: Char = Char.MIN_VALUE, var validateCondition: String? = null, var audioTrackId: String? = null, - var isHls: Boolean = false + var isHls: Boolean = false, + var isSabr: Boolean = false, + var itag: Int = -1, + var deliveryMethodInfo: Serializable? = null ) : Serializable, Parcelable { constructor(stream: Stream) : this(format = stream.getFormat()!!) { isHls = stream.getDeliveryMethod() == DeliveryMethod.HLS + isSabr = stream.getDeliveryMethod() == DeliveryMethod.SABR + deliveryMethodInfo = stream.deliveryMethodInfo when (stream) { is AudioStream -> { desiredBitrate = stream.averageBitrate isDesired2 = false kind = 'a' audioTrackId = stream.audioTrackId + itag = stream.itag } is VideoStream -> { desired = stream.resolution isDesired2 = stream.isVideoOnly kind = 'v' audioTrackId = stream.audioTrackId + itag = stream.itag } is SubtitlesStream -> { desired = stream.languageTag diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt new file mode 100644 index 000000000..d945a02e7 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt @@ -0,0 +1,75 @@ +package us.shandian.giga.get + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import java.io.File +import java.io.IOException + +internal object SabrDownloadFormatResolver { + @Throws(IOException::class) + fun resolveInfo(recoveries: Array): YoutubeSabrInfo { + return recoveries.firstNotNullOfOrNull { + it.deliveryMethodInfo as? YoutubeSabrInfo + } ?: throw IOException("Missing SABR info") + } + + @Throws(IOException::class) + fun selectedAudioFormat( + info: YoutubeSabrInfo, + recoveries: Array, + ): YoutubeSabrFormat { + val audioRecovery = recoveries.firstOrNull { it.kind == 'a' } + return audioRecovery?.let { findAudioFormat(info, it) } + ?: info.findBestAudioFormat() + ?: throw IOException("Missing SABR audio format") + } + + @Throws(IOException::class) + fun selectedVideoFormat( + info: YoutubeSabrInfo, + recoveries: Array, + ): YoutubeSabrFormat { + val videoRecovery = recoveries.firstOrNull { it.kind == 'v' } + return videoRecovery?.let { findVideoFormat(info, it) } + ?: info.findBestVideoFormat() + ?: throw IOException("Missing SABR video format") + } + + @Throws(IOException::class) + fun buildTargets( + info: YoutubeSabrInfo, + recoveries: Array, + workDir: File, + ): List { + return recoveries.mapIndexed { index, recovery -> + val format = when (recovery.kind) { + 'a' -> findAudioFormat(info, recovery) + 'v' -> findVideoFormat(info, recovery) + else -> throw IOException("Unsupported SABR resource kind: ${recovery.kind}") + } + SabrDownloadTarget(index, recovery, format, File(workDir, "input-$index.media")) + } + } + + @Throws(IOException::class) + private fun findAudioFormat( + info: YoutubeSabrInfo, + recovery: MissionRecoveryInfo, + ): YoutubeSabrFormat { + return info.formats.firstOrNull { format -> + format.isAudio && + (recovery.itag <= 0 || format.itag == recovery.itag) && + (recovery.audioTrackId == null || recovery.audioTrackId == format.audioTrackId) + } ?: throw IOException("Could not resolve SABR audio format: itag=${recovery.itag}") + } + + @Throws(IOException::class) + private fun findVideoFormat( + info: YoutubeSabrInfo, + recovery: MissionRecoveryInfo, + ): YoutubeSabrFormat { + return info.formats.firstOrNull { format -> + format.isVideo && (recovery.itag <= 0 || format.itag == recovery.itag) + } ?: throw IOException("Could not resolve SABR video format: itag=${recovery.itag}") + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadStreamHelper.java b/app/src/main/java/us/shandian/giga/get/SabrDownloadStreamHelper.java new file mode 100644 index 000000000..3dc15d63b --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadStreamHelper.java @@ -0,0 +1,69 @@ +package us.shandian.giga.get; + +import org.schabi.newpipe.extractor.stream.DeliveryMethod; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.util.ArrayList; +import java.util.List; + +public final class SabrDownloadStreamHelper { + private SabrDownloadStreamHelper() { + } + + public static boolean containsSabrResource(final String[] deliveryMethods, + final MissionRecoveryInfo[] recoveryInfo) { + if (deliveryMethods != null) { + for (final String deliveryMethod : deliveryMethods) { + if (DeliveryMethod.SABR.name().equals(deliveryMethod)) { + return true; + } + } + } + if (recoveryInfo != null) { + for (final MissionRecoveryInfo recovery : recoveryInfo) { + if (recovery != null && recovery.isSabr()) { + return true; + } + } + } + return false; + } + + public static boolean containsSabrStream(final Stream selectedStream, + final Stream secondaryStream) { + return isSabr(selectedStream) || isSabr(secondaryStream); + } + + public static List audioStreamsForVideo(final List audioStreams, + final VideoStream videoStream) { + if (audioStreams == null || audioStreams.isEmpty()) { + return audioStreams; + } + + final List result = new ArrayList<>(); + for (final AudioStream audioStream : audioStreams) { + if (isSabr(audioStream) == isSabr(videoStream)) { + result.add(audioStream); + } + } + return result; + } + + public static boolean isCompatibleSecondaryStream(final Stream selectedStream, + final Stream secondaryStream) { + if (secondaryStream == null) { + return true; + } + if (!isSabr(selectedStream) && !isSabr(secondaryStream)) { + return true; + } + return isSabr(selectedStream) && isSabr(secondaryStream); + } + + private static boolean isSabr(final Stream stream) { + return stream != null && stream.getDeliveryMethod() == DeliveryMethod.SABR; + } + +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt new file mode 100644 index 000000000..15f37dbfa --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt @@ -0,0 +1,16 @@ +package us.shandian.giga.get + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import java.io.File +import java.util.TreeMap + +internal data class SabrDownloadTarget( + val resourceIndex: Int, + val recovery: MissionRecoveryInfo, + val format: YoutubeSabrFormat, + val file: File, + var nextRequestSequence: Int = 1, + var nextWriteSequence: Int = 1, + var initializationWritten: Boolean = false, + val pending: TreeMap = TreeMap(), +) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt new file mode 100644 index 000000000..45fd1772c --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -0,0 +1,218 @@ +package us.shandian.giga.get + +import android.util.Log +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.player.datasource.WebViewPoTokenProvider +import java.io.File +import java.io.IOException + +internal class SabrDownloader( + private val mission: DownloadMission, +) : Runnable { + override fun run() { + Log.d(TAG, "local-sabr-run start urls=${mission.urls.size} running=${mission.running}") + try { + ensureRunning() + val recoveries = validateRecoveryInfo() + prepareMission() + + val info = SabrDownloadFormatResolver.resolveInfo(recoveries) + val audioRecovery = recoveries.firstOrNull { it.kind == 'a' } + val videoRecovery = recoveries.firstOrNull { it.kind == 'v' } + val session = YoutubeSabrSession( + info, + SabrDownloadFormatResolver.selectedAudioFormat(info, recoveries), + SabrDownloadFormatResolver.selectedVideoFormat(info, recoveries), + WebViewPoTokenProvider(mission.context), + ) + configureRequestMode(session, audioRecovery, videoRecovery) + + val workDir = prepareWorkDirectory() + val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) + val outputs = targets.associate { target -> + target.resourceIndex to target.file.outputStream() + } + + try { + downloadSegments(session, targets, SabrSegmentWriter(mission, session, targets, outputs)) + } finally { + outputs.values.forEach { output -> + try { + output.close() + } catch (ignored: Exception) { + // Nothing to do. + } + } + } + + ensureRunning() + val finalBytes = SabrFfmpegMuxer(mission).remuxAndCopy(targets.map { it.file }, workDir) + completeMission(finalBytes) + } catch (error: InterruptedException) { + Thread.currentThread().interrupt() + } catch (error: SabrProtocolException) { + if (mission.running) { + mission.notifyError(IOException(error)) + } + } catch (error: Exception) { + if (mission.running) { + mission.notifyError(error) + } + } + } + + @Throws(IOException::class) + private fun validateRecoveryInfo(): Array { + val recoveries = mission.recoveryInfo ?: throw IOException("Missing SABR recovery info") + Log.d(TAG, "local-sabr-run recoveries=${recoveries.size} sabr=${recoveries.count { it.isSabr }}") + if (recoveries.size != mission.urls.size || recoveries.any { !it.isSabr }) { + throw IOException("Mixed SABR/non-SABR missions are not supported") + } + return recoveries + } + + private fun prepareMission() { + mission.unknownLength = true + mission.sabrStarted = true + if (mission.done > 0) { + mission.notifyProgress(-mission.done) + } + mission.done = 0 + mission.current = 0 + mission.length = mission.nearLength + mission.writeThisToFile() + } + + private fun configureRequestMode( + session: YoutubeSabrSession, + audioRecovery: MissionRecoveryInfo?, + videoRecovery: MissionRecoveryInfo?, + ) { + when { + videoRecovery == null -> session.streamState.setAudioOnlyRequestMode() + audioRecovery == null -> session.streamState.setVideoOnlyRequestMode() + else -> session.streamState.setVideoAndAudioRequestMode() + } + } + + @Throws(IOException::class) + private fun prepareWorkDirectory(): File { + val workDir = workDirectory(mission) + cleanup(mission) + if (!workDir.mkdirs()) { + throw IOException("Cannot create SABR work directory: $workDir") + } + Log.d(TAG, "local-sabr-run workDir=$workDir") + return workDir + } + + @Throws(IOException::class, InterruptedException::class) + private fun downloadSegments( + session: YoutubeSabrSession, + targets: List, + writer: SabrSegmentWriter, + ) { + val localization = Localization("en", "US") + writer.writeDirectInitializations() + + var emptyResponses = 0 + while (true) { + ensureRunning() + var allComplete = true + var wroteSegment = writer.drainCachedInitializations() + wroteSegment = writer.drainCachedSegments() || wroteSegment + + for (target in targets) { + ensureRunning() + val request = SabrSegmentRequest.media(target.format, target.nextRequestSequence) + if (session.isBeyondEnd(request) || session.streamState.isComplete(target.format)) { + continue + } + + allComplete = false + session.streamState.setPlayerTimeMs(session.streamState.minBufferedEndMs) + val segment = try { + session.fetchSegment(request, localization) + } catch (error: SabrProtocolException) { + if (isRetryablePolicyOnly(error)) { + Log.d(TAG, "local-sabr-policy-idle itag=${target.format.itag}" + + " seq=${target.nextRequestSequence}: ${error.message}") + continue + } + throw error + } + + writer.writeFetchedSegment(target, segment) + wroteSegment = true + wroteSegment = writer.drainCachedInitializations() || wroteSegment + wroteSegment = writer.drainCachedSegments() || wroteSegment + } + + if ((allComplete || targets.all { session.streamState.isComplete(it.format) }) + && targets.all { it.pending.isEmpty() } + ) { + break + } + if (wroteSegment) { + emptyResponses = 0 + } else { + emptyResponses++ + if (emptyResponses > MAX_EMPTY_RESPONSES) { + throw IOException("SABR download stalled with no media") + } + Thread.sleep(IDLE_POLL_MS) + } + } + } + + private fun isRetryablePolicyOnly(error: SabrProtocolException): Boolean { + return error.message?.contains("repeated policy-only responses") == true + } + + private fun completeMission(finalBytes: Long) { + if (finalBytes > 0) { + mission.done = finalBytes + mission.length = finalBytes + } + mission.current = mission.urls.size + mission.psState = 2 + cleanup(mission) + mission.unknownLength = false + mission.notifyFinished() + } + + @Throws(InterruptedException::class) + private fun ensureRunning() { + if (!mission.running || Thread.currentThread().isInterrupted) { + throw InterruptedException() + } + } + + companion object { + private const val TAG = "SabrDownloader" + private const val IDLE_POLL_MS = 250L + private const val MAX_EMPTY_RESPONSES = 60 + + @JvmStatic + fun cleanup(mission: DownloadMission) { + try { + workDirectory(mission).deleteRecursively() + } catch (ignored: Exception) { + // Nothing to do. + } + } + + private fun workDirectory(mission: DownloadMission): File { + val base = mission.context.getExternalFilesDir(null) ?: mission.context.filesDir + val missionId = if (mission.timestamp > 0) { + mission.timestamp.toString() + } else { + mission.storage.name.hashCode().toString() + } + return File(base, "sabr-downloader/$missionId") + } + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt new file mode 100644 index 000000000..3c96474f0 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt @@ -0,0 +1,99 @@ +package us.shandian.giga.get + +import android.util.Log +import com.arthenica.ffmpegkit.FFmpegKit +import com.arthenica.ffmpegkit.ReturnCode +import java.io.File +import java.io.IOException + +internal class SabrFfmpegMuxer( + private val mission: DownloadMission, +) { + @Throws(IOException::class, InterruptedException::class) + fun remuxAndCopy(inputs: List, workDir: File): Long { + val output = File(workDir, "output.${outputExtension()}") + remuxWithFfmpeg(inputs, output) + return copyOutputToStorage(output) + } + + @Throws(IOException::class) + private fun remuxWithFfmpeg(inputs: List, output: File) { + mission.psState = 1 + mission.writeThisToFile() + val command = buildString { + append("-y ") + inputs.forEach { input -> + append("-i ").append(quote(input.absolutePath)).append(' ') + } + if (mission.kind == 'a' && inputs.size == 1) { + append("-map 0:a? -vn ") + } else if (inputs.size > 1) { + inputs.indices.forEach { index -> + append("-map ").append(index).append(":v? ") + append("-map ").append(index).append(":a? ") + } + } + append("-c copy ") + if (supportsFastStart(output)) { + append("-movflags +faststart ") + } + append(quote(output.absolutePath)) + } + Log.d(TAG, "remuxWithFfmpeg inputs=${inputs.size}") + val session = FFmpegKit.execute(command) + if (!ReturnCode.isSuccess(session.returnCode)) { + mission.psState = 0 + throw IOException("SABR ffmpeg remux failed: ${session.returnCode}") + } + } + + @Throws(IOException::class, InterruptedException::class) + private fun copyOutputToStorage(output: File): Long { + var copied = 0L + output.inputStream().use { input -> + mission.storage.getStream().use { storage -> + storage.setLength(0) + storage.seek(0) + val buffer = ByteArray(DownloadMission.BUFFER_SIZE) + while (true) { + ensureRunning() + val read = input.read(buffer) + if (read == -1) { + break + } + storage.write(buffer, 0, read) + copied += read.toLong() + } + } + } + return copied + } + + @Throws(InterruptedException::class) + private fun ensureRunning() { + if (!mission.running || Thread.currentThread().isInterrupted) { + throw InterruptedException() + } + } + + private fun quote(value: String): String { + return '"' + value.replace("\\", "\\\\").replace("\"", "\\\"") + '"' + } + + private fun outputExtension(): String { + val name = mission.storage.name ?: return "mp4" + val extension = name.substringAfterLast('.', missingDelimiterValue = "") + return extension.takeIf { it.isNotBlank() && it.all { char -> char.isLetterOrDigit() } } ?: "mp4" + } + + private fun supportsFastStart(output: File): Boolean { + return when (output.extension.lowercase()) { + "m4a", "m4v", "mov", "mp4" -> true + else -> false + } + } + + private companion object { + private const val TAG = "SabrFfmpegMuxer" + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt new file mode 100644 index 000000000..74d35c3ac --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -0,0 +1,177 @@ +package us.shandian.giga.get + +import android.util.Log +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import java.io.IOException +import java.io.OutputStream + +internal class SabrSegmentWriter( + private val mission: DownloadMission, + private val session: YoutubeSabrSession, + private val targets: List, + private val outputs: Map, +) { + @Throws(IOException::class) + fun writeDirectInitializations() { + for (target in targets) { + writeDirectInitializationIfAvailable(target, outputs.getValue(target.resourceIndex)) + } + } + + @Throws(IOException::class) + fun drainCachedInitializations(): Boolean { + var wroteInitialization = false + for (target in targets) { + if (target.initializationWritten) { + continue + } + val request = SabrSegmentRequest.initialization(target.format) + val segment = session.getCachedSegment(request) ?: continue + writeInitializationSegment(target, outputs.getValue(target.resourceIndex), segment.data) + session.discardCachedSegment(request) + Log.d(TAG, "local-sabr-init itag=${target.format.itag} bytes=${segment.data.size}") + wroteInitialization = true + } + return wroteInitialization + } + + @Throws(IOException::class) + fun drainCachedSegments(): Boolean { + var wroteSegment = false + for (target in targets) { + while (true) { + val request = SabrSegmentRequest.media(target.format, target.nextRequestSequence) + val segment = session.getCachedSegment(request) ?: break + if (segment.header.isInitSegment) { + session.discardCachedSegment(request) + continue + } + writeMediaSegment(target, outputs.getValue(target.resourceIndex), segment) + session.discardCachedSegment(request) + logWrittenSegment(target, segment) + wroteSegment = true + } + } + return wroteSegment + } + + @Throws(IOException::class) + fun writeFetchedSegment(target: SabrDownloadTarget, segment: SabrMediaSegment) { + if (segment.header.isInitSegment) { + return + } + val request = SabrSegmentRequest.media(target.format, segment.header.sequenceNumber) + writeMediaSegment(target, outputs.getValue(target.resourceIndex), segment) + session.discardCachedSegment(request) + logWrittenSegment(target, segment) + } + + @Throws(IOException::class) + private fun writeDirectInitializationIfAvailable( + target: SabrDownloadTarget, + output: OutputStream, + ) { + val data = fetchDirectInitializationData(target.format) ?: return + writeInitializationSegment(target, output, data) + } + + @Throws(IOException::class) + private fun writeInitializationSegment( + target: SabrDownloadTarget, + output: OutputStream, + data: ByteArray, + ): Boolean { + if (target.initializationWritten) { + return false + } + output.write(data) + target.initializationWritten = true + mission.notifyProgress(data.size.toLong()) + flushPendingMedia(target, output) + return true + } + + @Throws(IOException::class) + private fun fetchDirectInitializationData(format: YoutubeSabrFormat): ByteArray? { + val url = format.initializationUrl + val start = format.initRangeStart + val end = format.initRangeEnd + if (url.isNullOrBlank() || start < 0 || end < start) { + return null + } + val range = "bytes=$start-$end" + val response = NewPipe.getDownloader().get(url, mapOf("Range" to listOf(range))) + val data = response.rawResponseBody() + if (response.responseCode() != 206 && response.responseCode() != 200) { + throw IOException("Could not fetch SABR init for itag=${format.itag}: HTTP ${response.responseCode()}") + } + if (data == null || data.isEmpty()) { + throw IOException("Empty SABR init for itag=${format.itag}") + } + return data + } + + @Throws(IOException::class) + private fun writeMediaSegment( + target: SabrDownloadTarget, + output: OutputStream, + segment: SabrMediaSegment, + ) { + val sequence = segment.header.sequenceNumber + if (sequence < target.nextWriteSequence) { + return + } + if (!target.initializationWritten) { + target.pending[sequence] = segment.data + advanceNextRequestSequence(target, sequence) + return + } + if (sequence > target.nextWriteSequence) { + target.pending[sequence] = segment.data + advanceNextRequestSequence(target, sequence) + return + } + writeMediaBytes(target, output, segment.data) + flushPendingMedia(target, output) + } + + private fun flushPendingMedia(target: SabrDownloadTarget, output: OutputStream) { + while (true) { + val pending = target.pending.remove(target.nextWriteSequence) ?: return + writeMediaBytes(target, output, pending) + } + } + + private fun writeMediaBytes(target: SabrDownloadTarget, output: OutputStream, data: ByteArray) { + output.write(data) + target.nextWriteSequence++ + if (target.nextRequestSequence < target.nextWriteSequence) { + target.nextRequestSequence = target.nextWriteSequence + } + mission.notifyProgress(data.size.toLong()) + } + + private fun logWrittenSegment(target: SabrDownloadTarget, segment: SabrMediaSegment) { + val sequence = segment.header.sequenceNumber + if (sequence <= 3 || sequence % LOG_EVERY_SEGMENTS == 0) { + Log.d(TAG, "local-sabr-write itag=${target.format.itag}" + + " seq=$sequence" + + " bytes=${segment.data.size}") + } + } + + private fun advanceNextRequestSequence(target: SabrDownloadTarget, observedSequence: Int) { + if (observedSequence >= target.nextRequestSequence) { + target.nextRequestSequence = observedSequence + 1 + } + } + + private companion object { + private const val TAG = "SabrSegmentWriter" + private const val LOG_EVERY_SEGMENTS = 50 + } +} diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index 0e50b132a..4039476ae 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -49,6 +49,7 @@ import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.HlsDownloadStreamHelper; import us.shandian.giga.get.MissionRecoveryInfo; +import us.shandian.giga.get.SabrDownloadStreamHelper; import org.schabi.newpipe.streams.io.StoredDirectoryHelper; import org.schabi.newpipe.streams.io.StoredFileHelper; import org.schabi.newpipe.util.Localization; @@ -452,7 +453,9 @@ private void startMission(Intent intent) { if (DEBUG && ps == null && urls != null && urls.length > 1 && !HlsDownloadStreamHelper.containsHlsResource(mission.resourceDeliveryMethods, - mission.resourceManifestUrls, urls)) { + mission.resourceManifestUrls, urls) + && !SabrDownloadStreamHelper.containsSabrResource(mission.resourceDeliveryMethods, + mission.recoveryInfo)) { Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?"); } From ceeb52444f0e2372d72e67b9c19477d45649e2cc Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 1 Jul 2026 11:29:21 +0200 Subject: [PATCH 02/12] fix(download): drive SABR downloads with session pump --- .../us/shandian/giga/get/SabrDownloader.kt | 49 +++++++------------ .../us/shandian/giga/get/SabrSegmentWriter.kt | 11 ----- 2 files changed, 18 insertions(+), 42 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index 45fd1772c..85594e680 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -121,42 +121,22 @@ internal class SabrDownloader( var emptyResponses = 0 while (true) { ensureRunning() - var allComplete = true var wroteSegment = writer.drainCachedInitializations() wroteSegment = writer.drainCachedSegments() || wroteSegment - for (target in targets) { - ensureRunning() - val request = SabrSegmentRequest.media(target.format, target.nextRequestSequence) - if (session.isBeyondEnd(request) || session.streamState.isComplete(target.format)) { - continue - } - - allComplete = false - session.streamState.setPlayerTimeMs(session.streamState.minBufferedEndMs) - val segment = try { - session.fetchSegment(request, localization) - } catch (error: SabrProtocolException) { - if (isRetryablePolicyOnly(error)) { - Log.d(TAG, "local-sabr-policy-idle itag=${target.format.itag}" - + " seq=${target.nextRequestSequence}: ${error.message}") - continue - } - throw error - } - - writer.writeFetchedSegment(target, segment) - wroteSegment = true - wroteSegment = writer.drainCachedInitializations() || wroteSegment - wroteSegment = writer.drainCachedSegments() || wroteSegment + if (isDownloadComplete(session, targets)) { + break } - if ((allComplete || targets.all { session.streamState.isComplete(it.format) }) - && targets.all { it.pending.isEmpty() } - ) { + session.streamState.setPlayerTimeMs(session.streamState.minBufferedEndMs) + val segments = session.pumpOnce(localization) + wroteSegment = writer.drainCachedInitializations() || wroteSegment + wroteSegment = writer.drainCachedSegments() || wroteSegment + + if (isDownloadComplete(session, targets)) { break } - if (wroteSegment) { + if (wroteSegment || segments.isNotEmpty()) { emptyResponses = 0 } else { emptyResponses++ @@ -168,8 +148,15 @@ internal class SabrDownloader( } } - private fun isRetryablePolicyOnly(error: SabrProtocolException): Boolean { - return error.message?.contains("repeated policy-only responses") == true + private fun isDownloadComplete( + session: YoutubeSabrSession, + targets: List, + ): Boolean { + return targets.all { target -> + target.pending.isEmpty() && + (session.streamState.isComplete(target.format) || + session.isBeyondEnd(SabrSegmentRequest.media(target.format, target.nextWriteSequence))) + } } private fun completeMission(finalBytes: Long) { diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt index 74d35c3ac..ec1f0a2af 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -59,17 +59,6 @@ internal class SabrSegmentWriter( return wroteSegment } - @Throws(IOException::class) - fun writeFetchedSegment(target: SabrDownloadTarget, segment: SabrMediaSegment) { - if (segment.header.isInitSegment) { - return - } - val request = SabrSegmentRequest.media(target.format, segment.header.sequenceNumber) - writeMediaSegment(target, outputs.getValue(target.resourceIndex), segment) - session.discardCachedSegment(request) - logWrittenSegment(target, segment) - } - @Throws(IOException::class) private fun writeDirectInitializationIfAvailable( target: SabrDownloadTarget, From 64ade2e38e04c8a0759225c29676624a3e46a1fb Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 1 Jul 2026 11:39:01 +0200 Subject: [PATCH 03/12] fix(download): keep SABR segment writes ordered --- .../java/us/shandian/giga/get/SabrDownloadTarget.kt | 1 - .../java/us/shandian/giga/get/SabrSegmentWriter.kt | 13 +------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt index 15f37dbfa..1a9a9ddda 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt @@ -9,7 +9,6 @@ internal data class SabrDownloadTarget( val recovery: MissionRecoveryInfo, val format: YoutubeSabrFormat, val file: File, - var nextRequestSequence: Int = 1, var nextWriteSequence: Int = 1, var initializationWritten: Boolean = false, val pending: TreeMap = TreeMap(), diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt index ec1f0a2af..db774dfcc 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -44,7 +44,7 @@ internal class SabrSegmentWriter( var wroteSegment = false for (target in targets) { while (true) { - val request = SabrSegmentRequest.media(target.format, target.nextRequestSequence) + val request = SabrSegmentRequest.media(target.format, target.nextWriteSequence) val segment = session.getCachedSegment(request) ?: break if (segment.header.isInitSegment) { session.discardCachedSegment(request) @@ -116,12 +116,10 @@ internal class SabrSegmentWriter( } if (!target.initializationWritten) { target.pending[sequence] = segment.data - advanceNextRequestSequence(target, sequence) return } if (sequence > target.nextWriteSequence) { target.pending[sequence] = segment.data - advanceNextRequestSequence(target, sequence) return } writeMediaBytes(target, output, segment.data) @@ -138,9 +136,6 @@ internal class SabrSegmentWriter( private fun writeMediaBytes(target: SabrDownloadTarget, output: OutputStream, data: ByteArray) { output.write(data) target.nextWriteSequence++ - if (target.nextRequestSequence < target.nextWriteSequence) { - target.nextRequestSequence = target.nextWriteSequence - } mission.notifyProgress(data.size.toLong()) } @@ -153,12 +148,6 @@ internal class SabrSegmentWriter( } } - private fun advanceNextRequestSequence(target: SabrDownloadTarget, observedSequence: Int) { - if (observedSequence >= target.nextRequestSequence) { - target.nextRequestSequence = observedSequence + 1 - } - } - private companion object { private const val TAG = "SabrSegmentWriter" private const val LOG_EVERY_SEGMENTS = 50 From 01046ec316fb5e846e29cdfa4f5e5c9e8a7373f0 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 1 Jul 2026 13:40:59 +0200 Subject: [PATCH 04/12] fix(download): stabilize SABR single-track downloads --- .../shandian/giga/get/SabrDownloadTarget.kt | 2 + .../us/shandian/giga/get/SabrDownloader.kt | 127 +++++++++++++----- .../us/shandian/giga/get/SabrSegmentWriter.kt | 27 ++-- 3 files changed, 104 insertions(+), 52 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt index 1a9a9ddda..d416767d7 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt @@ -11,5 +11,7 @@ internal data class SabrDownloadTarget( val file: File, var nextWriteSequence: Int = 1, var initializationWritten: Boolean = false, + var initializationObserved: Boolean = false, + var initializationData: ByteArray? = null, val pending: TreeMap = TreeMap(), ) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index 85594e680..755ef0f7a 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -1,9 +1,9 @@ package us.shandian.giga.get -import android.util.Log import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession import org.schabi.newpipe.player.datasource.WebViewPoTokenProvider import java.io.File @@ -13,7 +13,6 @@ internal class SabrDownloader( private val mission: DownloadMission, ) : Runnable { override fun run() { - Log.d(TAG, "local-sabr-run start urls=${mission.urls.size} running=${mission.running}") try { ensureRunning() val recoveries = validateRecoveryInfo() @@ -22,35 +21,20 @@ internal class SabrDownloader( val info = SabrDownloadFormatResolver.resolveInfo(recoveries) val audioRecovery = recoveries.firstOrNull { it.kind == 'a' } val videoRecovery = recoveries.firstOrNull { it.kind == 'v' } - val session = YoutubeSabrSession( - info, - SabrDownloadFormatResolver.selectedAudioFormat(info, recoveries), - SabrDownloadFormatResolver.selectedVideoFormat(info, recoveries), - WebViewPoTokenProvider(mission.context), - ) - configureRequestMode(session, audioRecovery, videoRecovery) - - val workDir = prepareWorkDirectory() - val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) - val outputs = targets.associate { target -> - target.resourceIndex to target.file.outputStream() - } - try { - downloadSegments(session, targets, SabrSegmentWriter(mission, session, targets, outputs)) - } finally { - outputs.values.forEach { output -> - try { - output.close() - } catch (ignored: Exception) { - // Nothing to do. + var attempts = 0 + while (true) { + try { + runSessionAttempt(info, recoveries, audioRecovery, videoRecovery) + break + } catch (error: RetryColdStartException) { + attempts++ + cleanup(mission) + if (attempts > MAX_COLD_START_RETRIES) { + throw IOException("SABR cold start did not provide initialization", error) } } } - - ensureRunning() - val finalBytes = SabrFfmpegMuxer(mission).remuxAndCopy(targets.map { it.file }, workDir) - completeMission(finalBytes) } catch (error: InterruptedException) { Thread.currentThread().interrupt() } catch (error: SabrProtocolException) { @@ -64,10 +48,47 @@ internal class SabrDownloader( } } + @Throws(IOException::class, InterruptedException::class, SabrProtocolException::class) + private fun runSessionAttempt( + info: YoutubeSabrInfo, + recoveries: Array, + audioRecovery: MissionRecoveryInfo?, + videoRecovery: MissionRecoveryInfo?, + ) { + val session = YoutubeSabrSession( + info, + SabrDownloadFormatResolver.selectedAudioFormat(info, recoveries), + SabrDownloadFormatResolver.selectedVideoFormat(info, recoveries), + WebViewPoTokenProvider(mission.context), + ) + configureRequestMode(session, audioRecovery, videoRecovery) + + val workDir = prepareWorkDirectory() + val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) + val outputs = targets.associate { target -> + target.resourceIndex to target.file.outputStream() + } + + try { + downloadSegments(session, targets, SabrSegmentWriter(mission, session, targets, outputs)) + } finally { + outputs.values.forEach { output -> + try { + output.close() + } catch (ignored: Exception) { + // Nothing to do. + } + } + } + + ensureRunning() + val finalBytes = SabrFfmpegMuxer(mission).remuxAndCopy(targets.map { it.file }, workDir) + completeMission(finalBytes) + } + @Throws(IOException::class) private fun validateRecoveryInfo(): Array { val recoveries = mission.recoveryInfo ?: throw IOException("Missing SABR recovery info") - Log.d(TAG, "local-sabr-run recoveries=${recoveries.size} sabr=${recoveries.count { it.isSabr }}") if (recoveries.size != mission.urls.size || recoveries.any { !it.isSabr }) { throw IOException("Mixed SABR/non-SABR missions are not supported") } @@ -91,11 +112,7 @@ internal class SabrDownloader( audioRecovery: MissionRecoveryInfo?, videoRecovery: MissionRecoveryInfo?, ) { - when { - videoRecovery == null -> session.streamState.setAudioOnlyRequestMode() - audioRecovery == null -> session.streamState.setVideoOnlyRequestMode() - else -> session.streamState.setVideoAndAudioRequestMode() - } + session.streamState.setVideoAndAudioRequestMode() } @Throws(IOException::class) @@ -105,7 +122,6 @@ internal class SabrDownloader( if (!workDir.mkdirs()) { throw IOException("Cannot create SABR work directory: $workDir") } - Log.d(TAG, "local-sabr-run workDir=$workDir") return workDir } @@ -117,21 +133,30 @@ internal class SabrDownloader( ) { val localization = Localization("en", "US") writer.writeDirectInitializations() + writer.observeWrittenInitializations() var emptyResponses = 0 while (true) { ensureRunning() + writer.observeWrittenInitializations() var wroteSegment = writer.drainCachedInitializations() wroteSegment = writer.drainCachedSegments() || wroteSegment + configureInitializedSingleTargetMode(session, targets) if (isDownloadComplete(session, targets)) { break } - session.streamState.setPlayerTimeMs(session.streamState.minBufferedEndMs) + val playerTimeMs = downloadPlayerTimeMs(session, targets) + session.streamState.setPlayerTimeMs(playerTimeMs) val segments = session.pumpOnce(localization) + writer.observeWrittenInitializations() wroteSegment = writer.drainCachedInitializations() || wroteSegment wroteSegment = writer.drainCachedSegments() || wroteSegment + configureInitializedSingleTargetMode(session, targets) + if (hasMediaWaitingForInitialization(targets)) { + throw RetryColdStartException() + } if (isDownloadComplete(session, targets)) { break @@ -148,6 +173,34 @@ internal class SabrDownloader( } } + private fun configureInitializedSingleTargetMode( + session: YoutubeSabrSession, + targets: List, + ) { + if (targets.size != 1 || !targets.first().initializationWritten) { + return + } + if (targets.first().format.isAudio) { + session.streamState.setAudioOnlyRequestMode() + } else { + session.streamState.setVideoOnlyRequestMode() + } + } + + private fun hasMediaWaitingForInitialization(targets: List): Boolean { + return targets.any { target -> !target.initializationWritten && target.pending.isNotEmpty() } + } + + private fun downloadPlayerTimeMs( + session: YoutubeSabrSession, + targets: List, + ): Long { + if (targets.size == 1) { + return session.streamState.getBufferedEndMs(targets.first().format) + } + return session.streamState.minBufferedEndMs + } + private fun isDownloadComplete( session: YoutubeSabrSession, targets: List, @@ -179,9 +232,9 @@ internal class SabrDownloader( } companion object { - private const val TAG = "SabrDownloader" private const val IDLE_POLL_MS = 250L private const val MAX_EMPTY_RESPONSES = 60 + private const val MAX_COLD_START_RETRIES = 3 @JvmStatic fun cleanup(mission: DownloadMission) { @@ -202,4 +255,6 @@ internal class SabrDownloader( return File(base, "sabr-downloader/$missionId") } } + + private class RetryColdStartException : IOException() } diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt index db774dfcc..7a29cf374 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -1,6 +1,5 @@ package us.shandian.giga.get -import android.util.Log import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest @@ -22,6 +21,16 @@ internal class SabrSegmentWriter( } } + fun observeWrittenInitializations() { + for (target in targets) { + val data = target.initializationData ?: continue + if (!target.initializationObserved) { + target.initializationObserved = + session.streamState.ingestInitializationData(target.format, data) + } + } + } + @Throws(IOException::class) fun drainCachedInitializations(): Boolean { var wroteInitialization = false @@ -33,7 +42,6 @@ internal class SabrSegmentWriter( val segment = session.getCachedSegment(request) ?: continue writeInitializationSegment(target, outputs.getValue(target.resourceIndex), segment.data) session.discardCachedSegment(request) - Log.d(TAG, "local-sabr-init itag=${target.format.itag} bytes=${segment.data.size}") wroteInitialization = true } return wroteInitialization @@ -52,7 +60,6 @@ internal class SabrSegmentWriter( } writeMediaSegment(target, outputs.getValue(target.resourceIndex), segment) session.discardCachedSegment(request) - logWrittenSegment(target, segment) wroteSegment = true } } @@ -79,6 +86,7 @@ internal class SabrSegmentWriter( } output.write(data) target.initializationWritten = true + target.initializationData = data mission.notifyProgress(data.size.toLong()) flushPendingMedia(target, output) return true @@ -139,17 +147,4 @@ internal class SabrSegmentWriter( mission.notifyProgress(data.size.toLong()) } - private fun logWrittenSegment(target: SabrDownloadTarget, segment: SabrMediaSegment) { - val sequence = segment.header.sequenceNumber - if (sequence <= 3 || sequence % LOG_EVERY_SEGMENTS == 0) { - Log.d(TAG, "local-sabr-write itag=${target.format.itag}" - + " seq=$sequence" - + " bytes=${segment.data.size}") - } - } - - private companion object { - private const val TAG = "SabrSegmentWriter" - private const val LOG_EVERY_SEGMENTS = 50 - } } From 4bf0b5d0480e5aed2af01c2f969b2baad4169592 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 1 Jul 2026 14:22:53 +0200 Subject: [PATCH 05/12] Clean up SABR download handling --- .../newpipe/download/DownloadDialog.java | 25 --------------- .../us/shandian/giga/get/DownloadMission.java | 9 ------ .../us/shandian/giga/get/SabrDownloader.kt | 31 +++++++++---------- 3 files changed, 14 insertions(+), 51 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index e1d56fab8..3fe6779f3 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -341,7 +341,6 @@ public void onViewCreated(@NonNull final View view, @Nullable final Bundle saved initToolbar(dialogBinding.toolbarLayout.toolbar); setupDownloadOptions(); - dumpDownloadStreamsForLocalTest(); prefs = PreferenceManager.getDefaultSharedPreferences(requireContext()); @@ -362,30 +361,6 @@ public void onProgressChanged(@NonNull final SeekBar seekbar, final int progress fetchStreamsSize(); } - private void dumpDownloadStreamsForLocalTest() { - for (int i = 0; i < wrappedVideoStreams.getStreamsList().size(); i++) { - final VideoStream stream = wrappedVideoStreams.getStreamsList().get(i); - Log.d(TAG, "local-download-stream video index=" + i - + " delivery=" + stream.getDeliveryMethod() - + " format=" + stream.getFormat() - + " codec=" + stream.getCodec() - + " resolution=" + stream.getResolution() - + " itag=" + stream.getItag() - + " videoOnly=" + stream.isVideoOnly() - + " audioTrackId=" + stream.getAudioTrackId()); - } - for (int i = 0; i < wrappedAudioStreams.getStreamsList().size(); i++) { - final AudioStream stream = wrappedAudioStreams.getStreamsList().get(i); - Log.d(TAG, "local-download-stream audio index=" + i - + " delivery=" + stream.getDeliveryMethod() - + " format=" + stream.getFormat() - + " codec=" + stream.getCodec() - + " quality=" + stream.getQuality() - + " itag=" + stream.getItag() - + " audioTrackId=" + stream.getAudioTrackId()); - } - } - private void initToolbar(final Toolbar toolbar) { if (DEBUG) { Log.d(TAG, "initToolbar() called with: toolbar = [" + toolbar + "]"); diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 02ab94bb3..dd20f657c 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -438,13 +438,6 @@ private void notifyPostProcessing(int state) { * Start downloading with multiple threads. */ public void start() { - Log.d(TAG, "local-download-start running=" + running - + " finished=" + isFinished() - + " urls=" + urls.length - + " current=" + current - + " hasSabr=" + hasSabrResource() - + " hasHls=" + hasHlsResource() - + " sabrStarted=" + sabrStarted); if (running || isFinished() || urls.length < 1) return; // ensure that the previous state is completely paused. @@ -471,13 +464,11 @@ public void start() { } if (hasSabrResource()) { - Log.d(TAG, "local-download-route SABR"); init = runAsync(DownloadInitializer.mId, new SabrDownloader(this)); return; } if (hasHlsResource()) { - Log.d(TAG, "local-download-route HLS"); init = runAsync(DownloadInitializer.mId, new HlsDownloader(this)); return; } diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index 755ef0f7a..d2b0a8982 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -19,13 +19,11 @@ internal class SabrDownloader( prepareMission() val info = SabrDownloadFormatResolver.resolveInfo(recoveries) - val audioRecovery = recoveries.firstOrNull { it.kind == 'a' } - val videoRecovery = recoveries.firstOrNull { it.kind == 'v' } var attempts = 0 while (true) { try { - runSessionAttempt(info, recoveries, audioRecovery, videoRecovery) + runSessionAttempt(info, recoveries) break } catch (error: RetryColdStartException) { attempts++ @@ -38,13 +36,9 @@ internal class SabrDownloader( } catch (error: InterruptedException) { Thread.currentThread().interrupt() } catch (error: SabrProtocolException) { - if (mission.running) { - mission.notifyError(IOException(error)) - } + notifyErrorAndCleanup(IOException(error)) } catch (error: Exception) { - if (mission.running) { - mission.notifyError(error) - } + notifyErrorAndCleanup(error) } } @@ -52,8 +46,6 @@ internal class SabrDownloader( private fun runSessionAttempt( info: YoutubeSabrInfo, recoveries: Array, - audioRecovery: MissionRecoveryInfo?, - videoRecovery: MissionRecoveryInfo?, ) { val session = YoutubeSabrSession( info, @@ -61,7 +53,7 @@ internal class SabrDownloader( SabrDownloadFormatResolver.selectedVideoFormat(info, recoveries), WebViewPoTokenProvider(mission.context), ) - configureRequestMode(session, audioRecovery, videoRecovery) + configureRequestMode(session) val workDir = prepareWorkDirectory() val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) @@ -107,11 +99,9 @@ internal class SabrDownloader( mission.writeThisToFile() } - private fun configureRequestMode( - session: YoutubeSabrSession, - audioRecovery: MissionRecoveryInfo?, - videoRecovery: MissionRecoveryInfo?, - ) { + private fun configureRequestMode(session: YoutubeSabrSession) { + // Cold starts are most reliable in the normal two-track mode; for single-track downloads we + // switch to audio-only/video-only once the selected track initialization has been written. session.streamState.setVideoAndAudioRequestMode() } @@ -231,6 +221,13 @@ internal class SabrDownloader( } } + private fun notifyErrorAndCleanup(error: Exception) { + cleanup(mission) + if (mission.running) { + mission.notifyError(error) + } + } + companion object { private const val IDLE_POLL_MS = 250L private const val MAX_EMPTY_RESPONSES = 60 From 205553e458edcd667729fb77d22c74c492ff6d07 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 1 Jul 2026 16:04:43 +0200 Subject: [PATCH 06/12] Improve SABR download resilience --- .../us/shandian/giga/get/DownloadMission.java | 3 + .../giga/get/SabrDownloadException.kt | 20 +++ .../giga/get/SabrDownloadFormatResolver.kt | 64 ++++++++- .../shandian/giga/get/SabrDownloadTarget.kt | 1 + .../us/shandian/giga/get/SabrDownloader.kt | 136 ++++++++++++++++-- .../us/shandian/giga/get/SabrFfmpegMuxer.kt | 119 +++++++++++---- .../us/shandian/giga/get/SabrSegmentWriter.kt | 64 ++++++++- .../giga/ui/adapter/MissionAdapter.java | 34 +++++ app/src/main/res/values/strings.xml | 9 ++ 9 files changed, 397 insertions(+), 53 deletions(-) create mode 100644 app/src/main/java/us/shandian/giga/get/SabrDownloadException.kt diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index dd20f657c..98f8e0f5a 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -47,6 +47,7 @@ public class DownloadMission extends Mission { public static final int ERROR_PROGRESS_LOST = 1011; public static final int ERROR_TIMEOUT = 1012; public static final int ERROR_RESOURCE_GONE = 1013; + public static final int ERROR_SABR_DOWNLOAD = 1014; public static final int ERROR_HTTP_NO_CONTENT = 204; static final int ERROR_HTTP_FORBIDDEN = 403; static final int ERROR_HTTP_AUTH = 401; @@ -319,6 +320,8 @@ synchronized void notifyError(Exception err) { notifyError(ERROR_UNKNOWN_HOST, null); } else if (err instanceof SocketTimeoutException) { notifyError(ERROR_TIMEOUT, null); + } else if (err instanceof SabrDownloadException) { + notifyError(ERROR_SABR_DOWNLOAD, err); } else { notifyError(ERROR_UNKNOWN_EXCEPTION, err); } diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadException.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadException.kt new file mode 100644 index 000000000..3bac566ba --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadException.kt @@ -0,0 +1,20 @@ +package us.shandian.giga.get + +import java.io.IOException + +internal class SabrDownloadException( + val reason: Reason, + message: String, + cause: Throwable? = null, +) : IOException(message, cause) { + enum class Reason { + FORMAT, + INITIALIZATION, + PROTECTED, + STALLED, + NETWORK, + MUXING, + STORAGE, + PROTOCOL, + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt index d945a02e7..4336a5602 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt @@ -20,8 +20,16 @@ internal object SabrDownloadFormatResolver { ): YoutubeSabrFormat { val audioRecovery = recoveries.firstOrNull { it.kind == 'a' } return audioRecovery?.let { findAudioFormat(info, it) } + ?: if (recoveries.any { it.kind == 'v' }) { + findLightweightAudioFormat(info) + } else { + null + } ?: info.findBestAudioFormat() - ?: throw IOException("Missing SABR audio format") + ?: throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: missing audio format", + ) } @Throws(IOException::class) @@ -31,8 +39,16 @@ internal object SabrDownloadFormatResolver { ): YoutubeSabrFormat { val videoRecovery = recoveries.firstOrNull { it.kind == 'v' } return videoRecovery?.let { findVideoFormat(info, it) } + ?: if (recoveries.any { it.kind == 'a' }) { + findLightweightVideoFormat(info) + } else { + null + } ?: info.findBestVideoFormat() - ?: throw IOException("Missing SABR video format") + ?: throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: missing video format", + ) } @Throws(IOException::class) @@ -45,7 +61,10 @@ internal object SabrDownloadFormatResolver { val format = when (recovery.kind) { 'a' -> findAudioFormat(info, recovery) 'v' -> findVideoFormat(info, recovery) - else -> throw IOException("Unsupported SABR resource kind: ${recovery.kind}") + else -> throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: unsupported resource kind ${recovery.kind}", + ) } SabrDownloadTarget(index, recovery, format, File(workDir, "input-$index.media")) } @@ -60,7 +79,10 @@ internal object SabrDownloadFormatResolver { format.isAudio && (recovery.itag <= 0 || format.itag == recovery.itag) && (recovery.audioTrackId == null || recovery.audioTrackId == format.audioTrackId) - } ?: throw IOException("Could not resolve SABR audio format: itag=${recovery.itag}") + } ?: throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: could not resolve audio itag ${recovery.itag}", + ) } @Throws(IOException::class) @@ -70,6 +92,38 @@ internal object SabrDownloadFormatResolver { ): YoutubeSabrFormat { return info.formats.firstOrNull { format -> format.isVideo && (recovery.itag <= 0 || format.itag == recovery.itag) - } ?: throw IOException("Could not resolve SABR video format: itag=${recovery.itag}") + } ?: throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: could not resolve video itag ${recovery.itag}", + ) + } + + private fun findLightweightAudioFormat(info: YoutubeSabrInfo): YoutubeSabrFormat? { + return info.formats + .filter { it.isAudio } + .sortedWith( + compareBy { !it.isOriginalAudio } + .thenBy { it.isDrc } + .thenBy { normalizedBitrate(it) }, + ) + .firstOrNull() + } + + private fun findLightweightVideoFormat(info: YoutubeSabrInfo): YoutubeSabrFormat? { + return info.formats + .filter { it.isVideo } + .sortedWith( + compareBy { normalizedHeight(it) } + .thenBy { normalizedBitrate(it) }, + ) + .firstOrNull() + } + + private fun normalizedBitrate(format: YoutubeSabrFormat): Int { + return format.bitrate.takeIf { it > 0 } ?: Int.MAX_VALUE + } + + private fun normalizedHeight(format: YoutubeSabrFormat): Int { + return format.height.takeIf { it > 0 } ?: Int.MAX_VALUE } } diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt index d416767d7..fd0efc394 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt @@ -14,4 +14,5 @@ internal data class SabrDownloadTarget( var initializationObserved: Boolean = false, var initializationData: ByteArray? = null, val pending: TreeMap = TreeMap(), + var pendingBytes: Long = 0, ) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index d2b0a8982..6df3aec52 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -1,5 +1,7 @@ package us.shandian.giga.get +import android.util.Log +import org.schabi.newpipe.BuildConfig import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest @@ -8,6 +10,9 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession import org.schabi.newpipe.player.datasource.WebViewPoTokenProvider import java.io.File import java.io.IOException +import java.net.ConnectException +import java.net.SocketTimeoutException +import java.net.UnknownHostException internal class SabrDownloader( private val mission: DownloadMission, @@ -16,27 +21,47 @@ internal class SabrDownloader( try { ensureRunning() val recoveries = validateRecoveryInfo() - prepareMission() - val info = SabrDownloadFormatResolver.resolveInfo(recoveries) - var attempts = 0 + var coldStartAttempts = 0 + var transientAttempts = 0 while (true) { try { + prepareMission() runSessionAttempt(info, recoveries) break } catch (error: RetryColdStartException) { - attempts++ + coldStartAttempts++ cleanup(mission) - if (attempts > MAX_COLD_START_RETRIES) { - throw IOException("SABR cold start did not provide initialization", error) + if (coldStartAttempts > MAX_COLD_START_RETRIES) { + throw SabrDownloadException( + SabrDownloadException.Reason.INITIALIZATION, + "SABR download failed: cold start did not provide initialization", + error, + ) + } + logDebug("retry cold start attempt=$coldStartAttempts") + } catch (error: Exception) { + if (!isRetryableAttemptFailure(error)) { + throw error + } + if (transientAttempts >= MAX_TRANSIENT_RETRIES) { + throw SabrDownloadException( + SabrDownloadException.Reason.NETWORK, + "SABR download failed: network error after retries", + error, + ) } + transientAttempts++ + cleanup(mission) + logDebug("retry transient attempt=$transientAttempts error=${error.javaClass.simpleName}") + Thread.sleep(transientRetryDelayMs(transientAttempts)) } } } catch (error: InterruptedException) { Thread.currentThread().interrupt() } catch (error: SabrProtocolException) { - notifyErrorAndCleanup(IOException(error)) + notifyErrorAndCleanup(classifyProtocolException(error)) } catch (error: Exception) { notifyErrorAndCleanup(error) } @@ -57,9 +82,7 @@ internal class SabrDownloader( val workDir = prepareWorkDirectory() val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) - val outputs = targets.associate { target -> - target.resourceIndex to target.file.outputStream() - } + val outputs = targets.associate { target -> target.resourceIndex to target.file.outputStream() } try { downloadSegments(session, targets, SabrSegmentWriter(mission, session, targets, outputs)) @@ -71,10 +94,15 @@ internal class SabrDownloader( // Nothing to do. } } + session.clearCache() } ensureRunning() - val finalBytes = SabrFfmpegMuxer(mission).remuxAndCopy(targets.map { it.file }, workDir) + val finalBytes = SabrFfmpegMuxer(mission).remuxAndCopy( + targets.map { it.file }, + targets, + workDir, + ) completeMission(finalBytes) } @@ -82,7 +110,10 @@ internal class SabrDownloader( private fun validateRecoveryInfo(): Array { val recoveries = mission.recoveryInfo ?: throw IOException("Missing SABR recovery info") if (recoveries.size != mission.urls.size || recoveries.any { !it.isSabr }) { - throw IOException("Mixed SABR/non-SABR missions are not supported") + throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: mixed SABR/non-SABR resources are not supported", + ) } return recoveries } @@ -143,9 +174,17 @@ internal class SabrDownloader( writer.observeWrittenInitializations() wroteSegment = writer.drainCachedInitializations() || wroteSegment wroteSegment = writer.drainCachedSegments() || wroteSegment + enforceSessionCacheLimit(session, writer) configureInitializedSingleTargetMode(session, targets) if (hasMediaWaitingForInitialization(targets)) { - throw RetryColdStartException() + writer.fetchMissingInitializations(localization) + writer.observeWrittenInitializations() + wroteSegment = writer.drainCachedInitializations() || wroteSegment + wroteSegment = writer.drainCachedSegments() || wroteSegment + configureInitializedSingleTargetMode(session, targets) + if (hasMediaWaitingForInitialization(targets)) { + throw RetryColdStartException() + } } if (isDownloadComplete(session, targets)) { @@ -156,13 +195,34 @@ internal class SabrDownloader( } else { emptyResponses++ if (emptyResponses > MAX_EMPTY_RESPONSES) { - throw IOException("SABR download stalled with no media") + throw SabrDownloadException( + SabrDownloadException.Reason.STALLED, + "SABR download stalled: no media received after $MAX_EMPTY_RESPONSES rounds", + ) } Thread.sleep(IDLE_POLL_MS) } } } + @Throws(IOException::class) + private fun enforceSessionCacheLimit( + session: YoutubeSabrSession, + writer: SabrSegmentWriter, + ) { + if (session.cachedBytes <= MAX_SESSION_CACHE_BYTES) { + return + } + writer.drainCachedSegments() + if (session.cachedBytes <= MAX_SESSION_CACHE_BYTES) { + return + } + throw SabrDownloadException( + SabrDownloadException.Reason.STALLED, + "SABR download stalled: cached media grew to ${session.cachedBytes} bytes", + ) + } + private fun configureInitializedSingleTargetMode( session: YoutubeSabrSession, targets: List, @@ -228,10 +288,58 @@ internal class SabrDownloader( } } + private fun isRetryableAttemptFailure(error: Exception): Boolean { + if (error is RetryColdStartException || error is SabrDownloadException) { + return false + } + if (error is SabrProtocolException) { + return false + } + return error is SocketTimeoutException || + error is ConnectException || + error is UnknownHostException || + error is IOException + } + + private fun transientRetryDelayMs(attempt: Int): Long { + return (500L shl (attempt - 1)).coerceAtMost(MAX_TRANSIENT_RETRY_DELAY_MS) + } + + private fun classifyProtocolException(error: SabrProtocolException): SabrDownloadException { + val message = error.message.orEmpty() + val reason = when { + message.contains("protected", ignoreCase = true) || + message.contains("PO token", ignoreCase = true) -> { + SabrDownloadException.Reason.PROTECTED + } + message.contains("policy-only", ignoreCase = true) || + message.contains("not returned", ignoreCase = true) || + message.contains("integrity", ignoreCase = true) -> { + SabrDownloadException.Reason.STALLED + } + else -> SabrDownloadException.Reason.PROTOCOL + } + return SabrDownloadException( + reason, + "SABR download failed: ${message.ifBlank { "protocol error" }}", + error, + ) + } + + private fun logDebug(message: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, message) + } + } + companion object { + private const val TAG = "SabrDownloader" private const val IDLE_POLL_MS = 250L private const val MAX_EMPTY_RESPONSES = 60 private const val MAX_COLD_START_RETRIES = 3 + private const val MAX_TRANSIENT_RETRIES = 2 + private const val MAX_TRANSIENT_RETRY_DELAY_MS = 2_000L + private const val MAX_SESSION_CACHE_BYTES = 48L * 1024L * 1024L @JvmStatic fun cleanup(mission: DownloadMission) { diff --git a/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt index 3c96474f0..7b4e611da 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt @@ -3,6 +3,7 @@ package us.shandian.giga.get import android.util.Log import com.arthenica.ffmpegkit.FFmpegKit import com.arthenica.ffmpegkit.ReturnCode +import org.schabi.newpipe.BuildConfig import java.io.File import java.io.IOException @@ -10,61 +11,97 @@ internal class SabrFfmpegMuxer( private val mission: DownloadMission, ) { @Throws(IOException::class, InterruptedException::class) - fun remuxAndCopy(inputs: List, workDir: File): Long { + fun remuxAndCopy(inputs: List, targets: List, workDir: File): Long { + if (canCopySingleInputDirectly(inputs, targets)) { + logDebug("copySingleInputDirectly input=${inputs.first().length()}") + return try { + copyOutputToStorage(inputs.first()) + } finally { + deleteQuietly(inputs.first()) + } + } val output = File(workDir, "output.${outputExtension()}") remuxWithFfmpeg(inputs, output) - return copyOutputToStorage(output) + inputs.forEach(::deleteQuietly) + return try { + copyOutputToStorage(output) + } finally { + deleteQuietly(output) + } } @Throws(IOException::class) private fun remuxWithFfmpeg(inputs: List, output: File) { mission.psState = 1 mission.writeThisToFile() - val command = buildString { - append("-y ") + val command = buildList { + add("-hide_banner") + add("-nostats") + add("-loglevel") + add("error") + add("-y") inputs.forEach { input -> - append("-i ").append(quote(input.absolutePath)).append(' ') + add("-i") + add(input.absolutePath) } if (mission.kind == 'a' && inputs.size == 1) { - append("-map 0:a? -vn ") + add("-map") + add("0:a?") + add("-vn") } else if (inputs.size > 1) { inputs.indices.forEach { index -> - append("-map ").append(index).append(":v? ") - append("-map ").append(index).append(":a? ") + add("-map") + add("$index:v?") + add("-map") + add("$index:a?") } } - append("-c copy ") + add("-c") + add("copy") if (supportsFastStart(output)) { - append("-movflags +faststart ") + add("-movflags") + add("+faststart") } - append(quote(output.absolutePath)) + add(output.absolutePath) } - Log.d(TAG, "remuxWithFfmpeg inputs=${inputs.size}") - val session = FFmpegKit.execute(command) + logDebug("remuxWithFfmpeg inputs=${inputs.size} output=${output.name}") + val session = FFmpegKit.executeWithArguments(command.toTypedArray()) if (!ReturnCode.isSuccess(session.returnCode)) { mission.psState = 0 - throw IOException("SABR ffmpeg remux failed: ${session.returnCode}") + throw SabrDownloadException( + SabrDownloadException.Reason.MUXING, + "SABR download failed: ffmpeg remux failed (${session.returnCode})" + + session.output.takeIf { it.isNotBlank() }?.let { ": $it" }.orEmpty(), + ) } } @Throws(IOException::class, InterruptedException::class) private fun copyOutputToStorage(output: File): Long { var copied = 0L - output.inputStream().use { input -> - mission.storage.getStream().use { storage -> - storage.setLength(0) - storage.seek(0) - val buffer = ByteArray(DownloadMission.BUFFER_SIZE) - while (true) { - ensureRunning() - val read = input.read(buffer) - if (read == -1) { - break + try { + output.inputStream().use { input -> + mission.storage.getStream().use { storage -> + storage.setLength(0) + storage.seek(0) + val buffer = ByteArray(DownloadMission.BUFFER_SIZE) + while (true) { + ensureRunning() + val read = input.read(buffer) + if (read == -1) { + break + } + storage.write(buffer, 0, read) + copied += read.toLong() } - storage.write(buffer, 0, read) - copied += read.toLong() } } + } catch (error: IOException) { + throw SabrDownloadException( + SabrDownloadException.Reason.STORAGE, + "SABR download failed: could not write final file", + error, + ) } return copied } @@ -76,10 +113,6 @@ internal class SabrFfmpegMuxer( } } - private fun quote(value: String): String { - return '"' + value.replace("\\", "\\\\").replace("\"", "\\\"") + '"' - } - private fun outputExtension(): String { val name = mission.storage.name ?: return "mp4" val extension = name.substringAfterLast('.', missingDelimiterValue = "") @@ -93,6 +126,32 @@ internal class SabrFfmpegMuxer( } } + private fun canCopySingleInputDirectly( + inputs: List, + targets: List, + ): Boolean { + if (inputs.size != 1 || targets.size != 1) { + return false + } + val extension = outputExtension().lowercase() + val mimeType = targets.first().format.mimeType ?: return false + return extension == "webm" && mimeType.contains("webm", ignoreCase = true) + } + + private fun logDebug(message: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, message) + } + } + + private fun deleteQuietly(file: File) { + try { + file.delete() + } catch (ignored: Exception) { + // Nothing to do. + } + } + private companion object { private const val TAG = "SabrFfmpegMuxer" } diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt index 7a29cf374..f28102d13 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -1,6 +1,7 @@ package us.shandian.giga.get import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat @@ -66,6 +67,22 @@ internal class SabrSegmentWriter( return wroteSegment } + @Throws(IOException::class) + fun fetchMissingInitializations(localization: Localization): Boolean { + var wroteInitialization = false + for (target in targets) { + if (target.initializationWritten || target.pending.isEmpty()) { + continue + } + val request = SabrSegmentRequest.initialization(target.format) + val segment = session.fetchSegment(request, localization) + writeInitializationSegment(target, outputs.getValue(target.resourceIndex), segment.data) + session.discardCachedSegment(request) + wroteInitialization = true + } + return wroteInitialization + } + @Throws(IOException::class) private fun writeDirectInitializationIfAvailable( target: SabrDownloadTarget, @@ -104,10 +121,22 @@ internal class SabrSegmentWriter( val response = NewPipe.getDownloader().get(url, mapOf("Range" to listOf(range))) val data = response.rawResponseBody() if (response.responseCode() != 206 && response.responseCode() != 200) { - throw IOException("Could not fetch SABR init for itag=${format.itag}: HTTP ${response.responseCode()}") + if (response.responseCode() >= 500) { + throw IOException( + "SABR initialization request failed: HTTP ${response.responseCode()}", + ) + } + throw SabrDownloadException( + SabrDownloadException.Reason.INITIALIZATION, + "SABR download failed: could not fetch initialization for itag ${format.itag}" + + " (HTTP ${response.responseCode()})", + ) } if (data == null || data.isEmpty()) { - throw IOException("Empty SABR init for itag=${format.itag}") + throw SabrDownloadException( + SabrDownloadException.Reason.INITIALIZATION, + "SABR download failed: empty initialization for itag ${format.itag}", + ) } return data } @@ -123,11 +152,11 @@ internal class SabrSegmentWriter( return } if (!target.initializationWritten) { - target.pending[sequence] = segment.data + cachePendingMedia(target, sequence, segment.data, "waiting for initialization") return } if (sequence > target.nextWriteSequence) { - target.pending[sequence] = segment.data + cachePendingMedia(target, sequence, segment.data, "waiting for sequence ${target.nextWriteSequence}") return } writeMediaBytes(target, output, segment.data) @@ -137,14 +166,41 @@ internal class SabrSegmentWriter( private fun flushPendingMedia(target: SabrDownloadTarget, output: OutputStream) { while (true) { val pending = target.pending.remove(target.nextWriteSequence) ?: return + target.pendingBytes = (target.pendingBytes - pending.size).coerceAtLeast(0) writeMediaBytes(target, output, pending) } } + private fun cachePendingMedia( + target: SabrDownloadTarget, + sequence: Int, + data: ByteArray, + reason: String, + ) { + val previous = target.pending.put(sequence, data) + if (previous != null) { + target.pendingBytes -= previous.size.toLong() + } + target.pendingBytes += data.size.toLong() + if (target.pending.size > MAX_PENDING_SEGMENTS + || target.pendingBytes > MAX_PENDING_BYTES + ) { + throw SabrDownloadException( + SabrDownloadException.Reason.STALLED, + "SABR download stalled while writing itag ${target.format.itag}: $reason" + + " (${target.pending.size} pending segments, ${target.pendingBytes} bytes)", + ) + } + } + private fun writeMediaBytes(target: SabrDownloadTarget, output: OutputStream, data: ByteArray) { output.write(data) target.nextWriteSequence++ mission.notifyProgress(data.size.toLong()) } + private companion object { + private const val MAX_PENDING_SEGMENTS = 64 + private const val MAX_PENDING_BYTES = 24L * 1024L * 1024L + } } diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 20b8bb7f9..70ec5bfa0 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -65,6 +65,7 @@ import us.shandian.giga.get.FinishedMission; import us.shandian.giga.get.Mission; import us.shandian.giga.get.MissionRecoveryInfo; +import us.shandian.giga.get.SabrDownloadException; import org.schabi.newpipe.streams.io.StoredFileHelper; import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; @@ -88,6 +89,7 @@ import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_STOPPED; import static us.shandian.giga.get.DownloadMission.ERROR_PROGRESS_LOST; import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; +import static us.shandian.giga.get.DownloadMission.ERROR_SABR_DOWNLOAD; import static us.shandian.giga.get.DownloadMission.ERROR_SSL_EXCEPTION; import static us.shandian.giga.get.DownloadMission.ERROR_TIMEOUT; import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; @@ -547,6 +549,12 @@ private void showError(@NonNull DownloadMission mission) { case ERROR_RESOURCE_GONE: msg = R.string.error_download_resource_gone; break; + case ERROR_SABR_DOWNLOAD: + msg = sabrDownloadErrorMessage(mission.errObject); + if (mission.errObject != null && mission.errObject.getMessage() != null) { + msgEx = mission.errObject.getMessage(); + } + break; default: if (mission.errCode >= 100 && mission.errCode < 600) { msgEx = "HTTP " + mission.errCode; @@ -580,6 +588,32 @@ private void showError(@NonNull DownloadMission mission) { .show(); } + @StringRes + private int sabrDownloadErrorMessage(final Exception error) { + if (!(error instanceof SabrDownloadException)) { + return R.string.error_sabr_download_failed; + } + switch (((SabrDownloadException) error).getReason()) { + case FORMAT: + return R.string.error_sabr_download_format; + case INITIALIZATION: + return R.string.error_sabr_download_initialization; + case PROTECTED: + return R.string.error_sabr_download_protected; + case STALLED: + return R.string.error_sabr_download_stalled; + case NETWORK: + return R.string.error_sabr_download_network; + case MUXING: + return R.string.error_sabr_download_muxing; + case STORAGE: + return R.string.error_sabr_download_storage; + case PROTOCOL: + default: + return R.string.error_sabr_download_protocol; + } + } + private void showError(DownloadMission mission, UserAction action, @StringRes int reason) { StringBuilder request = new StringBuilder(256); request.append(mission.source); diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 3a8368d9c..6092917d7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -571,6 +571,15 @@ Progress lost, because the file was deleted Connection timeout Cannot recover this download +SABR download failed +This SABR format cannot be downloaded +SABR initialization failed +This SABR stream needs attestation +SABR stopped returning media +SABR download failed because of the network +Could not mux SABR media +Could not write the SABR download +SABR protocol error Clear download history Do you want to clear your download history or delete all downloaded files? Delete downloaded files From ca791e91beac9607b711d74fcf881d1c2c00e712 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 1 Jul 2026 17:04:35 +0200 Subject: [PATCH 07/12] fix(download): harden SABR single-track recovery --- .../us/shandian/giga/get/SabrDownloader.kt | 77 ++++++++++++++++--- .../us/shandian/giga/get/SabrFfmpegMuxer.kt | 2 +- .../us/shandian/giga/get/SabrSegmentWriter.kt | 27 ++++++- 3 files changed, 90 insertions(+), 16 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index 6df3aec52..8e1e5c53c 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -28,7 +28,7 @@ internal class SabrDownloader( while (true) { try { prepareMission() - runSessionAttempt(info, recoveries) + runSessionAttempt(info, recoveries, coldStartAttempts) break } catch (error: RetryColdStartException) { coldStartAttempts++ @@ -71,6 +71,7 @@ internal class SabrDownloader( private fun runSessionAttempt( info: YoutubeSabrInfo, recoveries: Array, + coldStartAttempt: Int, ) { val session = YoutubeSabrSession( info, @@ -78,10 +79,9 @@ internal class SabrDownloader( SabrDownloadFormatResolver.selectedVideoFormat(info, recoveries), WebViewPoTokenProvider(mission.context), ) - configureRequestMode(session) - val workDir = prepareWorkDirectory() val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) + configureRequestMode(session, targets, coldStartAttempt) val outputs = targets.associate { target -> target.resourceIndex to target.file.outputStream() } try { @@ -130,10 +130,21 @@ internal class SabrDownloader( mission.writeThisToFile() } - private fun configureRequestMode(session: YoutubeSabrSession) { - // Cold starts are most reliable in the normal two-track mode; for single-track downloads we - // switch to audio-only/video-only once the selected track initialization has been written. - session.streamState.setVideoAndAudioRequestMode() + private fun configureRequestMode( + session: YoutubeSabrSession, + targets: List, + coldStartAttempt: Int, + ) { + val useCompanionWarmup = targets.size == 1 && coldStartAttempt % 2 == 1 + if (useCompanionWarmup) { + session.streamState.setVideoAndAudioRequestMode() + } else if (targets.size == 1 && targets.first().format.isAudio) { + session.streamState.setAudioOnlyRequestMode() + } else if (targets.size == 1 && targets.first().format.isVideo) { + session.streamState.setVideoOnlyRequestMode() + } else { + session.streamState.setVideoAndAudioRequestMode() + } } @Throws(IOException::class) @@ -155,6 +166,11 @@ internal class SabrDownloader( val localization = Localization("en", "US") writer.writeDirectInitializations() writer.observeWrittenInitializations() + if (targets.size == 1 && !targets.first().initializationWritten) { + fetchInitializationsOrRetry(writer, localization) + writer.observeWrittenInitializations() + writer.drainCachedInitializations() + } var emptyResponses = 0 while (true) { @@ -177,7 +193,7 @@ internal class SabrDownloader( enforceSessionCacheLimit(session, writer) configureInitializedSingleTargetMode(session, targets) if (hasMediaWaitingForInitialization(targets)) { - writer.fetchMissingInitializations(localization) + fetchMissingInitializationsOrRetry(writer, localization) writer.observeWrittenInitializations() wroteSegment = writer.drainCachedInitializations() || wroteSegment wroteSegment = writer.drainCachedSegments() || wroteSegment @@ -205,6 +221,36 @@ internal class SabrDownloader( } } + @Throws(IOException::class) + private fun fetchInitializationsOrRetry( + writer: SabrSegmentWriter, + localization: Localization, + ) { + try { + writer.fetchUnwrittenInitializations(localization) + } catch (error: SabrProtocolException) { + if (isRetryableInitializationProtocolError(error)) { + throw RetryColdStartException(error) + } + throw error + } + } + + @Throws(IOException::class) + private fun fetchMissingInitializationsOrRetry( + writer: SabrSegmentWriter, + localization: Localization, + ) { + try { + writer.fetchMissingInitializations(localization) + } catch (error: SabrProtocolException) { + if (isRetryableInitializationProtocolError(error)) { + throw RetryColdStartException(error) + } + throw error + } + } + @Throws(IOException::class) private fun enforceSessionCacheLimit( session: YoutubeSabrSession, @@ -326,6 +372,15 @@ internal class SabrDownloader( ) } + private fun isRetryableInitializationProtocolError(error: SabrProtocolException): Boolean { + val message = error.message.orEmpty() + if (!message.contains(":init")) { + return false + } + return message.contains("policy-only", ignoreCase = true) || + message.contains("not returned", ignoreCase = true) + } + private fun logDebug(message: String) { if (BuildConfig.DEBUG) { Log.d(TAG, message) @@ -337,8 +392,8 @@ internal class SabrDownloader( private const val IDLE_POLL_MS = 250L private const val MAX_EMPTY_RESPONSES = 60 private const val MAX_COLD_START_RETRIES = 3 - private const val MAX_TRANSIENT_RETRIES = 2 - private const val MAX_TRANSIENT_RETRY_DELAY_MS = 2_000L + private const val MAX_TRANSIENT_RETRIES = 5 + private const val MAX_TRANSIENT_RETRY_DELAY_MS = 5_000L private const val MAX_SESSION_CACHE_BYTES = 48L * 1024L * 1024L @JvmStatic @@ -361,5 +416,5 @@ internal class SabrDownloader( } } - private class RetryColdStartException : IOException() + private class RetryColdStartException(cause: Throwable? = null) : IOException(cause) } diff --git a/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt index 7b4e611da..7c7198e1b 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt @@ -38,7 +38,7 @@ internal class SabrFfmpegMuxer( add("-hide_banner") add("-nostats") add("-loglevel") - add("error") + add("fatal") add("-y") inputs.forEach { input -> add("-i") diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt index f28102d13..241c0f622 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -69,9 +69,28 @@ internal class SabrSegmentWriter( @Throws(IOException::class) fun fetchMissingInitializations(localization: Localization): Boolean { + return fetchInitializations(localization, onlyWhenMediaIsPending = true) + } + + @Throws(IOException::class) + fun fetchUnwrittenInitializations(localization: Localization): Boolean { + return fetchInitializations(localization, onlyWhenMediaIsPending = false) + } + + @Throws(IOException::class) + private fun fetchInitializations( + localization: Localization, + onlyWhenMediaIsPending: Boolean, + ): Boolean { var wroteInitialization = false for (target in targets) { - if (target.initializationWritten || target.pending.isEmpty()) { + if (target.initializationWritten || + (onlyWhenMediaIsPending && target.pending.isEmpty()) + ) { + continue + } + if (writeDirectInitializationIfAvailable(target, outputs.getValue(target.resourceIndex))) { + wroteInitialization = true continue } val request = SabrSegmentRequest.initialization(target.format) @@ -87,9 +106,9 @@ internal class SabrSegmentWriter( private fun writeDirectInitializationIfAvailable( target: SabrDownloadTarget, output: OutputStream, - ) { - val data = fetchDirectInitializationData(target.format) ?: return - writeInitializationSegment(target, output, data) + ): Boolean { + val data = fetchDirectInitializationData(target.format) ?: return false + return writeInitializationSegment(target, output, data) } @Throws(IOException::class) From ab87b7bf64efd8f51f663debddb6752755bf49be Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 2 Jul 2026 09:10:09 +0200 Subject: [PATCH 08/12] fix(download): keep SABR resume progress visible --- .../us/shandian/giga/get/SabrDownloader.kt | 42 ++++++++++++++++--- .../us/shandian/giga/get/SabrSegmentWriter.kt | 6 +-- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index 8e1e5c53c..2583cb54d 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -17,6 +17,9 @@ import java.net.UnknownHostException internal class SabrDownloader( private val mission: DownloadMission, ) : Runnable { + private var progressFloor = 0L + private var attemptBytesWritten = 0L + override fun run() { try { ensureRunning() @@ -85,7 +88,11 @@ internal class SabrDownloader( val outputs = targets.associate { target -> target.resourceIndex to target.file.outputStream() } try { - downloadSegments(session, targets, SabrSegmentWriter(mission, session, targets, outputs)) + downloadSegments( + session, + targets, + SabrSegmentWriter(session, targets, outputs, ::reportBytesWritten), + ) } finally { outputs.values.forEach { output -> try { @@ -119,17 +126,40 @@ internal class SabrDownloader( } private fun prepareMission() { - mission.unknownLength = true + // SABR currently restarts the temp transfer on retry/resume. Keep the previous visible + // progress as a floor, then count again once the restarted transfer catches up. + progressFloor = mission.done.coerceAtLeast(0L) + attemptBytesWritten = 0L + mission.unknownLength = mission.nearLength <= 0 mission.sabrStarted = true - if (mission.done > 0) { - mission.notifyProgress(-mission.done) + if (mission.nearLength > 0) { + mission.length = mission.length + .coerceAtLeast(mission.nearLength) + .coerceAtLeast(progressFloor) } - mission.done = 0 mission.current = 0 - mission.length = mission.nearLength mission.writeThisToFile() } + private fun reportBytesWritten(delta: Long) { + if (delta <= 0) { + return + } + attemptBytesWritten += delta + val visibleProgress = attemptBytesWritten.coerceAtLeast(progressFloor) + val visibleDelta = visibleProgress - mission.done + if (visibleDelta <= 0) { + return + } + if (mission.nearLength > 0) { + mission.length = mission.length + .coerceAtLeast(mission.nearLength) + .coerceAtLeast(visibleProgress) + mission.unknownLength = false + } + mission.notifyProgress(visibleDelta) + } + private fun configureRequestMode( session: YoutubeSabrSession, targets: List, diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt index 241c0f622..c2d67642c 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -10,10 +10,10 @@ import java.io.IOException import java.io.OutputStream internal class SabrSegmentWriter( - private val mission: DownloadMission, private val session: YoutubeSabrSession, private val targets: List, private val outputs: Map, + private val onBytesWritten: (Long) -> Unit, ) { @Throws(IOException::class) fun writeDirectInitializations() { @@ -123,7 +123,7 @@ internal class SabrSegmentWriter( output.write(data) target.initializationWritten = true target.initializationData = data - mission.notifyProgress(data.size.toLong()) + onBytesWritten(data.size.toLong()) flushPendingMedia(target, output) return true } @@ -215,7 +215,7 @@ internal class SabrSegmentWriter( private fun writeMediaBytes(target: SabrDownloadTarget, output: OutputStream, data: ByteArray) { output.write(data) target.nextWriteSequence++ - mission.notifyProgress(data.size.toLong()) + onBytesWritten(data.size.toLong()) } private companion object { From 09efd84841a13e7387b142e205a791a584d36811 Mon Sep 17 00:00:00 2001 From: InfinityLoop1308 <96324692+InfinityLoop1308@users.noreply.github.com> Date: Thu, 2 Jul 2026 05:48:19 +0800 Subject: [PATCH 09/12] fix: keep HLS playback available with SABR --- .../resolver/AudioPlaybackResolver.java | 5 ----- .../resolver/VideoPlaybackResolver.java | 19 ++----------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/player/resolver/AudioPlaybackResolver.java b/app/src/main/java/org/schabi/newpipe/player/resolver/AudioPlaybackResolver.java index 438200b56..217f2b417 100644 --- a/app/src/main/java/org/schabi/newpipe/player/resolver/AudioPlaybackResolver.java +++ b/app/src/main/java/org/schabi/newpipe/player/resolver/AudioPlaybackResolver.java @@ -12,7 +12,6 @@ import androidx.media3.exoplayer.source.MediaSource; import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.player.helper.PlayerDataSource; import org.schabi.newpipe.player.helper.PlayerHelper; @@ -52,10 +51,6 @@ public MediaSource resolve(@NonNull final StreamInfo info) { List audioStreams = info.getAudioStreams() .stream().filter(s -> !blacklistUrls.contains(s.getContent())).collect(Collectors.toList()); - if (audioStreams.stream().anyMatch( - stream -> stream.getDeliveryMethod() == DeliveryMethod.SABR)) { - audioStreams.removeIf(stream -> stream.getDeliveryMethod() == DeliveryMethod.HLS); - } removeTorrentStreams(audioStreams); audioStreams = filterUnsupportedFormats(audioStreams, context); diff --git a/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java b/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java index 096ee7634..760da19dd 100644 --- a/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java +++ b/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java @@ -14,7 +14,6 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -80,20 +79,6 @@ public MediaSource resolve(@NonNull final StreamInfo info) { final List mediaSources = new ArrayList<>(); final List videoStreams = new ArrayList<>(info.getVideoStreams()); final List videoOnlyStreams = new ArrayList<>(info.getVideoOnlyStreams()); - final List playbackAudioStreams = new ArrayList<>(info.getAudioStreams()); - - final boolean hasSabr = videoStreams.stream().anyMatch( - stream -> stream.getDeliveryMethod() == DeliveryMethod.SABR) - || videoOnlyStreams.stream().anyMatch( - stream -> stream.getDeliveryMethod() == DeliveryMethod.SABR) - || playbackAudioStreams.stream().anyMatch( - stream -> stream.getDeliveryMethod() == DeliveryMethod.SABR); - if (hasSabr) { - videoStreams.removeIf(stream -> stream.getDeliveryMethod() == DeliveryMethod.HLS); - videoOnlyStreams.removeIf(stream -> stream.getDeliveryMethod() == DeliveryMethod.HLS); - playbackAudioStreams.removeIf( - stream -> stream.getDeliveryMethod() == DeliveryMethod.HLS); - } removeTorrentStreams(videoStreams); removeTorrentStreams(videoOnlyStreams); @@ -115,7 +100,7 @@ public MediaSource resolve(@NonNull final StreamInfo info) { .anyMatch(s -> s.getAudioTrackId() != null); if (hasVideoAudioTracks) { final List allAudioStreams = ListHelper.getFilteredAudioStreams( - context, playbackAudioStreams); + context, info.getAudioStreams()); final int defaultIdx = ListHelper.getDefaultAudioFormat(context, allAudioStreams); if (defaultIdx >= 0 && defaultIdx < allAudioStreams.size()) { final String defaultTrackId = allAudioStreams.get(defaultIdx).getAudioTrackId(); @@ -164,7 +149,7 @@ public MediaSource resolve(@NonNull final StreamInfo info) { // Create optional audio stream source final List audioStreams = ListHelper.getFilteredAudioStreams(context, - playbackAudioStreams + info.getAudioStreams() .stream().filter(s -> !blacklistUrls.contains(s.getContent())) .collect(Collectors.toList())); final int audioIndex = ListHelper.getAudioFormatIndex(context, audioStreams, audioTrack); From 661c5a8adce5850b3781f2c5f17553047a1638fa Mon Sep 17 00:00:00 2001 From: InfinityLoop1308 <96324692+InfinityLoop1308@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:49:07 +0800 Subject: [PATCH 10/12] fix: resume SABR downloads from checkpoints --- .../us/shandian/giga/get/DownloadMission.java | 2 + .../giga/get/SabrDownloadCheckpoint.kt | 26 ++++ .../us/shandian/giga/get/SabrDownloader.kt | 123 ++++++++++++++---- .../us/shandian/giga/get/SabrFfmpegMuxer.kt | 7 +- .../us/shandian/giga/get/SabrSegmentWriter.kt | 6 +- 5 files changed, 130 insertions(+), 34 deletions(-) create mode 100644 app/src/main/java/us/shandian/giga/get/SabrDownloadCheckpoint.kt diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 98f8e0f5a..045cae330 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -136,6 +136,7 @@ public class DownloadMission extends Mission { public String[] resourceManifestUrls; public boolean[] resourceIsUrls; public HlsDownloadCheckpoint hlsCheckpoint; + public SabrDownloadCheckpoint sabrCheckpoint; public boolean sabrStarted; private transient int finishCount; @@ -578,6 +579,7 @@ public void resetState(boolean rollback, boolean persistChanges, int errorCode) blockAcquired = null; if (rollback) { hlsCheckpoint = null; + sabrCheckpoint = null; sabrStarted = false; } diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadCheckpoint.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadCheckpoint.kt new file mode 100644 index 000000000..5c2560443 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadCheckpoint.kt @@ -0,0 +1,26 @@ +package us.shandian.giga.get + +import java.io.Serializable + +data class SabrDownloadCheckpoint( + val version: Int = VERSION, + val resources: List = emptyList(), +) : Serializable { + companion object { + private const val serialVersionUID = 1L + const val VERSION = 1 + } +} + +data class SabrResourceCheckpoint( + val resourceIndex: Int, + val itag: Int, + val tempFilePath: String, + val nextWriteSequence: Int, + val bytesWritten: Long, + val initializationBytes: Int, +) : Serializable { + companion object { + private const val serialVersionUID = 1L + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index 2583cb54d..f27286b81 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -9,7 +9,9 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession import org.schabi.newpipe.player.datasource.WebViewPoTokenProvider import java.io.File +import java.io.FileOutputStream import java.io.IOException +import java.io.RandomAccessFile import java.net.ConnectException import java.net.SocketTimeoutException import java.net.UnknownHostException @@ -17,25 +19,30 @@ import java.net.UnknownHostException internal class SabrDownloader( private val mission: DownloadMission, ) : Runnable { - private var progressFloor = 0L - private var attemptBytesWritten = 0L - override fun run() { try { ensureRunning() val recoveries = validateRecoveryInfo() val info = SabrDownloadFormatResolver.resolveInfo(recoveries) + val expectedLength = recoveries.map { recovery -> + when (recovery.kind) { + 'a' -> SabrDownloadFormatResolver.selectedAudioFormat(info, arrayOf(recovery)) + 'v' -> SabrDownloadFormatResolver.selectedVideoFormat(info, arrayOf(recovery)) + else -> null + } + }.takeIf { formats -> formats.all { it != null && it.contentLength > 0 } } + ?.sumOf { it!!.contentLength } + ?: 0L + prepareMission(expectedLength) var coldStartAttempts = 0 var transientAttempts = 0 while (true) { try { - prepareMission() runSessionAttempt(info, recoveries, coldStartAttempts) break } catch (error: RetryColdStartException) { coldStartAttempts++ - cleanup(mission) if (coldStartAttempts > MAX_COLD_START_RETRIES) { throw SabrDownloadException( SabrDownloadException.Reason.INITIALIZATION, @@ -56,7 +63,6 @@ internal class SabrDownloader( ) } transientAttempts++ - cleanup(mission) logDebug("retry transient attempt=$transientAttempts error=${error.javaClass.simpleName}") Thread.sleep(transientRetryDelayMs(transientAttempts)) } @@ -84,8 +90,14 @@ internal class SabrDownloader( ) val workDir = prepareWorkDirectory() val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) + restoreTargets(targets) + targets.forEach { target -> + session.streamState.jumpBufferedTo(target.format, target.nextWriteSequence) + } configureRequestMode(session, targets, coldStartAttempt) - val outputs = targets.associate { target -> target.resourceIndex to target.file.outputStream() } + val outputs = targets.associate { target -> + target.resourceIndex to FileOutputStream(target.file, true) + } try { downloadSegments( @@ -125,39 +137,36 @@ internal class SabrDownloader( return recoveries } - private fun prepareMission() { - // SABR currently restarts the temp transfer on retry/resume. Keep the previous visible - // progress as a floor, then count again once the restarted transfer catches up. - progressFloor = mission.done.coerceAtLeast(0L) - attemptBytesWritten = 0L + private fun prepareMission(expectedLength: Long) { + if (mission.sabrCheckpoint?.version != SabrDownloadCheckpoint.VERSION) { + mission.sabrCheckpoint = null + cleanup(mission) + } + mission.done = restoredProgress() + mission.nearLength = expectedLength.coerceAtLeast(mission.nearLength) mission.unknownLength = mission.nearLength <= 0 mission.sabrStarted = true if (mission.nearLength > 0) { mission.length = mission.length .coerceAtLeast(mission.nearLength) - .coerceAtLeast(progressFloor) + .coerceAtLeast(mission.done) } mission.current = 0 mission.writeThisToFile() } - private fun reportBytesWritten(delta: Long) { + private fun reportBytesWritten(target: SabrDownloadTarget, delta: Long) { if (delta <= 0) { return } - attemptBytesWritten += delta - val visibleProgress = attemptBytesWritten.coerceAtLeast(progressFloor) - val visibleDelta = visibleProgress - mission.done - if (visibleDelta <= 0) { - return - } + updateCheckpoint(target) if (mission.nearLength > 0) { mission.length = mission.length .coerceAtLeast(mission.nearLength) - .coerceAtLeast(visibleProgress) + .coerceAtLeast(mission.done + delta) mission.unknownLength = false } - mission.notifyProgress(visibleDelta) + mission.notifyProgress(delta) } private fun configureRequestMode( @@ -180,13 +189,76 @@ internal class SabrDownloader( @Throws(IOException::class) private fun prepareWorkDirectory(): File { val workDir = workDirectory(mission) - cleanup(mission) - if (!workDir.mkdirs()) { + if (!workDir.exists() && !workDir.mkdirs()) { throw IOException("Cannot create SABR work directory: $workDir") } return workDir } + private fun restoreTargets(targets: List) { + targets.forEach { target -> + val checkpoint = mission.sabrCheckpoint?.resources?.firstOrNull { + it.resourceIndex == target.resourceIndex && + it.itag == target.format.itag && + it.tempFilePath == target.file.absolutePath && + it.nextWriteSequence > 0 && + it.bytesWritten >= it.initializationBytes && + it.initializationBytes >= 0 && + it.initializationBytes <= MAX_INITIALIZATION_BYTES && + target.file.exists() && + target.file.length() >= it.bytesWritten + } + if (checkpoint == null) { + target.file.delete() + return@forEach + } + RandomAccessFile(target.file, "rw").use { file -> + file.setLength(checkpoint.bytesWritten) + if (checkpoint.initializationBytes > 0) { + val initialization = ByteArray(checkpoint.initializationBytes) + file.seek(0) + file.readFully(initialization) + target.initializationData = initialization + target.initializationWritten = true + } + } + target.nextWriteSequence = checkpoint.nextWriteSequence + } + } + + private fun updateCheckpoint(target: SabrDownloadTarget) { + val current = mission.sabrCheckpoint ?: SabrDownloadCheckpoint() + val resources = current.resources + .filterNot { it.resourceIndex == target.resourceIndex } + .toMutableList() + val previousInitializationBytes = current.resources + .firstOrNull { it.resourceIndex == target.resourceIndex } + ?.initializationBytes + ?: 0 + resources += SabrResourceCheckpoint( + resourceIndex = target.resourceIndex, + itag = target.format.itag, + tempFilePath = target.file.absolutePath, + nextWriteSequence = target.nextWriteSequence, + bytesWritten = target.file.length(), + initializationBytes = if (target.initializationWritten && previousInitializationBytes == 0) { + target.initializationData?.size ?: 0 + } else { + previousInitializationBytes + }, + ) + mission.sabrCheckpoint = current.copy(resources = resources.sortedBy { it.resourceIndex }) + } + + private fun restoredProgress(): Long { + return mission.sabrCheckpoint?.resources + ?.filter { checkpoint -> + File(checkpoint.tempFilePath).let { it.exists() && it.length() >= checkpoint.bytesWritten } + } + ?.sumOf { it.bytesWritten } + ?: 0L + } + @Throws(IOException::class, InterruptedException::class) private fun downloadSegments( session: YoutubeSabrSession, @@ -345,6 +417,7 @@ internal class SabrDownloader( } mission.current = mission.urls.size mission.psState = 2 + mission.sabrCheckpoint = null cleanup(mission) mission.unknownLength = false mission.notifyFinished() @@ -358,7 +431,6 @@ internal class SabrDownloader( } private fun notifyErrorAndCleanup(error: Exception) { - cleanup(mission) if (mission.running) { mission.notifyError(error) } @@ -425,6 +497,7 @@ internal class SabrDownloader( private const val MAX_TRANSIENT_RETRIES = 5 private const val MAX_TRANSIENT_RETRY_DELAY_MS = 5_000L private const val MAX_SESSION_CACHE_BYTES = 48L * 1024L * 1024L + private const val MAX_INITIALIZATION_BYTES = 16 * 1024 * 1024 @JvmStatic fun cleanup(mission: DownloadMission) { diff --git a/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt index 7c7198e1b..e813b9c1c 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt @@ -14,15 +14,10 @@ internal class SabrFfmpegMuxer( fun remuxAndCopy(inputs: List, targets: List, workDir: File): Long { if (canCopySingleInputDirectly(inputs, targets)) { logDebug("copySingleInputDirectly input=${inputs.first().length()}") - return try { - copyOutputToStorage(inputs.first()) - } finally { - deleteQuietly(inputs.first()) - } + return copyOutputToStorage(inputs.first()) } val output = File(workDir, "output.${outputExtension()}") remuxWithFfmpeg(inputs, output) - inputs.forEach(::deleteQuietly) return try { copyOutputToStorage(output) } finally { diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt index c2d67642c..6e199224f 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -13,7 +13,7 @@ internal class SabrSegmentWriter( private val session: YoutubeSabrSession, private val targets: List, private val outputs: Map, - private val onBytesWritten: (Long) -> Unit, + private val onBytesWritten: (SabrDownloadTarget, Long) -> Unit, ) { @Throws(IOException::class) fun writeDirectInitializations() { @@ -123,7 +123,7 @@ internal class SabrSegmentWriter( output.write(data) target.initializationWritten = true target.initializationData = data - onBytesWritten(data.size.toLong()) + onBytesWritten(target, data.size.toLong()) flushPendingMedia(target, output) return true } @@ -215,7 +215,7 @@ internal class SabrSegmentWriter( private fun writeMediaBytes(target: SabrDownloadTarget, output: OutputStream, data: ByteArray) { output.write(data) target.nextWriteSequence++ - onBytesWritten(data.size.toLong()) + onBytesWritten(target, data.size.toLong()) } private companion object { From a342c934a11a21022b7e8067fe2ba57b608f50bd Mon Sep 17 00:00:00 2001 From: InfinityLoop1308 <96324692+InfinityLoop1308@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:11:35 +0800 Subject: [PATCH 11/12] fix: harden SABR initialization and storage errors --- .../us/shandian/giga/get/SabrDownloader.kt | 81 ++++++++++++------- .../us/shandian/giga/get/SabrSegmentWriter.kt | 20 ++++- 2 files changed, 69 insertions(+), 32 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index f27286b81..37abab01d 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -95,8 +95,19 @@ internal class SabrDownloader( session.streamState.jumpBufferedTo(target.format, target.nextWriteSequence) } configureRequestMode(session, targets, coldStartAttempt) - val outputs = targets.associate { target -> - target.resourceIndex to FileOutputStream(target.file, true) + val outputs = mutableMapOf() + try { + targets.forEach { target -> + outputs[target.resourceIndex] = FileOutputStream(target.file, true) + } + } catch (error: IOException) { + outputs.values.forEach { output -> + try { + output.close() + } catch (ignored: IOException) { + } + } + throw storageException("could not open temporary media", error) } try { @@ -190,42 +201,56 @@ internal class SabrDownloader( private fun prepareWorkDirectory(): File { val workDir = workDirectory(mission) if (!workDir.exists() && !workDir.mkdirs()) { - throw IOException("Cannot create SABR work directory: $workDir") + throw storageException("could not create temporary directory", null) } return workDir } private fun restoreTargets(targets: List) { - targets.forEach { target -> - val checkpoint = mission.sabrCheckpoint?.resources?.firstOrNull { - it.resourceIndex == target.resourceIndex && - it.itag == target.format.itag && - it.tempFilePath == target.file.absolutePath && - it.nextWriteSequence > 0 && - it.bytesWritten >= it.initializationBytes && - it.initializationBytes >= 0 && - it.initializationBytes <= MAX_INITIALIZATION_BYTES && - target.file.exists() && - target.file.length() >= it.bytesWritten - } - if (checkpoint == null) { - target.file.delete() - return@forEach - } - RandomAccessFile(target.file, "rw").use { file -> - file.setLength(checkpoint.bytesWritten) - if (checkpoint.initializationBytes > 0) { - val initialization = ByteArray(checkpoint.initializationBytes) - file.seek(0) - file.readFully(initialization) - target.initializationData = initialization - target.initializationWritten = true + try { + targets.forEach { target -> + val checkpoint = mission.sabrCheckpoint?.resources?.firstOrNull { + it.resourceIndex == target.resourceIndex && + it.itag == target.format.itag && + it.tempFilePath == target.file.absolutePath && + it.nextWriteSequence > 0 && + it.bytesWritten >= it.initializationBytes && + it.initializationBytes >= 0 && + it.initializationBytes <= MAX_INITIALIZATION_BYTES && + target.file.exists() && + target.file.length() >= it.bytesWritten + } + if (checkpoint == null) { + if (target.file.exists() && !target.file.delete()) { + throw IOException("Could not reset ${target.file}") + } + return@forEach + } + RandomAccessFile(target.file, "rw").use { file -> + file.setLength(checkpoint.bytesWritten) + if (checkpoint.initializationBytes > 0) { + val initialization = ByteArray(checkpoint.initializationBytes) + file.seek(0) + file.readFully(initialization) + target.initializationData = initialization + target.initializationWritten = true + } } + target.nextWriteSequence = checkpoint.nextWriteSequence } - target.nextWriteSequence = checkpoint.nextWriteSequence + } catch (error: IOException) { + throw storageException("could not restore temporary media", error) } } + private fun storageException(message: String, cause: IOException?): SabrDownloadException { + return SabrDownloadException( + SabrDownloadException.Reason.STORAGE, + "SABR download failed: $message", + cause, + ) + } + private fun updateCheckpoint(target: SabrDownloadTarget) { val current = mission.sabrCheckpoint ?: SabrDownloadCheckpoint() val resources = current.resources diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt index 6e199224f..c2b33c141 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -120,7 +120,7 @@ internal class SabrSegmentWriter( if (target.initializationWritten) { return false } - output.write(data) + writeToStorage(output, data) target.initializationWritten = true target.initializationData = data onBytesWritten(target, data.size.toLong()) @@ -138,8 +138,7 @@ internal class SabrSegmentWriter( } val range = "bytes=$start-$end" val response = NewPipe.getDownloader().get(url, mapOf("Range" to listOf(range))) - val data = response.rawResponseBody() - if (response.responseCode() != 206 && response.responseCode() != 200) { + if (response.responseCode() != 206) { if (response.responseCode() >= 500) { throw IOException( "SABR initialization request failed: HTTP ${response.responseCode()}", @@ -151,6 +150,7 @@ internal class SabrSegmentWriter( + " (HTTP ${response.responseCode()})", ) } + val data = response.rawResponseBody() if (data == null || data.isEmpty()) { throw SabrDownloadException( SabrDownloadException.Reason.INITIALIZATION, @@ -213,11 +213,23 @@ internal class SabrSegmentWriter( } private fun writeMediaBytes(target: SabrDownloadTarget, output: OutputStream, data: ByteArray) { - output.write(data) + writeToStorage(output, data) target.nextWriteSequence++ onBytesWritten(target, data.size.toLong()) } + private fun writeToStorage(output: OutputStream, data: ByteArray) { + try { + output.write(data) + } catch (error: IOException) { + throw SabrDownloadException( + SabrDownloadException.Reason.STORAGE, + "SABR download failed: could not write temporary media", + error, + ) + } + } + private companion object { private const val MAX_PENDING_SEGMENTS = 64 private const val MAX_PENDING_BYTES = 24L * 1024L * 1024L From a1c6a4068467af272a03663fbfbdab4bd8810ee1 Mon Sep 17 00:00:00 2001 From: InfinityLoop1308 <96324692+InfinityLoop1308@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:16:44 +0800 Subject: [PATCH 12/12] UI: show SABR stream sizes --- .../newpipe/util/StreamItemAdapter.java | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java b/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java index ff2ccbf26..0c94fb888 100644 --- a/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java +++ b/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java @@ -13,6 +13,8 @@ import org.schabi.newpipe.DownloaderImpl; import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat; +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Stream; @@ -259,12 +261,16 @@ public static Single fetchSizeForWrapper( if (streamsWrapper.getSizeInBytes(stream) > -2) { continue; } - if (stream.getDeliveryMethod() == DeliveryMethod.HLS - || stream.getDeliveryMethod() == DeliveryMethod.SABR) { + if (stream.getDeliveryMethod() == DeliveryMethod.HLS) { streamsWrapper.setSize(stream, -1); hasChanged = true; continue; } + if (stream.getDeliveryMethod() == DeliveryMethod.SABR) { + streamsWrapper.setSize(stream, getSabrContentLength(stream)); + hasChanged = true; + continue; + } final long contentLength = DownloaderImpl.getInstance().getContentLength( stream.getUrl()); @@ -280,6 +286,24 @@ public static Single fetchSizeForWrapper( .onErrorReturnItem(true); } + private static long getSabrContentLength(final Stream stream) { + if (!(stream.getDeliveryMethodInfo() instanceof YoutubeSabrInfo)) { + return -1; + } + final int itag; + if (stream instanceof AudioStream) { + itag = ((AudioStream) stream).getItag(); + } else if (stream instanceof VideoStream) { + itag = ((VideoStream) stream).getItag(); + } else { + return -1; + } + final YoutubeSabrFormat format = ((YoutubeSabrInfo) stream.getDeliveryMethodInfo()) + .findFormatByItag(itag); + return format != null && format.getContentLength() > 0 + ? format.getContentLength() : -1; + } + public static StreamSizeWrapper empty() { //noinspection unchecked return (StreamSizeWrapper) EMPTY;