diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 8f1e98524..3fe6779f3 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -52,7 +52,6 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.SubtitlesStream; @@ -88,6 +87,7 @@ import io.reactivex.rxjava3.disposables.CompositeDisposable; import us.shandian.giga.get.HlsDownloadStreamHelper; import us.shandian.giga.get.MissionRecoveryInfo; +import us.shandian.giga.get.SabrDownloadStreamHelper; import us.shandian.giga.postprocessing.Postprocessing; import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; @@ -157,29 +157,20 @@ public static DownloadDialog newInstance(final StreamInfo info) { } public static DownloadDialog newInstance(final Context context, final StreamInfo info) { - final List videoStreams = info.getVideoStreams().stream() - .filter(stream -> stream.getDeliveryMethod() != DeliveryMethod.SABR) - .collect(Collectors.toList()); - final List videoOnlyStreams = info.getVideoOnlyStreams().stream() - .filter(stream -> stream.getDeliveryMethod() != DeliveryMethod.SABR) - .collect(Collectors.toList()); - final List audioStreams = info.getAudioStreams().stream() - .filter(stream -> stream.getDeliveryMethod() != DeliveryMethod.SABR) - .collect(Collectors.toList()); final ArrayList streamsList = new ArrayList<>(ListHelper - .getSortedStreamVideosList(context, videoStreams, - videoOnlyStreams, false, false)); + .getSortedStreamVideosList(context, info.getVideoStreams(), + info.getVideoOnlyStreams(), false, false)); final List filteredVideoStreams = ListHelper .filterVideoStreamsByPreferredLanguage(context, streamsList, - audioStreams); + info.getAudioStreams()); final int selectedStreamIndex = ListHelper.getDefaultResolutionIndex( context, filteredVideoStreams); HlsDownloadStreamHelper.addManifestFallbackIfNeeded(filteredVideoStreams, info); final List downloadableAudio = ListHelper - .filterDownloadableAudioStreams(audioStreams); + .filterDownloadableAudioStreams(info.getAudioStreams()); HlsDownloadStreamHelper.addAudioFallbackIfNeeded(downloadableAudio, info); final DownloadDialog instance = newInstance(info); @@ -278,9 +269,11 @@ public void onCreate(@Nullable final Bundle savedInstanceState) { continue; } final AudioStream audioStream = SecondaryStreamHelper - .getAudioStreamFor(getContext(), wrappedAudioStreams.getStreamsList(), videoStreams.get(i)); + .getAudioStreamFor(getContext(), SabrDownloadStreamHelper.audioStreamsForVideo( + wrappedAudioStreams.getStreamsList(), videoStreams.get(i)), videoStreams.get(i)); - if (audioStream != null) { + if (audioStream != null && SabrDownloadStreamHelper + .isCompatibleSecondaryStream(videoStreams.get(i), audioStream)) { secondaryStreams .append(i, new SecondaryStreamHelper<>(wrappedAudioStreams, audioStream)); } else if (DEBUG) { @@ -1154,7 +1147,8 @@ private void continueSelectedDownload(@NonNull final StoredFileHelper storage) { resourceIsUrls = HlsDownloadStreamHelper .buildResourceIsUrls(selectedStream, secondaryStream); if (HlsDownloadStreamHelper.containsHlsResource(resourceDeliveryMethods, - resourceManifestUrls, urls)) { + resourceManifestUrls, urls) + || SabrDownloadStreamHelper.containsSabrStream(selectedStream, secondaryStream)) { psName = null; psArgs = null; } diff --git a/app/src/main/java/org/schabi/newpipe/player/resolver/AudioPlaybackResolver.java b/app/src/main/java/org/schabi/newpipe/player/resolver/AudioPlaybackResolver.java index 438200b56..217f2b417 100644 --- a/app/src/main/java/org/schabi/newpipe/player/resolver/AudioPlaybackResolver.java +++ b/app/src/main/java/org/schabi/newpipe/player/resolver/AudioPlaybackResolver.java @@ -12,7 +12,6 @@ import androidx.media3.exoplayer.source.MediaSource; import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.player.helper.PlayerDataSource; import org.schabi.newpipe.player.helper.PlayerHelper; @@ -52,10 +51,6 @@ public MediaSource resolve(@NonNull final StreamInfo info) { List audioStreams = info.getAudioStreams() .stream().filter(s -> !blacklistUrls.contains(s.getContent())).collect(Collectors.toList()); - if (audioStreams.stream().anyMatch( - stream -> stream.getDeliveryMethod() == DeliveryMethod.SABR)) { - audioStreams.removeIf(stream -> stream.getDeliveryMethod() == DeliveryMethod.HLS); - } removeTorrentStreams(audioStreams); audioStreams = filterUnsupportedFormats(audioStreams, context); diff --git a/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java b/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java index 096ee7634..760da19dd 100644 --- a/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java +++ b/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java @@ -14,7 +14,6 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -80,20 +79,6 @@ public MediaSource resolve(@NonNull final StreamInfo info) { final List mediaSources = new ArrayList<>(); final List videoStreams = new ArrayList<>(info.getVideoStreams()); final List videoOnlyStreams = new ArrayList<>(info.getVideoOnlyStreams()); - final List playbackAudioStreams = new ArrayList<>(info.getAudioStreams()); - - final boolean hasSabr = videoStreams.stream().anyMatch( - stream -> stream.getDeliveryMethod() == DeliveryMethod.SABR) - || videoOnlyStreams.stream().anyMatch( - stream -> stream.getDeliveryMethod() == DeliveryMethod.SABR) - || playbackAudioStreams.stream().anyMatch( - stream -> stream.getDeliveryMethod() == DeliveryMethod.SABR); - if (hasSabr) { - videoStreams.removeIf(stream -> stream.getDeliveryMethod() == DeliveryMethod.HLS); - videoOnlyStreams.removeIf(stream -> stream.getDeliveryMethod() == DeliveryMethod.HLS); - playbackAudioStreams.removeIf( - stream -> stream.getDeliveryMethod() == DeliveryMethod.HLS); - } removeTorrentStreams(videoStreams); removeTorrentStreams(videoOnlyStreams); @@ -115,7 +100,7 @@ public MediaSource resolve(@NonNull final StreamInfo info) { .anyMatch(s -> s.getAudioTrackId() != null); if (hasVideoAudioTracks) { final List allAudioStreams = ListHelper.getFilteredAudioStreams( - context, playbackAudioStreams); + context, info.getAudioStreams()); final int defaultIdx = ListHelper.getDefaultAudioFormat(context, allAudioStreams); if (defaultIdx >= 0 && defaultIdx < allAudioStreams.size()) { final String defaultTrackId = allAudioStreams.get(defaultIdx).getAudioTrackId(); @@ -164,7 +149,7 @@ public MediaSource resolve(@NonNull final StreamInfo info) { // Create optional audio stream source final List audioStreams = ListHelper.getFilteredAudioStreams(context, - playbackAudioStreams + info.getAudioStreams() .stream().filter(s -> !blacklistUrls.contains(s.getContent())) .collect(Collectors.toList())); final int audioIndex = ListHelper.getAudioFormatIndex(context, audioStreams, audioTrack); diff --git a/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java b/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java index 04cca6bf6..0c94fb888 100644 --- a/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java +++ b/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java @@ -13,6 +13,8 @@ import org.schabi.newpipe.DownloaderImpl; import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat; +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Stream; @@ -264,6 +266,11 @@ public static Single fetchSizeForWrapper( hasChanged = true; continue; } + if (stream.getDeliveryMethod() == DeliveryMethod.SABR) { + streamsWrapper.setSize(stream, getSabrContentLength(stream)); + hasChanged = true; + continue; + } final long contentLength = DownloaderImpl.getInstance().getContentLength( stream.getUrl()); @@ -279,6 +286,24 @@ public static Single fetchSizeForWrapper( .onErrorReturnItem(true); } + private static long getSabrContentLength(final Stream stream) { + if (!(stream.getDeliveryMethodInfo() instanceof YoutubeSabrInfo)) { + return -1; + } + final int itag; + if (stream instanceof AudioStream) { + itag = ((AudioStream) stream).getItag(); + } else if (stream instanceof VideoStream) { + itag = ((VideoStream) stream).getItag(); + } else { + return -1; + } + final YoutubeSabrFormat format = ((YoutubeSabrInfo) stream.getDeliveryMethodInfo()) + .findFormatByItag(itag); + return format != null && format.getContentLength() > 0 + ? format.getContentLength() : -1; + } + public static StreamSizeWrapper empty() { //noinspection unchecked return (StreamSizeWrapper) EMPTY; diff --git a/app/src/main/java/us/shandian/giga/get/DirectDownloader.java b/app/src/main/java/us/shandian/giga/get/DirectDownloader.java index f8631b2f2..637273bad 100644 --- a/app/src/main/java/us/shandian/giga/get/DirectDownloader.java +++ b/app/src/main/java/us/shandian/giga/get/DirectDownloader.java @@ -105,9 +105,11 @@ public void init(){ continue; } final AudioStream audioStream = SecondaryStreamHelper - .getAudioStreamFor(context, wrappedAudioStreams.getStreamsList(), videoStreams.get(i)); + .getAudioStreamFor(context, SabrDownloadStreamHelper.audioStreamsForVideo( + wrappedAudioStreams.getStreamsList(), videoStreams.get(i)), videoStreams.get(i)); - if (audioStream != null) { + if (audioStream != null && SabrDownloadStreamHelper + .isCompatibleSecondaryStream(videoStreams.get(i), audioStream)) { secondaryStreams .append(i, new SecondaryStreamHelper<>(wrappedAudioStreams, audioStream)); } @@ -270,7 +272,8 @@ public void startDownload(StoredFileHelper storage) { resourceIsUrls = HlsDownloadStreamHelper .buildResourceIsUrls(selectedStream, secondaryStream); if (HlsDownloadStreamHelper.containsHlsResource(resourceDeliveryMethods, - resourceManifestUrls, urls)) { + resourceManifestUrls, urls) + || SabrDownloadStreamHelper.containsSabrStream(selectedStream, secondaryStream)) { psName = null; psArgs = null; } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 359c17b35..045cae330 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -47,6 +47,7 @@ public class DownloadMission extends Mission { public static final int ERROR_PROGRESS_LOST = 1011; public static final int ERROR_TIMEOUT = 1012; public static final int ERROR_RESOURCE_GONE = 1013; + public static final int ERROR_SABR_DOWNLOAD = 1014; public static final int ERROR_HTTP_NO_CONTENT = 204; static final int ERROR_HTTP_FORBIDDEN = 403; static final int ERROR_HTTP_AUTH = 401; @@ -128,12 +129,15 @@ public class DownloadMission extends Mission { public MissionRecoveryInfo[] recoveryInfo; /** - * Optional typed metadata for resources. Used only to route HLS resources to the HLS downloader. + * Optional typed metadata for resources. Used to route session/manifest resources to their + * dedicated downloaders instead of the direct HTTP range downloader. */ public String[] resourceDeliveryMethods; public String[] resourceManifestUrls; public boolean[] resourceIsUrls; public HlsDownloadCheckpoint hlsCheckpoint; + public SabrDownloadCheckpoint sabrCheckpoint; + public boolean sabrStarted; private transient int finishCount; public transient volatile boolean running; @@ -317,6 +321,8 @@ synchronized void notifyError(Exception err) { notifyError(ERROR_UNKNOWN_HOST, null); } else if (err instanceof SocketTimeoutException) { notifyError(ERROR_TIMEOUT, null); + } else if (err instanceof SabrDownloadException) { + notifyError(ERROR_SABR_DOWNLOAD, err); } else { notifyError(ERROR_UNKNOWN_EXCEPTION, err); } @@ -461,6 +467,11 @@ public void start() { return; } + if (hasSabrResource()) { + init = runAsync(DownloadInitializer.mId, new SabrDownloader(this)); + return; + } + if (hasHlsResource()) { init = runAsync(DownloadInitializer.mId, new HlsDownloader(this)); return; @@ -540,6 +551,7 @@ private void pauseThreads() { public boolean delete() { if (psAlgorithm != null) psAlgorithm.cleanupTemporalDir(); HlsDownloader.cleanup(this); + SabrDownloader.cleanup(this); notify(DownloadManagerService.MESSAGE_DELETED); @@ -565,7 +577,11 @@ public void resetState(boolean rollback, boolean persistChanges, int errorCode) fallbackResumeOffset = 0; blocks = null; blockAcquired = null; - if (rollback) hlsCheckpoint = null; + if (rollback) { + hlsCheckpoint = null; + sabrCheckpoint = null; + sabrStarted = false; + } if (rollback) current = 0; if (persistChanges) writeThisToFile(); @@ -630,7 +646,7 @@ public boolean isPsRunning() { * @return true, otherwise, false */ public boolean isInitialized() { - return blocks != null || hlsCheckpoint != null; // DownloadMissionInitializer or HLS downloader was executed + return blocks != null || hlsCheckpoint != null || sabrStarted; } boolean hasHlsResource() { @@ -638,6 +654,10 @@ boolean hasHlsResource() { resourceManifestUrls, urls); } + boolean hasSabrResource() { + return SabrDownloadStreamHelper.containsSabrResource(resourceDeliveryMethods, recoveryInfo); + } + /** * Gets the approximated final length of the file * diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.kt b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.kt index e2f1f6e63..4995e8219 100644 --- a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.kt +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.kt @@ -19,22 +19,29 @@ class MissionRecoveryInfo( var kind: Char = Char.MIN_VALUE, var validateCondition: String? = null, var audioTrackId: String? = null, - var isHls: Boolean = false + var isHls: Boolean = false, + var isSabr: Boolean = false, + var itag: Int = -1, + var deliveryMethodInfo: Serializable? = null ) : Serializable, Parcelable { constructor(stream: Stream) : this(format = stream.getFormat()!!) { isHls = stream.getDeliveryMethod() == DeliveryMethod.HLS + isSabr = stream.getDeliveryMethod() == DeliveryMethod.SABR + deliveryMethodInfo = stream.deliveryMethodInfo when (stream) { is AudioStream -> { desiredBitrate = stream.averageBitrate isDesired2 = false kind = 'a' audioTrackId = stream.audioTrackId + itag = stream.itag } is VideoStream -> { desired = stream.resolution isDesired2 = stream.isVideoOnly kind = 'v' audioTrackId = stream.audioTrackId + itag = stream.itag } is SubtitlesStream -> { desired = stream.languageTag diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadCheckpoint.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadCheckpoint.kt new file mode 100644 index 000000000..5c2560443 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadCheckpoint.kt @@ -0,0 +1,26 @@ +package us.shandian.giga.get + +import java.io.Serializable + +data class SabrDownloadCheckpoint( + val version: Int = VERSION, + val resources: List = emptyList(), +) : Serializable { + companion object { + private const val serialVersionUID = 1L + const val VERSION = 1 + } +} + +data class SabrResourceCheckpoint( + val resourceIndex: Int, + val itag: Int, + val tempFilePath: String, + val nextWriteSequence: Int, + val bytesWritten: Long, + val initializationBytes: Int, +) : Serializable { + companion object { + private const val serialVersionUID = 1L + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadException.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadException.kt new file mode 100644 index 000000000..3bac566ba --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadException.kt @@ -0,0 +1,20 @@ +package us.shandian.giga.get + +import java.io.IOException + +internal class SabrDownloadException( + val reason: Reason, + message: String, + cause: Throwable? = null, +) : IOException(message, cause) { + enum class Reason { + FORMAT, + INITIALIZATION, + PROTECTED, + STALLED, + NETWORK, + MUXING, + STORAGE, + PROTOCOL, + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt new file mode 100644 index 000000000..4336a5602 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadFormatResolver.kt @@ -0,0 +1,129 @@ +package us.shandian.giga.get + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import java.io.File +import java.io.IOException + +internal object SabrDownloadFormatResolver { + @Throws(IOException::class) + fun resolveInfo(recoveries: Array): YoutubeSabrInfo { + return recoveries.firstNotNullOfOrNull { + it.deliveryMethodInfo as? YoutubeSabrInfo + } ?: throw IOException("Missing SABR info") + } + + @Throws(IOException::class) + fun selectedAudioFormat( + info: YoutubeSabrInfo, + recoveries: Array, + ): YoutubeSabrFormat { + val audioRecovery = recoveries.firstOrNull { it.kind == 'a' } + return audioRecovery?.let { findAudioFormat(info, it) } + ?: if (recoveries.any { it.kind == 'v' }) { + findLightweightAudioFormat(info) + } else { + null + } + ?: info.findBestAudioFormat() + ?: throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: missing audio format", + ) + } + + @Throws(IOException::class) + fun selectedVideoFormat( + info: YoutubeSabrInfo, + recoveries: Array, + ): YoutubeSabrFormat { + val videoRecovery = recoveries.firstOrNull { it.kind == 'v' } + return videoRecovery?.let { findVideoFormat(info, it) } + ?: if (recoveries.any { it.kind == 'a' }) { + findLightweightVideoFormat(info) + } else { + null + } + ?: info.findBestVideoFormat() + ?: throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: missing video format", + ) + } + + @Throws(IOException::class) + fun buildTargets( + info: YoutubeSabrInfo, + recoveries: Array, + workDir: File, + ): List { + return recoveries.mapIndexed { index, recovery -> + val format = when (recovery.kind) { + 'a' -> findAudioFormat(info, recovery) + 'v' -> findVideoFormat(info, recovery) + else -> throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: unsupported resource kind ${recovery.kind}", + ) + } + SabrDownloadTarget(index, recovery, format, File(workDir, "input-$index.media")) + } + } + + @Throws(IOException::class) + private fun findAudioFormat( + info: YoutubeSabrInfo, + recovery: MissionRecoveryInfo, + ): YoutubeSabrFormat { + return info.formats.firstOrNull { format -> + format.isAudio && + (recovery.itag <= 0 || format.itag == recovery.itag) && + (recovery.audioTrackId == null || recovery.audioTrackId == format.audioTrackId) + } ?: throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: could not resolve audio itag ${recovery.itag}", + ) + } + + @Throws(IOException::class) + private fun findVideoFormat( + info: YoutubeSabrInfo, + recovery: MissionRecoveryInfo, + ): YoutubeSabrFormat { + return info.formats.firstOrNull { format -> + format.isVideo && (recovery.itag <= 0 || format.itag == recovery.itag) + } ?: throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: could not resolve video itag ${recovery.itag}", + ) + } + + private fun findLightweightAudioFormat(info: YoutubeSabrInfo): YoutubeSabrFormat? { + return info.formats + .filter { it.isAudio } + .sortedWith( + compareBy { !it.isOriginalAudio } + .thenBy { it.isDrc } + .thenBy { normalizedBitrate(it) }, + ) + .firstOrNull() + } + + private fun findLightweightVideoFormat(info: YoutubeSabrInfo): YoutubeSabrFormat? { + return info.formats + .filter { it.isVideo } + .sortedWith( + compareBy { normalizedHeight(it) } + .thenBy { normalizedBitrate(it) }, + ) + .firstOrNull() + } + + private fun normalizedBitrate(format: YoutubeSabrFormat): Int { + return format.bitrate.takeIf { it > 0 } ?: Int.MAX_VALUE + } + + private fun normalizedHeight(format: YoutubeSabrFormat): Int { + return format.height.takeIf { it > 0 } ?: Int.MAX_VALUE + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadStreamHelper.java b/app/src/main/java/us/shandian/giga/get/SabrDownloadStreamHelper.java new file mode 100644 index 000000000..3dc15d63b --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadStreamHelper.java @@ -0,0 +1,69 @@ +package us.shandian.giga.get; + +import org.schabi.newpipe.extractor.stream.DeliveryMethod; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.util.ArrayList; +import java.util.List; + +public final class SabrDownloadStreamHelper { + private SabrDownloadStreamHelper() { + } + + public static boolean containsSabrResource(final String[] deliveryMethods, + final MissionRecoveryInfo[] recoveryInfo) { + if (deliveryMethods != null) { + for (final String deliveryMethod : deliveryMethods) { + if (DeliveryMethod.SABR.name().equals(deliveryMethod)) { + return true; + } + } + } + if (recoveryInfo != null) { + for (final MissionRecoveryInfo recovery : recoveryInfo) { + if (recovery != null && recovery.isSabr()) { + return true; + } + } + } + return false; + } + + public static boolean containsSabrStream(final Stream selectedStream, + final Stream secondaryStream) { + return isSabr(selectedStream) || isSabr(secondaryStream); + } + + public static List audioStreamsForVideo(final List audioStreams, + final VideoStream videoStream) { + if (audioStreams == null || audioStreams.isEmpty()) { + return audioStreams; + } + + final List result = new ArrayList<>(); + for (final AudioStream audioStream : audioStreams) { + if (isSabr(audioStream) == isSabr(videoStream)) { + result.add(audioStream); + } + } + return result; + } + + public static boolean isCompatibleSecondaryStream(final Stream selectedStream, + final Stream secondaryStream) { + if (secondaryStream == null) { + return true; + } + if (!isSabr(selectedStream) && !isSabr(secondaryStream)) { + return true; + } + return isSabr(selectedStream) && isSabr(secondaryStream); + } + + private static boolean isSabr(final Stream stream) { + return stream != null && stream.getDeliveryMethod() == DeliveryMethod.SABR; + } + +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt new file mode 100644 index 000000000..fd0efc394 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloadTarget.kt @@ -0,0 +1,18 @@ +package us.shandian.giga.get + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import java.io.File +import java.util.TreeMap + +internal data class SabrDownloadTarget( + val resourceIndex: Int, + val recovery: MissionRecoveryInfo, + val format: YoutubeSabrFormat, + val file: File, + var nextWriteSequence: Int = 1, + var initializationWritten: Boolean = false, + var initializationObserved: Boolean = false, + var initializationData: ByteArray? = null, + val pending: TreeMap = TreeMap(), + var pendingBytes: Long = 0, +) diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt new file mode 100644 index 000000000..37abab01d --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -0,0 +1,548 @@ +package us.shandian.giga.get + +import android.util.Log +import org.schabi.newpipe.BuildConfig +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.player.datasource.WebViewPoTokenProvider +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.io.RandomAccessFile +import java.net.ConnectException +import java.net.SocketTimeoutException +import java.net.UnknownHostException + +internal class SabrDownloader( + private val mission: DownloadMission, +) : Runnable { + override fun run() { + try { + ensureRunning() + val recoveries = validateRecoveryInfo() + val info = SabrDownloadFormatResolver.resolveInfo(recoveries) + + val expectedLength = recoveries.map { recovery -> + when (recovery.kind) { + 'a' -> SabrDownloadFormatResolver.selectedAudioFormat(info, arrayOf(recovery)) + 'v' -> SabrDownloadFormatResolver.selectedVideoFormat(info, arrayOf(recovery)) + else -> null + } + }.takeIf { formats -> formats.all { it != null && it.contentLength > 0 } } + ?.sumOf { it!!.contentLength } + ?: 0L + prepareMission(expectedLength) + var coldStartAttempts = 0 + var transientAttempts = 0 + while (true) { + try { + runSessionAttempt(info, recoveries, coldStartAttempts) + break + } catch (error: RetryColdStartException) { + coldStartAttempts++ + if (coldStartAttempts > MAX_COLD_START_RETRIES) { + throw SabrDownloadException( + SabrDownloadException.Reason.INITIALIZATION, + "SABR download failed: cold start did not provide initialization", + error, + ) + } + logDebug("retry cold start attempt=$coldStartAttempts") + } catch (error: Exception) { + if (!isRetryableAttemptFailure(error)) { + throw error + } + if (transientAttempts >= MAX_TRANSIENT_RETRIES) { + throw SabrDownloadException( + SabrDownloadException.Reason.NETWORK, + "SABR download failed: network error after retries", + error, + ) + } + transientAttempts++ + logDebug("retry transient attempt=$transientAttempts error=${error.javaClass.simpleName}") + Thread.sleep(transientRetryDelayMs(transientAttempts)) + } + } + } catch (error: InterruptedException) { + Thread.currentThread().interrupt() + } catch (error: SabrProtocolException) { + notifyErrorAndCleanup(classifyProtocolException(error)) + } catch (error: Exception) { + notifyErrorAndCleanup(error) + } + } + + @Throws(IOException::class, InterruptedException::class, SabrProtocolException::class) + private fun runSessionAttempt( + info: YoutubeSabrInfo, + recoveries: Array, + coldStartAttempt: Int, + ) { + val session = YoutubeSabrSession( + info, + SabrDownloadFormatResolver.selectedAudioFormat(info, recoveries), + SabrDownloadFormatResolver.selectedVideoFormat(info, recoveries), + WebViewPoTokenProvider(mission.context), + ) + val workDir = prepareWorkDirectory() + val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) + restoreTargets(targets) + targets.forEach { target -> + session.streamState.jumpBufferedTo(target.format, target.nextWriteSequence) + } + configureRequestMode(session, targets, coldStartAttempt) + val outputs = mutableMapOf() + try { + targets.forEach { target -> + outputs[target.resourceIndex] = FileOutputStream(target.file, true) + } + } catch (error: IOException) { + outputs.values.forEach { output -> + try { + output.close() + } catch (ignored: IOException) { + } + } + throw storageException("could not open temporary media", error) + } + + try { + downloadSegments( + session, + targets, + SabrSegmentWriter(session, targets, outputs, ::reportBytesWritten), + ) + } finally { + outputs.values.forEach { output -> + try { + output.close() + } catch (ignored: Exception) { + // Nothing to do. + } + } + session.clearCache() + } + + ensureRunning() + val finalBytes = SabrFfmpegMuxer(mission).remuxAndCopy( + targets.map { it.file }, + targets, + workDir, + ) + completeMission(finalBytes) + } + + @Throws(IOException::class) + private fun validateRecoveryInfo(): Array { + val recoveries = mission.recoveryInfo ?: throw IOException("Missing SABR recovery info") + if (recoveries.size != mission.urls.size || recoveries.any { !it.isSabr }) { + throw SabrDownloadException( + SabrDownloadException.Reason.FORMAT, + "SABR download failed: mixed SABR/non-SABR resources are not supported", + ) + } + return recoveries + } + + private fun prepareMission(expectedLength: Long) { + if (mission.sabrCheckpoint?.version != SabrDownloadCheckpoint.VERSION) { + mission.sabrCheckpoint = null + cleanup(mission) + } + mission.done = restoredProgress() + mission.nearLength = expectedLength.coerceAtLeast(mission.nearLength) + mission.unknownLength = mission.nearLength <= 0 + mission.sabrStarted = true + if (mission.nearLength > 0) { + mission.length = mission.length + .coerceAtLeast(mission.nearLength) + .coerceAtLeast(mission.done) + } + mission.current = 0 + mission.writeThisToFile() + } + + private fun reportBytesWritten(target: SabrDownloadTarget, delta: Long) { + if (delta <= 0) { + return + } + updateCheckpoint(target) + if (mission.nearLength > 0) { + mission.length = mission.length + .coerceAtLeast(mission.nearLength) + .coerceAtLeast(mission.done + delta) + mission.unknownLength = false + } + mission.notifyProgress(delta) + } + + private fun configureRequestMode( + session: YoutubeSabrSession, + targets: List, + coldStartAttempt: Int, + ) { + val useCompanionWarmup = targets.size == 1 && coldStartAttempt % 2 == 1 + if (useCompanionWarmup) { + session.streamState.setVideoAndAudioRequestMode() + } else if (targets.size == 1 && targets.first().format.isAudio) { + session.streamState.setAudioOnlyRequestMode() + } else if (targets.size == 1 && targets.first().format.isVideo) { + session.streamState.setVideoOnlyRequestMode() + } else { + session.streamState.setVideoAndAudioRequestMode() + } + } + + @Throws(IOException::class) + private fun prepareWorkDirectory(): File { + val workDir = workDirectory(mission) + if (!workDir.exists() && !workDir.mkdirs()) { + throw storageException("could not create temporary directory", null) + } + return workDir + } + + private fun restoreTargets(targets: List) { + try { + targets.forEach { target -> + val checkpoint = mission.sabrCheckpoint?.resources?.firstOrNull { + it.resourceIndex == target.resourceIndex && + it.itag == target.format.itag && + it.tempFilePath == target.file.absolutePath && + it.nextWriteSequence > 0 && + it.bytesWritten >= it.initializationBytes && + it.initializationBytes >= 0 && + it.initializationBytes <= MAX_INITIALIZATION_BYTES && + target.file.exists() && + target.file.length() >= it.bytesWritten + } + if (checkpoint == null) { + if (target.file.exists() && !target.file.delete()) { + throw IOException("Could not reset ${target.file}") + } + return@forEach + } + RandomAccessFile(target.file, "rw").use { file -> + file.setLength(checkpoint.bytesWritten) + if (checkpoint.initializationBytes > 0) { + val initialization = ByteArray(checkpoint.initializationBytes) + file.seek(0) + file.readFully(initialization) + target.initializationData = initialization + target.initializationWritten = true + } + } + target.nextWriteSequence = checkpoint.nextWriteSequence + } + } catch (error: IOException) { + throw storageException("could not restore temporary media", error) + } + } + + private fun storageException(message: String, cause: IOException?): SabrDownloadException { + return SabrDownloadException( + SabrDownloadException.Reason.STORAGE, + "SABR download failed: $message", + cause, + ) + } + + private fun updateCheckpoint(target: SabrDownloadTarget) { + val current = mission.sabrCheckpoint ?: SabrDownloadCheckpoint() + val resources = current.resources + .filterNot { it.resourceIndex == target.resourceIndex } + .toMutableList() + val previousInitializationBytes = current.resources + .firstOrNull { it.resourceIndex == target.resourceIndex } + ?.initializationBytes + ?: 0 + resources += SabrResourceCheckpoint( + resourceIndex = target.resourceIndex, + itag = target.format.itag, + tempFilePath = target.file.absolutePath, + nextWriteSequence = target.nextWriteSequence, + bytesWritten = target.file.length(), + initializationBytes = if (target.initializationWritten && previousInitializationBytes == 0) { + target.initializationData?.size ?: 0 + } else { + previousInitializationBytes + }, + ) + mission.sabrCheckpoint = current.copy(resources = resources.sortedBy { it.resourceIndex }) + } + + private fun restoredProgress(): Long { + return mission.sabrCheckpoint?.resources + ?.filter { checkpoint -> + File(checkpoint.tempFilePath).let { it.exists() && it.length() >= checkpoint.bytesWritten } + } + ?.sumOf { it.bytesWritten } + ?: 0L + } + + @Throws(IOException::class, InterruptedException::class) + private fun downloadSegments( + session: YoutubeSabrSession, + targets: List, + writer: SabrSegmentWriter, + ) { + val localization = Localization("en", "US") + writer.writeDirectInitializations() + writer.observeWrittenInitializations() + if (targets.size == 1 && !targets.first().initializationWritten) { + fetchInitializationsOrRetry(writer, localization) + writer.observeWrittenInitializations() + writer.drainCachedInitializations() + } + + var emptyResponses = 0 + while (true) { + ensureRunning() + writer.observeWrittenInitializations() + var wroteSegment = writer.drainCachedInitializations() + wroteSegment = writer.drainCachedSegments() || wroteSegment + configureInitializedSingleTargetMode(session, targets) + + if (isDownloadComplete(session, targets)) { + break + } + + val playerTimeMs = downloadPlayerTimeMs(session, targets) + session.streamState.setPlayerTimeMs(playerTimeMs) + val segments = session.pumpOnce(localization) + writer.observeWrittenInitializations() + wroteSegment = writer.drainCachedInitializations() || wroteSegment + wroteSegment = writer.drainCachedSegments() || wroteSegment + enforceSessionCacheLimit(session, writer) + configureInitializedSingleTargetMode(session, targets) + if (hasMediaWaitingForInitialization(targets)) { + fetchMissingInitializationsOrRetry(writer, localization) + writer.observeWrittenInitializations() + wroteSegment = writer.drainCachedInitializations() || wroteSegment + wroteSegment = writer.drainCachedSegments() || wroteSegment + configureInitializedSingleTargetMode(session, targets) + if (hasMediaWaitingForInitialization(targets)) { + throw RetryColdStartException() + } + } + + if (isDownloadComplete(session, targets)) { + break + } + if (wroteSegment || segments.isNotEmpty()) { + emptyResponses = 0 + } else { + emptyResponses++ + if (emptyResponses > MAX_EMPTY_RESPONSES) { + throw SabrDownloadException( + SabrDownloadException.Reason.STALLED, + "SABR download stalled: no media received after $MAX_EMPTY_RESPONSES rounds", + ) + } + Thread.sleep(IDLE_POLL_MS) + } + } + } + + @Throws(IOException::class) + private fun fetchInitializationsOrRetry( + writer: SabrSegmentWriter, + localization: Localization, + ) { + try { + writer.fetchUnwrittenInitializations(localization) + } catch (error: SabrProtocolException) { + if (isRetryableInitializationProtocolError(error)) { + throw RetryColdStartException(error) + } + throw error + } + } + + @Throws(IOException::class) + private fun fetchMissingInitializationsOrRetry( + writer: SabrSegmentWriter, + localization: Localization, + ) { + try { + writer.fetchMissingInitializations(localization) + } catch (error: SabrProtocolException) { + if (isRetryableInitializationProtocolError(error)) { + throw RetryColdStartException(error) + } + throw error + } + } + + @Throws(IOException::class) + private fun enforceSessionCacheLimit( + session: YoutubeSabrSession, + writer: SabrSegmentWriter, + ) { + if (session.cachedBytes <= MAX_SESSION_CACHE_BYTES) { + return + } + writer.drainCachedSegments() + if (session.cachedBytes <= MAX_SESSION_CACHE_BYTES) { + return + } + throw SabrDownloadException( + SabrDownloadException.Reason.STALLED, + "SABR download stalled: cached media grew to ${session.cachedBytes} bytes", + ) + } + + private fun configureInitializedSingleTargetMode( + session: YoutubeSabrSession, + targets: List, + ) { + if (targets.size != 1 || !targets.first().initializationWritten) { + return + } + if (targets.first().format.isAudio) { + session.streamState.setAudioOnlyRequestMode() + } else { + session.streamState.setVideoOnlyRequestMode() + } + } + + private fun hasMediaWaitingForInitialization(targets: List): Boolean { + return targets.any { target -> !target.initializationWritten && target.pending.isNotEmpty() } + } + + private fun downloadPlayerTimeMs( + session: YoutubeSabrSession, + targets: List, + ): Long { + if (targets.size == 1) { + return session.streamState.getBufferedEndMs(targets.first().format) + } + return session.streamState.minBufferedEndMs + } + + private fun isDownloadComplete( + session: YoutubeSabrSession, + targets: List, + ): Boolean { + return targets.all { target -> + target.pending.isEmpty() && + (session.streamState.isComplete(target.format) || + session.isBeyondEnd(SabrSegmentRequest.media(target.format, target.nextWriteSequence))) + } + } + + private fun completeMission(finalBytes: Long) { + if (finalBytes > 0) { + mission.done = finalBytes + mission.length = finalBytes + } + mission.current = mission.urls.size + mission.psState = 2 + mission.sabrCheckpoint = null + cleanup(mission) + mission.unknownLength = false + mission.notifyFinished() + } + + @Throws(InterruptedException::class) + private fun ensureRunning() { + if (!mission.running || Thread.currentThread().isInterrupted) { + throw InterruptedException() + } + } + + private fun notifyErrorAndCleanup(error: Exception) { + if (mission.running) { + mission.notifyError(error) + } + } + + private fun isRetryableAttemptFailure(error: Exception): Boolean { + if (error is RetryColdStartException || error is SabrDownloadException) { + return false + } + if (error is SabrProtocolException) { + return false + } + return error is SocketTimeoutException || + error is ConnectException || + error is UnknownHostException || + error is IOException + } + + private fun transientRetryDelayMs(attempt: Int): Long { + return (500L shl (attempt - 1)).coerceAtMost(MAX_TRANSIENT_RETRY_DELAY_MS) + } + + private fun classifyProtocolException(error: SabrProtocolException): SabrDownloadException { + val message = error.message.orEmpty() + val reason = when { + message.contains("protected", ignoreCase = true) || + message.contains("PO token", ignoreCase = true) -> { + SabrDownloadException.Reason.PROTECTED + } + message.contains("policy-only", ignoreCase = true) || + message.contains("not returned", ignoreCase = true) || + message.contains("integrity", ignoreCase = true) -> { + SabrDownloadException.Reason.STALLED + } + else -> SabrDownloadException.Reason.PROTOCOL + } + return SabrDownloadException( + reason, + "SABR download failed: ${message.ifBlank { "protocol error" }}", + error, + ) + } + + private fun isRetryableInitializationProtocolError(error: SabrProtocolException): Boolean { + val message = error.message.orEmpty() + if (!message.contains(":init")) { + return false + } + return message.contains("policy-only", ignoreCase = true) || + message.contains("not returned", ignoreCase = true) + } + + private fun logDebug(message: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, message) + } + } + + companion object { + private const val TAG = "SabrDownloader" + private const val IDLE_POLL_MS = 250L + private const val MAX_EMPTY_RESPONSES = 60 + private const val MAX_COLD_START_RETRIES = 3 + private const val MAX_TRANSIENT_RETRIES = 5 + private const val MAX_TRANSIENT_RETRY_DELAY_MS = 5_000L + private const val MAX_SESSION_CACHE_BYTES = 48L * 1024L * 1024L + private const val MAX_INITIALIZATION_BYTES = 16 * 1024 * 1024 + + @JvmStatic + fun cleanup(mission: DownloadMission) { + try { + workDirectory(mission).deleteRecursively() + } catch (ignored: Exception) { + // Nothing to do. + } + } + + private fun workDirectory(mission: DownloadMission): File { + val base = mission.context.getExternalFilesDir(null) ?: mission.context.filesDir + val missionId = if (mission.timestamp > 0) { + mission.timestamp.toString() + } else { + mission.storage.name.hashCode().toString() + } + return File(base, "sabr-downloader/$missionId") + } + } + + private class RetryColdStartException(cause: Throwable? = null) : IOException(cause) +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt new file mode 100644 index 000000000..e813b9c1c --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrFfmpegMuxer.kt @@ -0,0 +1,153 @@ +package us.shandian.giga.get + +import android.util.Log +import com.arthenica.ffmpegkit.FFmpegKit +import com.arthenica.ffmpegkit.ReturnCode +import org.schabi.newpipe.BuildConfig +import java.io.File +import java.io.IOException + +internal class SabrFfmpegMuxer( + private val mission: DownloadMission, +) { + @Throws(IOException::class, InterruptedException::class) + fun remuxAndCopy(inputs: List, targets: List, workDir: File): Long { + if (canCopySingleInputDirectly(inputs, targets)) { + logDebug("copySingleInputDirectly input=${inputs.first().length()}") + return copyOutputToStorage(inputs.first()) + } + val output = File(workDir, "output.${outputExtension()}") + remuxWithFfmpeg(inputs, output) + return try { + copyOutputToStorage(output) + } finally { + deleteQuietly(output) + } + } + + @Throws(IOException::class) + private fun remuxWithFfmpeg(inputs: List, output: File) { + mission.psState = 1 + mission.writeThisToFile() + val command = buildList { + add("-hide_banner") + add("-nostats") + add("-loglevel") + add("fatal") + add("-y") + inputs.forEach { input -> + add("-i") + add(input.absolutePath) + } + if (mission.kind == 'a' && inputs.size == 1) { + add("-map") + add("0:a?") + add("-vn") + } else if (inputs.size > 1) { + inputs.indices.forEach { index -> + add("-map") + add("$index:v?") + add("-map") + add("$index:a?") + } + } + add("-c") + add("copy") + if (supportsFastStart(output)) { + add("-movflags") + add("+faststart") + } + add(output.absolutePath) + } + logDebug("remuxWithFfmpeg inputs=${inputs.size} output=${output.name}") + val session = FFmpegKit.executeWithArguments(command.toTypedArray()) + if (!ReturnCode.isSuccess(session.returnCode)) { + mission.psState = 0 + throw SabrDownloadException( + SabrDownloadException.Reason.MUXING, + "SABR download failed: ffmpeg remux failed (${session.returnCode})" + + session.output.takeIf { it.isNotBlank() }?.let { ": $it" }.orEmpty(), + ) + } + } + + @Throws(IOException::class, InterruptedException::class) + private fun copyOutputToStorage(output: File): Long { + var copied = 0L + try { + output.inputStream().use { input -> + mission.storage.getStream().use { storage -> + storage.setLength(0) + storage.seek(0) + val buffer = ByteArray(DownloadMission.BUFFER_SIZE) + while (true) { + ensureRunning() + val read = input.read(buffer) + if (read == -1) { + break + } + storage.write(buffer, 0, read) + copied += read.toLong() + } + } + } + } catch (error: IOException) { + throw SabrDownloadException( + SabrDownloadException.Reason.STORAGE, + "SABR download failed: could not write final file", + error, + ) + } + return copied + } + + @Throws(InterruptedException::class) + private fun ensureRunning() { + if (!mission.running || Thread.currentThread().isInterrupted) { + throw InterruptedException() + } + } + + private fun outputExtension(): String { + val name = mission.storage.name ?: return "mp4" + val extension = name.substringAfterLast('.', missingDelimiterValue = "") + return extension.takeIf { it.isNotBlank() && it.all { char -> char.isLetterOrDigit() } } ?: "mp4" + } + + private fun supportsFastStart(output: File): Boolean { + return when (output.extension.lowercase()) { + "m4a", "m4v", "mov", "mp4" -> true + else -> false + } + } + + private fun canCopySingleInputDirectly( + inputs: List, + targets: List, + ): Boolean { + if (inputs.size != 1 || targets.size != 1) { + return false + } + val extension = outputExtension().lowercase() + val mimeType = targets.first().format.mimeType ?: return false + return extension == "webm" && mimeType.contains("webm", ignoreCase = true) + } + + private fun logDebug(message: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, message) + } + } + + private fun deleteQuietly(file: File) { + try { + file.delete() + } catch (ignored: Exception) { + // Nothing to do. + } + } + + private companion object { + private const val TAG = "SabrFfmpegMuxer" + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt new file mode 100644 index 000000000..c2b33c141 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/SabrSegmentWriter.kt @@ -0,0 +1,237 @@ +package us.shandian.giga.get + +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import java.io.IOException +import java.io.OutputStream + +internal class SabrSegmentWriter( + private val session: YoutubeSabrSession, + private val targets: List, + private val outputs: Map, + private val onBytesWritten: (SabrDownloadTarget, Long) -> Unit, +) { + @Throws(IOException::class) + fun writeDirectInitializations() { + for (target in targets) { + writeDirectInitializationIfAvailable(target, outputs.getValue(target.resourceIndex)) + } + } + + fun observeWrittenInitializations() { + for (target in targets) { + val data = target.initializationData ?: continue + if (!target.initializationObserved) { + target.initializationObserved = + session.streamState.ingestInitializationData(target.format, data) + } + } + } + + @Throws(IOException::class) + fun drainCachedInitializations(): Boolean { + var wroteInitialization = false + for (target in targets) { + if (target.initializationWritten) { + continue + } + val request = SabrSegmentRequest.initialization(target.format) + val segment = session.getCachedSegment(request) ?: continue + writeInitializationSegment(target, outputs.getValue(target.resourceIndex), segment.data) + session.discardCachedSegment(request) + wroteInitialization = true + } + return wroteInitialization + } + + @Throws(IOException::class) + fun drainCachedSegments(): Boolean { + var wroteSegment = false + for (target in targets) { + while (true) { + val request = SabrSegmentRequest.media(target.format, target.nextWriteSequence) + val segment = session.getCachedSegment(request) ?: break + if (segment.header.isInitSegment) { + session.discardCachedSegment(request) + continue + } + writeMediaSegment(target, outputs.getValue(target.resourceIndex), segment) + session.discardCachedSegment(request) + wroteSegment = true + } + } + return wroteSegment + } + + @Throws(IOException::class) + fun fetchMissingInitializations(localization: Localization): Boolean { + return fetchInitializations(localization, onlyWhenMediaIsPending = true) + } + + @Throws(IOException::class) + fun fetchUnwrittenInitializations(localization: Localization): Boolean { + return fetchInitializations(localization, onlyWhenMediaIsPending = false) + } + + @Throws(IOException::class) + private fun fetchInitializations( + localization: Localization, + onlyWhenMediaIsPending: Boolean, + ): Boolean { + var wroteInitialization = false + for (target in targets) { + if (target.initializationWritten || + (onlyWhenMediaIsPending && target.pending.isEmpty()) + ) { + continue + } + if (writeDirectInitializationIfAvailable(target, outputs.getValue(target.resourceIndex))) { + wroteInitialization = true + continue + } + val request = SabrSegmentRequest.initialization(target.format) + val segment = session.fetchSegment(request, localization) + writeInitializationSegment(target, outputs.getValue(target.resourceIndex), segment.data) + session.discardCachedSegment(request) + wroteInitialization = true + } + return wroteInitialization + } + + @Throws(IOException::class) + private fun writeDirectInitializationIfAvailable( + target: SabrDownloadTarget, + output: OutputStream, + ): Boolean { + val data = fetchDirectInitializationData(target.format) ?: return false + return writeInitializationSegment(target, output, data) + } + + @Throws(IOException::class) + private fun writeInitializationSegment( + target: SabrDownloadTarget, + output: OutputStream, + data: ByteArray, + ): Boolean { + if (target.initializationWritten) { + return false + } + writeToStorage(output, data) + target.initializationWritten = true + target.initializationData = data + onBytesWritten(target, data.size.toLong()) + flushPendingMedia(target, output) + return true + } + + @Throws(IOException::class) + private fun fetchDirectInitializationData(format: YoutubeSabrFormat): ByteArray? { + val url = format.initializationUrl + val start = format.initRangeStart + val end = format.initRangeEnd + if (url.isNullOrBlank() || start < 0 || end < start) { + return null + } + val range = "bytes=$start-$end" + val response = NewPipe.getDownloader().get(url, mapOf("Range" to listOf(range))) + if (response.responseCode() != 206) { + if (response.responseCode() >= 500) { + throw IOException( + "SABR initialization request failed: HTTP ${response.responseCode()}", + ) + } + throw SabrDownloadException( + SabrDownloadException.Reason.INITIALIZATION, + "SABR download failed: could not fetch initialization for itag ${format.itag}" + + " (HTTP ${response.responseCode()})", + ) + } + val data = response.rawResponseBody() + if (data == null || data.isEmpty()) { + throw SabrDownloadException( + SabrDownloadException.Reason.INITIALIZATION, + "SABR download failed: empty initialization for itag ${format.itag}", + ) + } + return data + } + + @Throws(IOException::class) + private fun writeMediaSegment( + target: SabrDownloadTarget, + output: OutputStream, + segment: SabrMediaSegment, + ) { + val sequence = segment.header.sequenceNumber + if (sequence < target.nextWriteSequence) { + return + } + if (!target.initializationWritten) { + cachePendingMedia(target, sequence, segment.data, "waiting for initialization") + return + } + if (sequence > target.nextWriteSequence) { + cachePendingMedia(target, sequence, segment.data, "waiting for sequence ${target.nextWriteSequence}") + return + } + writeMediaBytes(target, output, segment.data) + flushPendingMedia(target, output) + } + + private fun flushPendingMedia(target: SabrDownloadTarget, output: OutputStream) { + while (true) { + val pending = target.pending.remove(target.nextWriteSequence) ?: return + target.pendingBytes = (target.pendingBytes - pending.size).coerceAtLeast(0) + writeMediaBytes(target, output, pending) + } + } + + private fun cachePendingMedia( + target: SabrDownloadTarget, + sequence: Int, + data: ByteArray, + reason: String, + ) { + val previous = target.pending.put(sequence, data) + if (previous != null) { + target.pendingBytes -= previous.size.toLong() + } + target.pendingBytes += data.size.toLong() + if (target.pending.size > MAX_PENDING_SEGMENTS + || target.pendingBytes > MAX_PENDING_BYTES + ) { + throw SabrDownloadException( + SabrDownloadException.Reason.STALLED, + "SABR download stalled while writing itag ${target.format.itag}: $reason" + + " (${target.pending.size} pending segments, ${target.pendingBytes} bytes)", + ) + } + } + + private fun writeMediaBytes(target: SabrDownloadTarget, output: OutputStream, data: ByteArray) { + writeToStorage(output, data) + target.nextWriteSequence++ + onBytesWritten(target, data.size.toLong()) + } + + private fun writeToStorage(output: OutputStream, data: ByteArray) { + try { + output.write(data) + } catch (error: IOException) { + throw SabrDownloadException( + SabrDownloadException.Reason.STORAGE, + "SABR download failed: could not write temporary media", + error, + ) + } + } + + private companion object { + private const val MAX_PENDING_SEGMENTS = 64 + private const val MAX_PENDING_BYTES = 24L * 1024L * 1024L + } +} diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index 0e50b132a..4039476ae 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -49,6 +49,7 @@ import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.HlsDownloadStreamHelper; import us.shandian.giga.get.MissionRecoveryInfo; +import us.shandian.giga.get.SabrDownloadStreamHelper; import org.schabi.newpipe.streams.io.StoredDirectoryHelper; import org.schabi.newpipe.streams.io.StoredFileHelper; import org.schabi.newpipe.util.Localization; @@ -452,7 +453,9 @@ private void startMission(Intent intent) { if (DEBUG && ps == null && urls != null && urls.length > 1 && !HlsDownloadStreamHelper.containsHlsResource(mission.resourceDeliveryMethods, - mission.resourceManifestUrls, urls)) { + mission.resourceManifestUrls, urls) + && !SabrDownloadStreamHelper.containsSabrResource(mission.resourceDeliveryMethods, + mission.recoveryInfo)) { Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?"); } diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 20b8bb7f9..70ec5bfa0 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -65,6 +65,7 @@ import us.shandian.giga.get.FinishedMission; import us.shandian.giga.get.Mission; import us.shandian.giga.get.MissionRecoveryInfo; +import us.shandian.giga.get.SabrDownloadException; import org.schabi.newpipe.streams.io.StoredFileHelper; import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; @@ -88,6 +89,7 @@ import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_STOPPED; import static us.shandian.giga.get.DownloadMission.ERROR_PROGRESS_LOST; import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; +import static us.shandian.giga.get.DownloadMission.ERROR_SABR_DOWNLOAD; import static us.shandian.giga.get.DownloadMission.ERROR_SSL_EXCEPTION; import static us.shandian.giga.get.DownloadMission.ERROR_TIMEOUT; import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; @@ -547,6 +549,12 @@ private void showError(@NonNull DownloadMission mission) { case ERROR_RESOURCE_GONE: msg = R.string.error_download_resource_gone; break; + case ERROR_SABR_DOWNLOAD: + msg = sabrDownloadErrorMessage(mission.errObject); + if (mission.errObject != null && mission.errObject.getMessage() != null) { + msgEx = mission.errObject.getMessage(); + } + break; default: if (mission.errCode >= 100 && mission.errCode < 600) { msgEx = "HTTP " + mission.errCode; @@ -580,6 +588,32 @@ private void showError(@NonNull DownloadMission mission) { .show(); } + @StringRes + private int sabrDownloadErrorMessage(final Exception error) { + if (!(error instanceof SabrDownloadException)) { + return R.string.error_sabr_download_failed; + } + switch (((SabrDownloadException) error).getReason()) { + case FORMAT: + return R.string.error_sabr_download_format; + case INITIALIZATION: + return R.string.error_sabr_download_initialization; + case PROTECTED: + return R.string.error_sabr_download_protected; + case STALLED: + return R.string.error_sabr_download_stalled; + case NETWORK: + return R.string.error_sabr_download_network; + case MUXING: + return R.string.error_sabr_download_muxing; + case STORAGE: + return R.string.error_sabr_download_storage; + case PROTOCOL: + default: + return R.string.error_sabr_download_protocol; + } + } + private void showError(DownloadMission mission, UserAction action, @StringRes int reason) { StringBuilder request = new StringBuilder(256); request.append(mission.source); diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 3a8368d9c..6092917d7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -571,6 +571,15 @@ Progress lost, because the file was deleted Connection timeout Cannot recover this download +SABR download failed +This SABR format cannot be downloaded +SABR initialization failed +This SABR stream needs attestation +SABR stopped returning media +SABR download failed because of the network +Could not mux SABR media +Could not write the SABR download +SABR protocol error Clear download history Do you want to clear your download history or delete all downloaded files? Delete downloaded files