diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 218e7c4a7..c814eb23a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1509,19 +1509,18 @@ public void onFetchPage(@Nonnull final Downloader downloader) CancellableCall webPageCall = YoutubeParsingHelper.getWebPlayerResponse( localization, contentCountry, videoId, this); - final CancellableCall webSafariPlayerCall = fetchWebSafariJsonPlayer( - contentCountry, localization, videoId); - final CancellableCall configuredPlayerCall; + final CancellableCall jsonPlayerCall; switch (NewPipe.getYoutubePlayerClient()) { case "web_safari": - configuredPlayerCall = null; + jsonPlayerCall = fetchWebSafariJsonPlayer( + contentCountry, localization, videoId); break; case "web": - configuredPlayerCall = fetchWebJsonPlayer( + jsonPlayerCall = fetchWebJsonPlayer( contentCountry, localization, videoId); break; default: - configuredPlayerCall = fetchMwebJsonPlayer( + jsonPlayerCall = fetchMwebJsonPlayer( contentCountry, localization, videoId); break; } @@ -1562,8 +1561,7 @@ public void onSuccess(Response response) throws ExtractionException { } long startTime = System.nanoTime(); do { - if (webSafariPlayerCall.isFinished() - && (configuredPlayerCall == null || configuredPlayerCall.isFinished()) + if (jsonPlayerCall.isFinished() && webPageCall.isFinished() && nextDataCall.isFinished()) { break; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrDecodedResponse.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrDecodedResponse.java index 322fbd6ae..0b4069d53 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrDecodedResponse.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrDecodedResponse.java @@ -396,6 +396,10 @@ public boolean isProtectedNoMediaResponse() { return isNoMediaResponse() && streamProtectionStatus >= 3; } + public boolean isProtectionBoundaryNoMediaResponse() { + return isNoMediaResponse() && streamProtectionStatus >= 2; + } + @Nonnull public String summarizeNoMediaResponse() { return "parts=" + parts.size() diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMp4SegmentIndexParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMp4SegmentIndexParser.java index 72e37f26d..b520a9f08 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMp4SegmentIndexParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMp4SegmentIndexParser.java @@ -15,26 +15,41 @@ private SabrMp4SegmentIndexParser() { static SabrSegmentIndex parse(@Nonnull final byte[] initData, @Nonnull final SabrFormatInitializationMetadata metadata) throws SabrProtocolException { - final int indexStart = checkedRangeOffset(metadata.getIndexRangeStart(), initData.length); - final int indexEnd = checkedRangeOffset(metadata.getIndexRangeEnd(), initData.length); + return parse(initData, + checkedRangeOffset(metadata.getIndexRangeStart(), initData.length), + checkedRangeOffset(metadata.getIndexRangeEnd(), initData.length)); + } + + @Nonnull + static SabrSegmentIndex parse(@Nonnull final byte[] initData, + @Nonnull final YoutubeSabrFormat format) + throws SabrProtocolException { + return parse(initData, 0, initData.length - 1); + } + + @Nonnull + private static SabrSegmentIndex parse(@Nonnull final byte[] initData, + final int indexStart, + final int indexEnd) + throws SabrProtocolException { if (indexEnd < indexStart) { throw new SabrProtocolException("Invalid MP4 SIDX range"); } final int sidxOffset = findSidxBox(initData, indexStart, indexEnd + 1); - return parse(initData, sidxOffset, indexEnd + 1); + return parseSidx(initData, sidxOffset, indexEnd + 1); } @Nonnull static SabrSegmentIndex parse(@Nonnull final byte[] initData) throws SabrProtocolException { final int sidxOffset = findSidxBox(initData, 0, initData.length); - return parse(initData, sidxOffset, initData.length); + return parseSidx(initData, sidxOffset, initData.length); } @Nonnull - private static SabrSegmentIndex parse(@Nonnull final byte[] initData, - final int sidxOffset, - final int rangeEnd) + private static SabrSegmentIndex parseSidx(@Nonnull final byte[] initData, + final int sidxOffset, + final int rangeEnd) throws SabrProtocolException { final long boxSize = readUint32(initData, sidxOffset); final int boxEnd = checkedBoxEnd(sidxOffset, boxSize, rangeEnd); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrWebmSegmentIndexParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrWebmSegmentIndexParser.java index 4b56c6228..18a232f09 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrWebmSegmentIndexParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrWebmSegmentIndexParser.java @@ -20,38 +20,36 @@ private SabrWebmSegmentIndexParser() { static SabrSegmentIndex parse(@Nonnull final byte[] initData, @Nonnull final SabrFormatInitializationMetadata metadata) throws SabrProtocolException { - final Element segment = findElement(initData, 0, initData.length, SEGMENT_ID); - final long timecodeScaleNanos = readTimecodeScale(initData, segment); - final Element cues = findElement(initData, - checkedRangeOffset(metadata.getIndexRangeStart(), initData.length), - checkedRangeEnd(metadata.getIndexRangeEnd(), initData.length), CUES_ID); - final long totalDurationMs = metadata.getDurationUnits() > 0 + return parse(initData, metadata, metadata.getDurationUnits() > 0 && metadata.getDurationTimescale() > 0 ? scaleToMs(metadata.getDurationUnits(), metadata.getDurationTimescale()) - : -1; - return parse(initData, cues, timecodeScaleNanos, totalDurationMs); + : -1); } @Nonnull static SabrSegmentIndex parse(@Nonnull final byte[] initData, - final long totalDurationMs) + @Nonnull final YoutubeSabrFormat format) throws SabrProtocolException { - final Element segment = findElement(initData, 0, initData.length, SEGMENT_ID); - final long timecodeScaleNanos = readTimecodeScale(initData, segment); - final Element cues = findElement(initData, 0, initData.length, CUES_ID); - return parse(initData, cues, timecodeScaleNanos, totalDurationMs); + return parse(initData, null, format.getApproxDurationMs()); } @Nonnull private static SabrSegmentIndex parse(@Nonnull final byte[] initData, - @Nonnull final Element cues, - final long timecodeScaleNanos, + final SabrFormatInitializationMetadata metadata, final long totalDurationMs) throws SabrProtocolException { + final Element segment = findElement(initData, 0, initData.length, SEGMENT_ID); + final long timecodeScaleNanos = readTimecodeScale(initData, segment); + final Element cues = metadata == null + ? findElement(initData, segment.contentStart, segment.contentEnd, CUES_ID) + : findElement(initData, + checkedRangeOffset(metadata.getIndexRangeStart(), initData.length), + checkedRangeEnd(metadata.getIndexRangeEnd(), initData.length), CUES_ID); final List cueTimes = readCueTimes(initData, cues, timecodeScaleNanos); if (cueTimes.isEmpty()) { throw new SabrProtocolException("WebM cues contain no cue times"); } + final List entries = new ArrayList<>(cueTimes.size()); for (int i = 0; i < cueTimes.size(); i++) { final long startMs = cueTimes.get(i); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrFormat.java index 61ed51049..0112ac3cf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrFormat.java @@ -152,13 +152,8 @@ private static String decodeStreamingUrl(@Nonnull final String videoId, final String signature = YoutubeJavaScriptPlayerManager .deobfuscateSignature(videoId, obfuscatedSignature); final String separator = url.contains("?") ? "&" : "?"; - try { - url = url + separator + URLEncoder.encode(signatureParameter, - StandardCharsets.UTF_8.name()) - + '=' + URLEncoder.encode(signature, StandardCharsets.UTF_8.name()); - } catch (final UnsupportedEncodingException e) { - throw new ParsingException("Could not encode signature", e); - } + url = url + separator + urlEncode(signatureParameter) + '=' + + urlEncode(signature); } } return YoutubeSabrProbe.maybeDeobfuscateNParameter(videoId, url); @@ -177,19 +172,30 @@ private static Map parseQuery(@Nullable final String value) if (equals <= 0) { continue; } - try { - final String key = URLDecoder.decode(part.substring(0, equals), - StandardCharsets.UTF_8.name()); - final String decodedValue = URLDecoder.decode(part.substring(equals + 1), - StandardCharsets.UTF_8.name()); - params.put(key, decodedValue); - } catch (final UnsupportedEncodingException e) { - throw new ParsingException("Could not decode query", e); - } + params.put(urlDecode(part.substring(0, equals)), + urlDecode(part.substring(equals + 1))); } return params; } + @Nonnull + private static String urlEncode(@Nonnull final String value) throws ParsingException { + try { + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); + } catch (final UnsupportedEncodingException e) { + throw new ParsingException("Could not encode SABR signature cipher", e); + } + } + + @Nonnull + private static String urlDecode(@Nonnull final String value) throws ParsingException { + try { + return URLDecoder.decode(value, StandardCharsets.UTF_8.name()); + } catch (final UnsupportedEncodingException e) { + throw new ParsingException("Could not decode SABR signature cipher", e); + } + } + private static long parseLong(@Nullable final Object value) { if (value instanceof Number) { return ((Number) value).longValue(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java index f4ea5d09d..184a2138f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java @@ -533,20 +533,14 @@ static String maybeDeobfuscateNParameter(@Nonnull final String videoId, final java.util.regex.Matcher queryMatcher = java.util.regex.Pattern.compile("([?&])n=([^&]+)") .matcher(url); if (queryMatcher.find()) { - try { - final String encryptedN = java.net.URLDecoder.decode(queryMatcher.group(2), - StandardCharsets.UTF_8.name()); - final org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder.BatchDecodeResult result = - YoutubeJavaScriptPlayerManager.deobfuscateBatch(videoId, null, - Collections.singletonList(encryptedN)); - final String decryptedN = result.getNParameters().get(encryptedN); - if (decryptedN != null) { - return url.substring(0, queryMatcher.start(2)) - + java.net.URLEncoder.encode(decryptedN, StandardCharsets.UTF_8.name()) - + url.substring(queryMatcher.end(2)); - } - } catch (final UnsupportedEncodingException e) { - throw new ParsingException("Could not decode n parameter", e); + final String encryptedN = urlDecode(queryMatcher.group(2)); + final org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder.BatchDecodeResult result = + YoutubeJavaScriptPlayerManager.deobfuscateBatch(videoId, null, + Collections.singletonList(encryptedN)); + final String decryptedN = result.getNParameters().get(encryptedN); + if (decryptedN != null) { + return url.substring(0, queryMatcher.start(2)) + urlEncode(decryptedN) + + url.substring(queryMatcher.end(2)); } } @@ -562,4 +556,22 @@ static String maybeDeobfuscateNParameter(@Nonnull final String videoId, final String decryptedN = result.getNParameters().get(encryptedN); return decryptedN == null ? url : url.replace("/n/" + encryptedN, "/n/" + decryptedN); } + + @Nonnull + private static String urlEncode(@Nonnull final String value) throws ParsingException { + try { + return java.net.URLEncoder.encode(value, StandardCharsets.UTF_8.name()); + } catch (final UnsupportedEncodingException e) { + throw new ParsingException("Could not encode SABR URL parameter", e); + } + } + + @Nonnull + private static String urlDecode(@Nonnull final String value) throws ParsingException { + try { + return java.net.URLDecoder.decode(value, StandardCharsets.UTF_8.name()); + } catch (final UnsupportedEncodingException e) { + throw new ParsingException("Could not decode SABR URL parameter", e); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java index 8923cbbeb..2d08ea386 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java @@ -138,7 +138,7 @@ public SabrMediaSegment fetchSegment(@Nonnull final SabrSegmentRequest request, + describeRequest(request) + " (reload budget spent): " + decoded.summarizeNoMediaResponse()); } - if (decoded.isProtectedNoMediaResponse()) { + if (decoded.isProtectionBoundaryNoMediaResponse()) { if (applyPoTokenForProtectedResponse()) { if (decoded.getBackoffTimeMs() > 0) { sleepBackoff(decoded.getBackoffTimeMs()); @@ -265,10 +265,10 @@ public List pumpOnce(@Nonnull final Localization localization) throw new SabrProtocolException("SABR requested player reload (reload budget spent): " + decoded.summarizeNoMediaResponse()); } - if (decoded.isProtectedNoMediaResponse()) { - // mint / re-mint the token, best effort. don't throw on a single status=3: it's normal - // pacing, the server usually clears it next round. pump keeps trying; the stall watchdog - // is the real give-up. + if (decoded.isProtectionBoundaryNoMediaResponse()) { + // Mint / re-mint the token as soon as SABR reaches the protection boundary. Do not throw + // on a single no-media round: the server usually clears it next round. The pump keeps + // trying; the stall watchdog is the real give-up. applyPoTokenForProtectedResponse(); } if (!segments.isEmpty()) { @@ -372,6 +372,15 @@ public SabrMediaSegment getCachedSegment(@Nonnull final SabrSegmentRequest reque return segmentCache.get(cacheKey(request)); } + public void discardCachedSegment(@Nonnull final SabrSegmentRequest request) { + final String key = cacheKey(request); + final SabrMediaSegment removed = segmentCache.remove(key); + if (removed != null && !removed.getHeader().isInitSegment()) { + cacheOrder.remove(key); + cachedBytes = Math.max(0, cachedBytes - removed.getLength()); + } + } + /** True once the requested media segment is known to be past the last segment of the stream. */ public boolean isBeyondEnd(@Nonnull final SabrSegmentRequest request) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrStreamState.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrStreamState.java index 9833c6511..41fdb6169 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrStreamState.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrStreamState.java @@ -11,6 +11,13 @@ import java.util.Set; public final class YoutubeSabrStreamState { + public static final int TRACK_MODE_VIDEO_AND_AUDIO = + YoutubeSabrRequestBuilder.ENABLED_TRACK_TYPES_VIDEO_AND_AUDIO; + public static final int TRACK_MODE_AUDIO_ONLY = + YoutubeSabrRequestBuilder.ENABLED_TRACK_TYPES_AUDIO_ONLY; + public static final int TRACK_MODE_VIDEO_ONLY = + YoutubeSabrRequestBuilder.ENABLED_TRACK_TYPES_VIDEO_ONLY; + private final FormatProgress audio; private final FormatProgress video; private final Map sabrContexts = new LinkedHashMap<>(); @@ -177,6 +184,10 @@ public long getMinBufferedEndMs() { return Math.min(audio.getBufferedEndMs(), video.getBufferedEndMs()); } + public long getBufferedEndMs(@Nonnull final YoutubeSabrFormat format) { + return progressForItag(format.getItag()).getBufferedEndMs(); + } + public void setPlayerTimeMs(final long playerTimeMs) { playerTimeMsOverride = Math.max(0, playerTimeMs); } @@ -371,6 +382,18 @@ public void setActiveTrackTypes(final boolean videoActive, final boolean audioAc } } + public void setAudioOnlyRequestMode() { + setRequestTrackMode(TRACK_MODE_AUDIO_ONLY, true, false); + } + + public void setVideoOnlyRequestMode() { + setRequestTrackMode(TRACK_MODE_VIDEO_ONLY, false, true); + } + + public void setVideoAndAudioRequestMode() { + setRequestTrackMode(TRACK_MODE_VIDEO_AND_AUDIO, true, true); + } + private boolean isAudioEnabled() { return enabledTrackTypesBitfield != YoutubeSabrRequestBuilder.ENABLED_TRACK_TYPES_VIDEO_ONLY; @@ -686,49 +709,65 @@ private boolean observeSegment(@Nonnull final SabrMediaSegment segment) { if (!segment.getHeader().isInitSegment() || metadata == null || segmentIndex != null) { return false; } - final String mimeType = metadata.getMimeType(); - if (mimeType == null) { - return false; - } - try { - if (mimeType.contains("mp4")) { - segmentIndex = SabrMp4SegmentIndexParser.parse(segment.getData(), metadata); - } else if (mimeType.contains("webm")) { - segmentIndex = SabrWebmSegmentIndexParser.parse(segment.getData(), metadata); - } else { - return false; - } - return true; - } catch (final SabrProtocolException ignored) { - return false; - } + return observeInitializationData(segment.getData()); } private boolean observeInitializationData(@Nonnull final byte[] data) { if (segmentIndex != null) { return false; } - final String mimeType = format.getMimeType(); + final String mimeType = metadata == null ? format.getMimeType() : metadata.getMimeType(); if (mimeType == null) { return false; } try { if (mimeType.contains("mp4")) { - segmentIndex = SabrMp4SegmentIndexParser.parse(data); + segmentIndex = metadata == null + ? SabrMp4SegmentIndexParser.parse(data, format) + : SabrMp4SegmentIndexParser.parse(data, metadata); } else if (mimeType.contains("webm")) { - segmentIndex = SabrWebmSegmentIndexParser.parse(data, - format.getApproxDurationMs()); + segmentIndex = metadata == null + ? SabrWebmSegmentIndexParser.parse(data, format) + : SabrWebmSegmentIndexParser.parse(data, metadata); } else { return false; } - initReceived = true; - endSegment = segmentIndex.size(); + observeSegmentIndex(); return true; } catch (final SabrProtocolException ignored) { + if (metadata == null) { + return false; + } + try { + if (mimeType.contains("mp4")) { + segmentIndex = SabrMp4SegmentIndexParser.parse(data, format); + } else if (mimeType.contains("webm")) { + segmentIndex = SabrWebmSegmentIndexParser.parse(data, format); + } else { + return false; + } + observeSegmentIndex(); + return true; + } catch (final SabrProtocolException ignoredFallback) { + return false; + } + } catch (final Exception ignored) { return false; } } + private void observeSegmentIndex() { + if (segmentIndex == null) { + return; + } + if (endSegment <= 0) { + endSegment = segmentIndex.size(); + } + if (format.getApproxDurationMs() > 0 && endSegment > 0) { + averageDurationMs = Math.max(1L, format.getApproxDurationMs() / endSegment); + } + } + private boolean observeHeader(@Nonnull final SabrMediaHeader header) { if (header.isInitSegment()) { final boolean changed = !initReceived;