From 6383c51cb6688cfd3c371444934493116a354195 Mon Sep 17 00:00:00 2001 From: FireMasterK <20838718+FireMasterK@users.noreply.github.com> Date: Sun, 20 Dec 2020 01:23:10 +0530 Subject: [PATCH] Add dynamic itag support. --- .../extractor/services/youtube/ItagItem.java | 61 +++++++++++++---- .../extractors/YoutubeStreamExtractor.java | 65 ++++++++++--------- 2 files changed, 84 insertions(+), 42 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java index 0b36768636..9d1239e94a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java @@ -57,6 +57,9 @@ public class ItagItem { new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"), new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60), new ItagItem(266, VIDEO_ONLY, MPEG_4, "2160p"), + new ItagItem(402, VIDEO_ONLY, MPEG_4, "4320p"), // can be 4320p60 as well + new ItagItem(571, VIDEO_ONLY, MPEG_4, "4320p"), // can be 4320p60 HDR as well (1La4QzGeaaQ) + new ItagItem(402, VIDEO_ONLY, MPEG_4, "4320p60"), new ItagItem(278, VIDEO_ONLY, WEBM, "144p"), new ItagItem(242, VIDEO_ONLY, WEBM, "240p"), @@ -67,27 +70,17 @@ public class ItagItem { new ItagItem(247, VIDEO_ONLY, WEBM, "720p"), new ItagItem(248, VIDEO_ONLY, WEBM, "1080p"), new ItagItem(271, VIDEO_ONLY, WEBM, "1440p"), - // #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) - new ItagItem(272, VIDEO_ONLY, WEBM, "2160p"), new ItagItem(302, VIDEO_ONLY, WEBM, "720p60", 60), new ItagItem(303, VIDEO_ONLY, WEBM, "1080p60", 60), new ItagItem(308, VIDEO_ONLY, WEBM, "1440p60", 60), new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"), - new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60) + new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60), + new ItagItem(272, VIDEO_ONLY, WEBM, "4320p60", 60) }; /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ - public static boolean isSupported(int itag) { - for (ItagItem item : ITAG_LIST) { - if (itag == item.id) { - return true; - } - } - return false; - } - public static ItagItem getItag(int itagId) throws ParsingException { for (ItagItem item : ITAG_LIST) { if (itagId == item.id) { @@ -97,6 +90,50 @@ public static ItagItem getItag(int itagId) throws ParsingException { throw new ParsingException("itag=" + itagId + " not supported"); } + public static ItagItem getItag(int itagId, int averageBitrate, int fps, String qualityLabel, String mimeType) throws ParsingException { + + String[] split = mimeType.split(";")[0].split("/"); + String streamType = split[0]; + String fileType = split[1]; + String codec = mimeType.split("\"")[1]; + + MediaFormat format = null; + ItagType itagType = null; + + if (codec.contains(",")) + itagType = VIDEO; + else { + if (streamType.equals("video")) + itagType = VIDEO_ONLY; + if (streamType.equals("audio")) + itagType = AUDIO; + } + + if (itagType == AUDIO) { + if (fileType.equals("mp4") && (codec.startsWith("m4a") || codec.startsWith("mp4a") )) + format = M4A; + if (fileType.startsWith("webm") && codec.equals("opus")) + format = WEBMA_OPUS; + } + + if (itagType == VIDEO) { + if (fileType.equals("mp4")) + format = MPEG_4; + } + + if (itagType == VIDEO_ONLY) { + if (fileType.equals("mp4")) + format = MPEG_4; + if (fileType.equals("webm")) + format = WEBM; + } + + if (itagType == null || format == null) + throw new ParsingException("Unknown mimeType: " + mimeType); + + return itagType == AUDIO ? new ItagItem(itagId, itagType, format, Math.round(averageBitrate / 1024f)) : new ItagItem(itagId, itagType, format, qualityLabel, fps); + } + /*////////////////////////////////////////////////////////////////////////// // Contructors and misc //////////////////////////////////////////////////////////////////////////*/ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index cca69dd219..61013e77d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -98,17 +98,21 @@ public static class DeobfuscateException extends ParsingException { /*//////////////////////////////////////////////////////////////////////////*/ - @Nullable private static String cachedDeobfuscationCode = null; - @Nullable private String playerJsUrl = null; + @Nullable + private static String cachedDeobfuscationCode = null; + @Nullable + private String playerJsUrl = null; private JsonArray initialAjaxJson; private JsonObject initialData; - @Nonnull private final Map videoInfoPage = new HashMap<>(); + @Nonnull + private final Map videoInfoPage = new HashMap<>(); private JsonObject playerResponse; private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; private int ageLimit = -1; - @Nullable private List subtitles = null; + @Nullable + private List subtitles = null; public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { super(service, linkHandler); @@ -776,8 +780,6 @@ private String getEmbeddedInfoStsAndStorePlayerJsUrl() { } - - private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { Parser.RegexException exception = null; for (final String regex : REGEXES) { @@ -935,35 +937,38 @@ private Map getItags(final String streamingDataKey, for (int i = 0; i != formats.size(); ++i) { JsonObject formatData = formats.getObject(i); int itag = formatData.getInt("itag"); + int averageBitrate = formatData.getInt("averageBitrate"); + int fps = formatData.getInt("fps"); + String qualityLabel = formatData.getString("qualityLabel"); + String mimeType = formatData.getString("mimeType"); - if (ItagItem.isSupported(itag)) { - try { - ItagItem itagItem = ItagItem.getItag(itag); - if (itagItem.itagType == itagTypeWanted) { - // Ignore streams that are delivered using YouTube's OTF format, - // as those only work with DASH and not with progressive HTTP. - if (formatData.getString("type", EMPTY_STRING) - .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { - continue; - } - String streamUrl; - if (formatData.has("url")) { - streamUrl = formatData.getString("url"); - } else { - // this url has an obfuscated signature - final String cipherString = formatData.has("cipher") - ? formatData.getString("cipher") - : formatData.getString("signatureCipher"); - final Map cipher = Parser.compatParseMap(cipherString); - streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" - + deobfuscateSignature(cipher.get("s")); - } + try { + ItagItem itagItem = ItagItem.getItag(itag, averageBitrate, fps, qualityLabel, mimeType); + if (itagItem.itagType == itagTypeWanted) { + // Ignore streams that are delivered using YouTube's OTF format, + // as those only work with DASH and not with progressive HTTP. + if (formatData.getString("type", EMPTY_STRING) + .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { + continue; + } - urlAndItags.put(streamUrl, itagItem); + String streamUrl; + if (formatData.has("url")) { + streamUrl = formatData.getString("url"); + } else { + // this url has an obfuscated signature + final String cipherString = formatData.has("cipher") + ? formatData.getString("cipher") + : formatData.getString("signatureCipher"); + final Map cipher = Parser.compatParseMap(cipherString); + streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + + deobfuscateSignature(cipher.get("s")); } - } catch (UnsupportedEncodingException ignored) { + + urlAndItags.put(streamUrl, itagItem); } + } catch (UnsupportedEncodingException ignored) { } }