Skip to content

Commit

Permalink
Add dynamic itag support.
Browse files Browse the repository at this point in the history
  • Loading branch information
@FireMasterK
FireMasterK committed Dec 19, 2020
1 parent 22a4151 commit 6383c51
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 42 deletions.
61 changes: 49 additions & 12 deletions extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ public class ItagItem {
new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"),
new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60),
new ItagItem(266, VIDEO_ONLY, MPEG_4, "2160p"),
new ItagItem(402, VIDEO_ONLY, MPEG_4, "4320p"), // can be 4320p60 as well
new ItagItem(571, VIDEO_ONLY, MPEG_4, "4320p"), // can be 4320p60 HDR as well (1La4QzGeaaQ)
new ItagItem(402, VIDEO_ONLY, MPEG_4, "4320p60"),

new ItagItem(278, VIDEO_ONLY, WEBM, "144p"),
new ItagItem(242, VIDEO_ONLY, WEBM, "240p"),
Expand All @@ -67,27 +70,17 @@ public class ItagItem {
new ItagItem(247, VIDEO_ONLY, WEBM, "720p"),
new ItagItem(248, VIDEO_ONLY, WEBM, "1080p"),
new ItagItem(271, VIDEO_ONLY, WEBM, "1440p"),
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
new ItagItem(272, VIDEO_ONLY, WEBM, "2160p"),
new ItagItem(302, VIDEO_ONLY, WEBM, "720p60", 60),
new ItagItem(303, VIDEO_ONLY, WEBM, "1080p60", 60),
new ItagItem(308, VIDEO_ONLY, WEBM, "1440p60", 60),
new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"),
new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60)
new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60),
new ItagItem(272, VIDEO_ONLY, WEBM, "4320p60", 60)
};
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/

public static boolean isSupported(int itag) {
for (ItagItem item : ITAG_LIST) {
if (itag == item.id) {
return true;
}
}
return false;
}

public static ItagItem getItag(int itagId) throws ParsingException {
for (ItagItem item : ITAG_LIST) {
if (itagId == item.id) {
Expand All @@ -97,6 +90,50 @@ public static ItagItem getItag(int itagId) throws ParsingException {
throw new ParsingException("itag=" + itagId + " not supported");
}

public static ItagItem getItag(int itagId, int averageBitrate, int fps, String qualityLabel, String mimeType) throws ParsingException {

String[] split = mimeType.split(";")[0].split("/");
String streamType = split[0];
String fileType = split[1];
String codec = mimeType.split("\"")[1];

MediaFormat format = null;
ItagType itagType = null;

if (codec.contains(","))
itagType = VIDEO;
else {
if (streamType.equals("video"))
itagType = VIDEO_ONLY;
if (streamType.equals("audio"))
itagType = AUDIO;
}

if (itagType == AUDIO) {
if (fileType.equals("mp4") && (codec.startsWith("m4a") || codec.startsWith("mp4a") ))
format = M4A;
if (fileType.startsWith("webm") && codec.equals("opus"))
format = WEBMA_OPUS;
}

if (itagType == VIDEO) {
if (fileType.equals("mp4"))
format = MPEG_4;
}

if (itagType == VIDEO_ONLY) {
if (fileType.equals("mp4"))
format = MPEG_4;
if (fileType.equals("webm"))
format = WEBM;
}

if (itagType == null || format == null)
throw new ParsingException("Unknown mimeType: " + mimeType);

return itagType == AUDIO ? new ItagItem(itagId, itagType, format, Math.round(averageBitrate / 1024f)) : new ItagItem(itagId, itagType, format, qualityLabel, fps);
}

/*//////////////////////////////////////////////////////////////////////////
// Contructors and misc
//////////////////////////////////////////////////////////////////////////*/
Expand Down
65 changes: 35 additions & 30 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,21 @@ public static class DeobfuscateException extends ParsingException {

/*//////////////////////////////////////////////////////////////////////////*/

@Nullable private static String cachedDeobfuscationCode = null;
@Nullable private String playerJsUrl = null;
@Nullable
private static String cachedDeobfuscationCode = null;
@Nullable
private String playerJsUrl = null;

private JsonArray initialAjaxJson;
private JsonObject initialData;
@Nonnull private final Map<String, String> videoInfoPage = new HashMap<>();
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit = -1;
@Nullable private List<SubtitlesStream> subtitles = null;
@Nullable
private List<SubtitlesStream> subtitles = null;

public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
Expand Down Expand Up @@ -776,8 +780,6 @@ private String getEmbeddedInfoStsAndStorePlayerJsUrl() {
}




private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
Expand Down Expand Up @@ -935,35 +937,38 @@ private Map<String, ItagItem> getItags(final String streamingDataKey,
for (int i = 0; i != formats.size(); ++i) {
JsonObject formatData = formats.getObject(i);
int itag = formatData.getInt("itag");
int averageBitrate = formatData.getInt("averageBitrate");
int fps = formatData.getInt("fps");
String qualityLabel = formatData.getString("qualityLabel");
String mimeType = formatData.getString("mimeType");

if (ItagItem.isSupported(itag)) {
try {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == itagTypeWanted) {
// Ignore streams that are delivered using YouTube's OTF format,
// as those only work with DASH and not with progressive HTTP.
if (formatData.getString("type", EMPTY_STRING)
.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
continue;
}

String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// this url has an obfuscated signature
final String cipherString = formatData.has("cipher")
? formatData.getString("cipher")
: formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ deobfuscateSignature(cipher.get("s"));
}
try {
ItagItem itagItem = ItagItem.getItag(itag, averageBitrate, fps, qualityLabel, mimeType);
if (itagItem.itagType == itagTypeWanted) {
// Ignore streams that are delivered using YouTube's OTF format,
// as those only work with DASH and not with progressive HTTP.
if (formatData.getString("type", EMPTY_STRING)
.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
continue;
}

urlAndItags.put(streamUrl, itagItem);
String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// this url has an obfuscated signature
final String cipherString = formatData.has("cipher")
? formatData.getString("cipher")
: formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ deobfuscateSignature(cipher.get("s"));
}
} catch (UnsupportedEncodingException ignored) {

urlAndItags.put(streamUrl, itagItem);
}
} catch (UnsupportedEncodingException ignored) {
}
}

Expand Down

0 comments on commit 6383c51

Please sign in to comment.