diff --git a/.gitignore b/.gitignore index 4a54be9..4980d19 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,13 @@ out/ !**/src/test/**/out/ /src/test/java/test.java +/src/test/java/com/github/felipeucelli/javatube/CaptionsTest.java +/src/test/java/com/github/felipeucelli/javatube/ChannelTest.java +/src/test/java/com/github/felipeucelli/javatube/InnerTubeTest.java +/src/test/java/com/github/felipeucelli/javatube/PlaylistTest.java +/src/test/java/com/github/felipeucelli/javatube/SearchTest.java +/src/test/java/com/github/felipeucelli/javatube/StreamQueryTest.java +/src/test/java/com/github/felipeucelli/javatube/YoutubeTest.java ### Eclipse ### .apt_generated @@ -46,3 +53,5 @@ bin/ *.mp3 *.webm *.srt + +.cache/* \ No newline at end of file diff --git a/README.md b/README.md index 79dbddd..e9d793f 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ _JavaTube_ is a library written in java and aims to be highly reliable. * Support downloading yt_otf streams * Possibility to choose the client (WEB, ANDROID, IOS) * Native js interpreter +* PoToken support ## Contribution Currently this project is maintained by only one person. Feel free to create issues with questions, bug reports or improvement ideas. @@ -72,15 +73,19 @@ Because the dubbed audio tracks have the same tag, we have to filter by name. This will only list tracks dubbed in the chosen language: ```java -for(Stream s : new Youtube("https://www.youtube.com/watch?v=g_VxOIlg7q8").streams().getExtraAudioTracksByName("English").getAll()){ - System.out.println(s.getItag() + " " + s.getAudioTrackName() + " " + s.getAbr() + " " + s.getUrl()); +public static void main(String[] args) throws Exception { + for(Stream s : new Youtube("https://www.youtube.com/watch?v=g_VxOIlg7q8").streams().getExtraAudioTracksByName("English").getAll()){ + System.out.println(s.getItag() + " " + s.getAudioTrackName() + " " + s.getAbr() + " " + s.getUrl()); + } } ``` You can check the dubbed tracks using: ```java -for(Stream s : new Youtube("https://www.youtube.com/watch?v=g_VxOIlg7q8").streams().getExtraAudioTracks().getAll()){ - System.out.println(s.getItag() + " " + s.getAudioTrackName() + " " + s.getAbr() + " " + s.getUrl()); +public static void main(String[] args) throws Exception { + for(Stream s : new Youtube("https://www.youtube.com/watch?v=g_VxOIlg7q8").streams().getExtraAudioTracks().getAll()){ + System.out.println(s.getItag() + " " + s.getAudioTrackName() + " " + s.getAbr() + " " + s.getUrl()); + } } ``` @@ -191,7 +196,6 @@ public static void main(String[] args) throws Exception { System.out.println(caption.getCode()); } } -} ``` Write to console in .srt format. @@ -200,7 +204,6 @@ Write to console in .srt format. public static void main(String[] args) throws Exception { System.out.println(new Youtube("https://www.youtube.com/watch?v=2lAe1cqCOXo&t=1s").getCaptions().getByCode("en").xmlCaptionToSrt()); } -} ``` Download it in .srt format (if the .srt format is not informed, the xml will be downloaded). @@ -209,8 +212,37 @@ Download it in .srt format (if the .srt format is not informed, the xml will be public static void main(String[] args) throws Exception { new Youtube("https://www.youtube.com/watch?v=2lAe1cqCOXo&t=1s").getCaptions().getByCode("en").download("caption.srt", "./") } +``` + +## PoToken +The proof of origin (PO) token is a parameter that YouTube requires to be sent with video playback requests from some clients. Without it, format URL requests from affected customers may return HTTP error 403, error with bot detection, or result in your account or IP address being blocked. + +This token is generated by BotGuard (Web) / DroidGuard (Android) to attest the requests are coming from a genuine client. + +### Manually acquiring a PO Token from a browser for use when logged out +This process involves manually obtaining a PO token generated from YouTube in a web browser and then manually passing it to JavaTube via the usePoToken=True argument. Steps: + +1. Open a browser and go to any video on YouTube Music or YouTube Embedded (e.g. https://www.youtube.com/embed/2lAe1cqCOXo). Make sure you are not logged in to any account! + +2. Open the developer console (F12), then go to the "Network" tab and filter by v1/player + +3. Click the video to play and a player request will appear in the network tab + +4. In the request payload JSON, find the PO Token at serviceIntegrityDimensions.poToken and save that value + +5. In the request payload JSON, find the visitorData at context.client.visitorData and save that value + +6. In your code, pass the parameter usePoToken=True, to send the visitorData and PoToken: + +```java +public static void main(String[] args) throws Exception { + Youtube yt = new Youtube("https://www.youtube.com/watch?v=2lAe1cqCOXo", true); + yt.streams().getHighestResolution().download("./"); } ``` +The terminal will ask you to insert the tokens. + +If you want to save the token in cache, just add one more argument `true`, this will create a _cache/tokens.json_ file where the visitorData and poToken will be stored. ## Filters Parameters: * `"res"` The video resolution (e.g.: "360p", "720p") diff --git a/src/main/java/com/github/felipeucelli/javatube/InnerTube.java b/src/main/java/com/github/felipeucelli/javatube/InnerTube.java index 56f09ff..1ad1de3 100644 --- a/src/main/java/com/github/felipeucelli/javatube/InnerTube.java +++ b/src/main/java/com/github/felipeucelli/javatube/InnerTube.java @@ -4,42 +4,26 @@ import org.json.JSONObject; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Objects; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; -class InnerTube{ +public class InnerTube{ private static JSONObject innerTubeContext; private static boolean requireJsPlayer; private static JSONObject header; private static String apiKey; - /** - * @Clients: - * WEB, - * WEB_EMBED, - * WEB_MUSIC, - * WEB_CREATOR, - * WEB_SAFARI, - * MWEB, - * ANDROID, - * ANDROID_VR, - * ANDROID_MUSIC, - * ANDROID_CREATOR, - * ANDROID_TESTSUITE, - * ANDROID_PRODUCER, - * IOS, - * IOS_MUSIC, - * IOS_CREATOR, - * TV_EMBED, - * MEDIA_CONNECT - * */ - public InnerTube(String client) throws JSONException { - JSONObject defaultClient = new JSONObject(""" + private final boolean usePoToken; + private String accessPoToken; + private String accessVisitorData; + + JSONObject defaultClient = new JSONObject(""" { "WEB": { "innerTubeContext": { @@ -379,13 +363,99 @@ public InnerTube(String client) throws JSONException { } """); + + /** + * @Clients: + * WEB, + * WEB_EMBED, + * WEB_MUSIC, + * WEB_CREATOR, + * WEB_SAFARI, + * MWEB, + * ANDROID, + * ANDROID_VR, + * ANDROID_MUSIC, + * ANDROID_CREATOR, + * ANDROID_TESTSUITE, + * ANDROID_PRODUCER, + * IOS, + * IOS_MUSIC, + * IOS_CREATOR, + * TV_EMBED, + * MEDIA_CONNECT + * */ + public InnerTube(String client, boolean usePoToken, boolean allowCache) throws JSONException { + innerTubeContext = defaultClient.getJSONObject(client).getJSONObject("innerTubeContext"); requireJsPlayer = defaultClient.getJSONObject(client).getBoolean("requireJsPlayer"); header = defaultClient.getJSONObject(client).getJSONObject("header"); // API keys are not required, see: https://github.com/TeamNewPipe/NewPipeExtractor/pull/1168 apiKey = defaultClient.getJSONObject(client).getString("apiKey"); + + this.usePoToken = usePoToken; + + try { + Path path = Paths.get(".cache/tokens.json"); + if(usePoToken && allowCache && Files.exists(path)){ + String content = new String(Files.readAllBytes(path)); + JSONObject jsonObject = new JSONObject(content); + accessVisitorData = jsonObject.getString("visitorData"); + accessPoToken = jsonObject.getString("poToken"); + } + }catch (IOException e){ + e.printStackTrace(); + } } + + /** + * @Clients: + * WEB, + * WEB_EMBED, + * WEB_MUSIC, + * WEB_CREATOR, + * WEB_SAFARI, + * MWEB, + * ANDROID, + * ANDROID_VR, + * ANDROID_MUSIC, + * ANDROID_CREATOR, + * ANDROID_TESTSUITE, + * ANDROID_PRODUCER, + * IOS, + * IOS_MUSIC, + * IOS_CREATOR, + * TV_EMBED, + * MEDIA_CONNECT + * */ + public InnerTube(String client, boolean usePoToken) throws JSONException { + this(client, usePoToken, false); + } + + /** + * @Clients: + * WEB, + * WEB_EMBED, + * WEB_MUSIC, + * WEB_CREATOR, + * WEB_SAFARI, + * MWEB, + * ANDROID, + * ANDROID_VR, + * ANDROID_MUSIC, + * ANDROID_CREATOR, + * ANDROID_TESTSUITE, + * ANDROID_PRODUCER, + * IOS, + * IOS_MUSIC, + * IOS_CREATOR, + * TV_EMBED, + * MEDIA_CONNECT + * */ + public InnerTube(String client) throws JSONException { + this(client, false, false); + } + public JSONObject getInnerTubeContext() throws JSONException { return innerTubeContext; } @@ -410,6 +480,14 @@ public boolean getRequireJsPlayer(){ return requireJsPlayer; } + public String getVisitorData(){ + return accessVisitorData; + } + + public String getPoToken(){ + return accessPoToken; + } + private String getBaseUrl(){ return "https://www.youtube.com/youtubei/v1"; } @@ -417,6 +495,60 @@ private String getBaseUrl(){ private String getBaseParam(){ return "{prettyPrint: \"false\"}"; } + + private String[] defaultPoTokenVerifier(){ + Scanner scanner = new Scanner(System.in); + System.out.println("You can use the tool: https://github.com/YunzheZJU/youtube-po-token-generator, to get the token"); + System.out.print("Enter with your visitorData: "); + String visitorData = scanner.nextLine(); + System.out.print("Enter with your PoToken: "); + String poToken = scanner.nextLine(); + return new String[]{visitorData, poToken}; + } + + public void cacheTokens() throws JSONException { + if (usePoToken){ + JSONObject data = new JSONObject( + "{" + + "\"visitorData\": \"" + accessVisitorData + "\"," + + "\"poToken\": \"" + accessPoToken + "\"" + + "}" + ); + + String filePath = ".cache/tokens.json"; + try { + Path path = Paths.get(filePath); + Files.write(path, data.toString(4).getBytes()); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + public void insetPoToken() throws JSONException { + JSONObject context = new JSONObject( + "{" + + "\"context\": {" + + "\"client\": {" + + "\"visitorData\": \"" + accessVisitorData + "\"" + + "}"+ + "}," + + "\"serviceIntegrityDimensions\": {" + + "\"poToken\": \"" + accessPoToken + "\"" + + "}" + + "}" + ); + updateInnerTubeContext(innerTubeContext, context); + } + + public void fetchPoToken() throws JSONException { + String[] token = defaultPoTokenVerifier(); + accessVisitorData = token[0]; + accessPoToken = token[1]; + cacheTokens(); + insetPoToken(); + } + private String urlEncode(JSONObject json) throws JSONException, UnsupportedEncodingException { StringBuilder query = new StringBuilder(); for (Iterator it = json.keys(); it.hasNext(); ) { @@ -444,11 +576,19 @@ private Map getHeaderMap() throws JSONException { return headers; } - private JSONObject callApi(String endpoint, JSONObject query, JSONObject data) throws Exception { + private JSONObject callApi(String endpoint, JSONObject query) throws Exception { String endpointUrl = endpoint + "?" + urlEncode(query); - ByteArrayOutputStream response = Request.post(endpointUrl, data.toString(), getHeaderMap()); + if(usePoToken){ + if(accessPoToken != null){ + insetPoToken(); + }else { + fetchPoToken(); + } + } + + ByteArrayOutputStream response = Request.post(endpointUrl, getInnerTubeContext().toString(), getHeaderMap()); return new JSONObject(response.toString()); } @@ -457,14 +597,14 @@ public JSONObject player(String videoId) throws Exception { JSONObject query = new JSONObject(getBaseParam()); JSONObject context = new JSONObject("{videoId: " + videoId + ", " + "contentCheckOk: \"true\"" + "}"); updateInnerTubeContext(getInnerTubeContext(), context); - return callApi(endpoint, query, getInnerTubeContext()); + return callApi(endpoint, query); } public JSONObject browse(JSONObject data) throws Exception { String endpoint = getBaseUrl() + "/browse"; JSONObject query = new JSONObject(getBaseParam()); updateInnerTubeContext(getInnerTubeContext(), data); - return callApi(endpoint, query, getInnerTubeContext()); + return callApi(endpoint, query); } public JSONObject search(String searchQuery, String continuationToken) throws Exception { @@ -475,6 +615,6 @@ public JSONObject search(String searchQuery, String continuationToken) throws Ex if(!Objects.equals(continuationToken, "")){ updateInnerTubeContext(getInnerTubeContext(), new JSONObject("{continuation:" + continuationToken + "}")); } - return callApi(endpoint, query, getInnerTubeContext()); + return callApi(endpoint, query); } } \ No newline at end of file diff --git a/src/main/java/com/github/felipeucelli/javatube/Youtube.java b/src/main/java/com/github/felipeucelli/javatube/Youtube.java index 7a9920d..c0714f8 100644 --- a/src/main/java/com/github/felipeucelli/javatube/Youtube.java +++ b/src/main/java/com/github/felipeucelli/javatube/Youtube.java @@ -17,21 +17,18 @@ public class Youtube { private InnerTube innerTube = null; private String client = null; private JSONObject vidInfo = null; - private int poTokenAttempts = 1; private String html = null; private String js = null; private JSONObject ytCfg = null; private JSONObject signatureTimestamp = null; private String playerJs = null; + private final boolean usePoToken; /** * Default client: WEB * */ public Youtube(String url) throws Exception { - urlVideo = url; - watchUrl = "https://www.youtube.com/watch?v=" + videoId(); - client = "ANDROID_TESTSUITE"; - innerTube = new InnerTube(client); + this(url, "ANDROID_TESTSUITE", false, false); } /** * @Clients: @@ -54,8 +51,44 @@ public Youtube(String url) throws Exception { * MEDIA_CONNECT * */ public Youtube(String url, String clientName) throws Exception { - client = clientName; - innerTube = new InnerTube(client); + this(url, clientName, false, false); + } + /** + * Default client: WEB + * */ + public Youtube(String url, boolean usePoToken) throws Exception { + this(url, "ANDROID_TESTSUITE", usePoToken, false); + } + /** + * Default client: WEB + * */ + public Youtube(String url, boolean usePoToken, boolean allowCache) throws Exception { + this(url, "ANDROID_TESTSUITE", usePoToken, allowCache); + } + /** + * @Clients: + * WEB, + * WEB_EMBED, + * WEB_MUSIC, + * WEB_CREATOR, + * WEB_SAFARI, + * MWEB, + * ANDROID, + * ANDROID_VR, + * ANDROID_MUSIC, + * ANDROID_CREATOR, + * ANDROID_TESTSUITE, + * ANDROID_PRODUCER, + * IOS, + * IOS_MUSIC, + * IOS_CREATOR, + * TV_EMBED, + * MEDIA_CONNECT + * */ + public Youtube(String url, String clientName, boolean usePoToken, boolean allowCache) throws Exception { + client = usePoToken ? "WEB" : clientName; + this.usePoToken = usePoToken; + innerTube = new InnerTube(client, usePoToken, allowCache); urlVideo = url; watchUrl = "https://www.youtube.com/watch?v=" + videoId(); } @@ -188,49 +221,8 @@ private JSONObject getVidInfo() throws Exception { return vidInfo; } - void checkPoToken() throws Exception { - if(innerTube == null || client.contains("WEB")){ - // https://github.com/yt-dlp/yt-dlp/pull/10456 - List poTokenExperiments = List.of("51217476", "51217102"); - - String value = ""; - - for (int i = 0; i < vidInfo.getJSONObject("responseContext").getJSONArray("serviceTrackingParams").length(); i++){ - JSONObject service = vidInfo.getJSONObject("responseContext").getJSONArray("serviceTrackingParams").getJSONObject(i); - if (service.has("service")){ - JSONArray params = service.getJSONArray("params"); - for (int j = 0; j < params.length(); j++){ - if (Objects.equals(params.getJSONObject(j).getString("key"), "e")){ - value = params.getJSONObject(j).getString("value"); - break; - } - } - } - } - - for (String po : poTokenExperiments){ - if (value.contains(po)){ - if(poTokenAttempts <= 3){ - System.out.print("API returned broken formats (poToken experiment detected). "); - System.out.println("Trying again " + poTokenAttempts + "/3."); - vidInfo = null; - poTokenAttempts += 1; - checkAvailability(); - break; - } - System.out.println("Trying ANDROID_TESTSUITE client."); - innerTube = new InnerTube("ANDROID_TESTSUITE"); - vidInfo = null; - break; - } - } - } - - } - void checkAvailability() throws Exception { JSONObject playabilityStatus = getVidInfo().getJSONObject("playabilityStatus"); - checkPoToken(); String status = ""; String reason = ""; @@ -373,6 +365,11 @@ private void applySignature(JSONArray streamManifest) throws Exception { String newUrl = oldUrl.replaceFirst("&n=(.*?)&", "&n=" + discoveredNSig.get(nSig) + "&"); streamManifest.getJSONObject(i).put("url", newUrl); } + if(usePoToken){ + oldUrl = streamManifest.getJSONObject(i).getString("url"); + String newUrl = oldUrl + "&pot=" + innerTube.getPoToken(); + streamManifest.getJSONObject(i).put("url", newUrl); + } } }