Merge pull request #1192 from TeamNewPipe/user-agent

[tests] Update user agent

extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java

@@ -18,8 +18,11 @@
 import okhttp3.ResponseBody;
 
 public final class DownloaderTestImpl extends Downloader {
+    /**
+     * Should be the latest Firefox ESR version.
+     */
     private static final String USER_AGENT
-            = "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0";
+            = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0";
     private static DownloaderTestImpl instance;
     private final OkHttpClient client;
 

extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java

@@ -15,6 +15,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Random;
 
 import javax.annotation.Nonnull;
 
@@ -37,17 +38,31 @@
  */
 class RecordingDownloader extends Downloader {
 
-    public final static String FILE_NAME_PREFIX = "generated_mock_";
+    public static final String FILE_NAME_PREFIX = "generated_mock_";
 
     // From https://stackoverflow.com/a/15875500/13516981
-    private final static String IP_V4_PATTERN =
+    private static final String IP_V4_PATTERN =
             "(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)";
 
     private int index = 0;
     private final String path;
 
+    // try to prevent ReCaptchaExceptions / rate limits by tracking and throttling the requests
     /**
-     * Creates the folder described by {@code stringPath} if it does not exists.
+     * The maximum number of requests per 20 seconds which are executed
+     * by the {@link RecordingDownloader}.
+     * 20 seconds is used as upper bound because the rate limit can be triggered within 30 seconds
+     * and hitting the rate limit should be prevented because it comes with a bigger delay.
+     * The values can be adjusted when executing the downloader and running into problems.
+     * <p>TODO: Allow adjusting the value by setting a param in the gradle command</p>
+     */
+    private static final int MAX_REQUESTS_PER_20_SECONDS = 30;
+    private static final long[] requestTimes = new long[MAX_REQUESTS_PER_20_SECONDS];
+    private static int requestTimesCursor = -1;
+    private static final Random throttleRandom = new Random();
+
+    /**
+     * Creates the folder described by {@code stringPath} if it does not exist.
      * Deletes existing files starting with {@link RecordingDownloader#FILE_NAME_PREFIX}.
      * @param stringPath Path to the folder where the json files will be saved to.
      */
@@ -69,6 +84,48 @@ public RecordingDownloader(final String stringPath) throws IOException {
     @Override
     public Response execute(@Nonnull final Request request) throws IOException,
             ReCaptchaException {
+
+        // Delay the execution if the max number of requests per minute is reached
+        final long currentTime = System.currentTimeMillis();
+        // the cursor points to the latest request time and the next position is the oldest one
+        final int oldestRequestTimeCursor = (requestTimesCursor + 1) % requestTimes.length;
+        final long oldestRequestTime = requestTimes[oldestRequestTimeCursor];
+        if (oldestRequestTime + 20_000 >= currentTime) {
+            try {
+                // sleep at least until the oldest request is 20s old, but not more than 20s
+                final int minSleepTime = (int) (currentTime - oldestRequestTime);
+                Thread.sleep(minSleepTime + throttleRandom.nextInt(20_000 - minSleepTime));
+            } catch (InterruptedException e) {
+                // handle the exception gracefully because it's not critical for the test
+                System.err.println("Error while throttling the RecordingDownloader.");
+                e.printStackTrace();
+            }
+        }
+        requestTimesCursor = oldestRequestTimeCursor; // the oldest value needs to be overridden
+        requestTimes[requestTimesCursor] = System.currentTimeMillis();
+
+        // Handle ReCaptchaExceptions by retrying the request once after a while
+        try {
+            return executeRequest(request);
+        } catch (ReCaptchaException e) {
+            try {
+                // sleep for 35-60 seconds to circumvent the rate limit
+                System.out.println("Throttling the RecordingDownloader to handle a ReCaptcha."
+                        + " Sleeping for 35-60 seconds.");
+                Thread.sleep(35_000 + throttleRandom.nextInt(25_000));
+            } catch (InterruptedException ie) {
+                // handle the exception gracefully because it's not critical for the test
+                System.err.println("Error while throttling the RecordingDownloader.");
+                ie.printStackTrace();
+                e.printStackTrace();
+            }
+            return executeRequest(request);
+        }
+    }
+
+    @Nonnull
+    private Response executeRequest(@Nonnull final Request request) throws IOException,
+            ReCaptchaException {
         final Downloader downloader = DownloaderTestImpl.getInstance();
         Response response = downloader.execute(request);
         String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -232,8 +232,7 @@ public void testVerified() throws Exception {
         @Test
         @Override
         public void testTabs() throws Exception {
-            assertTabsContain(extractor.getTabs(), ChannelTabs.VIDEOS,
-                    ChannelTabs.LIVESTREAMS, ChannelTabs.PLAYLISTS);
+            assertTabsContain(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS);
             assertTrue(extractor.getTabs().stream()
                     .filter(it -> ChannelTabs.VIDEOS.equals(it.getContentFilters().get(0)))
                     .allMatch(ReadyChannelTabListLinkHandler.class::isInstance));

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -40,7 +40,7 @@ public void testUploaderName() throws Exception {
 
     @Override public StreamExtractor extractor() { return extractor; }
     @Override public StreamingService expectedService() { return YouTube; }
-    @Override public String expectedName() { return "lofi hip hop radio \uD83D\uDCDA - beats to relax/study to"; }
+    @Override public String expectedName() { return "lofi hip hop radio \uD83D\uDCDA beats to relax/study to"; }
     @Override public String expectedId() { return ID; }
     @Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; }
     @Override public String expectedOriginalUrlContains() { return URL; }

extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json

@@ -3,10 +3,10 @@
     "httpMethod": "GET",
     "url": "https://www.youtube.com/sw.js",
     "headers": {
-      "Referer": [
+      "Origin": [
         "https://www.youtube.com"
       ],
-      "Origin": [
+      "Referer": [
         "https://www.youtube.com"
       ],
       "Accept-Language": [
@@ -34,17 +34,20 @@
       "cache-control": [
         "private, max-age\u003d0"
       ],
+      "content-security-policy": [
+        "require-trusted-types-for \u0027script\u0027"
+      ],
       "content-type": [
         "text/javascript; charset\u003dutf-8"
       ],
       "cross-origin-opener-policy": [
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Thu, 18 Jul 2024 18:09:57 GMT"
+        "Sun, 10 Nov 2024 17:54:30 GMT"
       ],
       "expires": [
-        "Thu, 18 Jul 2024 18:09:57 GMT"
+        "Sun, 10 Nov 2024 17:54:30 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -62,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dAc2NF4wLV18; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dFri, 22-Oct-2021 18:09:57 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "YSC\u003da4d0v1pvpMk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:54:30 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_0.json

@@ -34,17 +34,20 @@
       "cache-control": [
         "private, max-age\u003d0"
       ],
+      "content-security-policy": [
+        "require-trusted-types-for \u0027script\u0027"
+      ],
       "content-type": [
         "text/javascript; charset\u003dutf-8"
       ],
       "cross-origin-opener-policy": [
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Thu, 18 Jul 2024 17:47:54 GMT"
+        "Sun, 10 Nov 2024 17:47:59 GMT"
       ],
       "expires": [
-        "Thu, 18 Jul 2024 17:47:54 GMT"
+        "Sun, 10 Nov 2024 17:47:59 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -62,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dQ-hpT9jfKtU; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dFri, 22-Oct-2021 17:47:54 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "YSC\u003dj9_R69devYo; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:47:59 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_2.json

@@ -13,7 +13,7 @@
         "SOCS\u003dCAE\u003d"
       ],
       "X-YouTube-Client-Version": [
-        "2.20240718.01.00"
+        "2.20241107.11.00"
       ],
       "X-YouTube-Client-Name": [
         "1"
@@ -229,12 +229,12 @@
       48,
       50,
       52,
+      49,
+      49,
       48,
       55,
-      49,
-      56,
       46,
-      48,
+      49,
       49,
       46,
       48,
@@ -362,7 +362,7 @@
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Thu, 18 Jul 2024 17:47:56 GMT"
+        "Sun, 10 Nov 2024 17:48:19 GMT"
       ],
       "server": [
         "scaffolding on HTTPServer2"
@@ -382,7 +382,7 @@
         "0"
       ]
     },
-    "responseBody": "{\"responseContext\":{\"visitorData\":\"CgtzVEVQR1dURHphWSjMquW0BjIiCgJGUhIcEhgSFgsMDg8QERITFBUWFxgZGhscHR4fICEgGA%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20240718.01.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0xd625b21ac0ea5a53\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"23703445,23804281,23946420,23966208,23986028,23998056,24004644,24077241,24166867,24181174,24241378,24290971,24439361,24453989,24456089,24468724,24499533,24542367,24548627,24548629,24550458,24566687,24690004,24699899,39325854,39326848,39326916,51009781,51010235,51016856,51017346,51020570,51025415,51030101,51037346,51037353,51041512,51050361,51053689,51057848,51057851,51060353,51063643,51064835,51072748,51091058,51091331,51095478,51098297,51098299,51102410,51105630,51111738,51113658,51113661,51115184,51116067,51117319,51118932,51121939,51124104,51133103,51139379,51141472,51148688,51148974,51148983,51149607,51150450,51152050,51157411,51157430,51157432,51157838,51158470,51158514,51160545,51162170,51163635,51165467,51165568,51169117,51170249,51172670,51172684,51172691,51172700,51172707,51172716,51172723,51172728,51173021,51173508,51175606,51176511,51178310,51178333,51178340,51178357,51178706,51178982,51182275,51183508,51183909,51184022,51184990,51186528,51186670,51189826,51190059,51190075,51190078,51190085,51190198,51190213,51190220,51190229,51190652,51193591,51194137,51195231,51196476,51196769,51197569,51197687,51197694,51197697,51197708,51199193,51200251,51200256,51200295,51200298,51200568,51201350,51201363,51201372,51201381,51201428,51201435,51201444,51201451,51201814,51203141,51203200,51204329,51204587,51204938,51207174,51207191,51207196,51207209,51209172,51210770,51211461,51212464,51212553,51212567,51213807,51217504,51219800,51221011,51221152,51222152,51222695,51223962,51224134,51224747,51224922,51225437,51226344,51227408,51227772,51227881,51227902,51228202,51228349,51228351,51228695,51228771,51228776,51228787,51228796,51228805,51228814,51229628,51230124,51230423,51230478,51230492,51232125,51232143,51232230,51233332,51235147,51237540,51238400,51238514,51238569,51238736,51240880,51240888,51241028,51241600\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20240718\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMIusK-7pKxhwMVBDPxBR08yQnhMghleHRlcm5hbJoBAA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UC6nSFpj9HTCZ5t-N3Rm3-HA\",\"params\":\"EgC4AQCSAwDyBgQKAjIA\"}}}",
+    "responseBody": "{\"responseContext\":{\"visitorData\":\"Cgt2SjhfUUx2UnFmayjj48O5BjIKCgJERRIEEgAgRQ%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20241107.11.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0x3a41c164850da49a\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"9406121,23804281,23880829,23880835,23966208,23986021,24004644,24077241,24166867,24181174,24241378,24299873,24439361,24445497,24453989,24459436,24542367,24547317,24548629,24566687,24699899,39325854,39326986,51009781,51010235,51017346,51020570,51025415,51030101,51037344,51037351,51050361,51053689,51057844,51057851,51063643,51064835,51072748,51091058,51095478,51098299,51101169,51111738,51115184,51117319,51124104,51129210,51133103,51134507,51141472,51144925,51151423,51152050,51156055,51157411,51157841,51158514,51160545,51165467,51169118,51176511,51178310,51178331,51178344,51178355,51178982,51182851,51183909,51184990,51194137,51195231,51204329,51213773,51217504,51221150,51222382,51222973,51223961,51226709,51227037,51227778,51228350,51230241,51230478,51231220,51231814,51237842,51239093,51241028,51242448,51243940,51248255,51248734,51249749,51251836,51255676,51255680,51255743,51256074,51256084,51257900,51257911,51257916,51258066,51259133,51260456,51263449,51265335,51265364,51265369,51266454,51272458,51273608,51274583,51275782,51276557,51276565,51281227,51282069,51282086,51282792,51283950,51284503,51285052,51285417,51285717,51287196,51287500,51289926,51289935,51289938,51289952,51289961,51289970,51290043,51291889,51294322,51295132,51295578,51296439,51298019,51298020,51299154,51299710,51299724,51299977,51299999,51300010,51300176,51300241,51300699,51302492,51302680,51303667,51303669,51303789,51304004,51304155,51305839,51306259,51307502,51308045,51308060,51308871,51309313,51310323,51311031,51311034,51311505,51311520,51312150,51312688,51313149,51313767,51314158,51314669,51314681,51314692,51314699,51314710,51314727,51315041,51315914,51315919,51315926,51315935,51315940,51315949,51315956,51315963,51315968,51315979,51316415,51316749,51317749,51318845,51320778,51323366,51325576,51326208,51326527,51326641,51326762,51326932,51327144,51327165,51327178,51327614,51327636,51328144,51329146,51329227,51329392,51329506,51330194,51330660,51331481,51331500,51331522,51331531,51331538,51331547,51331554,51331561,51332896,51333739,51333878,51335364,51335570,51335973,51336632,51337186,51337349,51337702,51338495,51338524,51339163,51339747,51340618,51341226,51341729,51342093,51342576,51342845,51343110,51343368,51344672,51344926,51345126,51345228\"},{\"key\":\"visitor_data\",\"value\":\"Cgt2SjhfUUx2UnFmayjj48O5BjIKCgJERRIEEgAgRQ%3D%3D\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20241107\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMIxIjKt6nSiQMVJEF6BR10IyrwMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UC6nSFpj9HTCZ5t-N3Rm3-HA\",\"params\":\"EgC4AQCSAwDyBgQKAjIA\"}}}",
     "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?prettyPrint\u003dfalse"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/ageRestricted/generated_mock_0.json

@@ -3,10 +3,10 @@
     "httpMethod": "GET",
     "url": "https://www.youtube.com/sw.js",
     "headers": {
-      "Referer": [
+      "Origin": [
         "https://www.youtube.com"
       ],
-      "Origin": [
+      "Referer": [
         "https://www.youtube.com"
       ],
       "Accept-Language": [
@@ -34,17 +34,20 @@
       "cache-control": [
         "private, max-age\u003d0"
       ],
+      "content-security-policy": [
+        "require-trusted-types-for \u0027script\u0027"
+      ],
       "content-type": [
         "text/javascript; charset\u003dutf-8"
       ],
       "cross-origin-opener-policy": [
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Thu, 18 Jul 2024 17:49:35 GMT"
+        "Sun, 10 Nov 2024 17:46:56 GMT"
       ],
       "expires": [
-        "Thu, 18 Jul 2024 17:49:35 GMT"
+        "Sun, 10 Nov 2024 17:46:56 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -62,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dZHHPT-DrXJQ; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dFri, 22-Oct-2021 17:49:35 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "YSC\u003dHEqrYtop-08; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:46:56 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"