Merge pull request #1093 from AudricV/yt_support-shorts-ui-playlists

[YouTube] Support Shorts UI in playlists

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -41,6 +41,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     // Names of some objects in JSON response frequently used in this class
     private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
     private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
+    private static final String RICH_GRID_RENDERER = "richGridRenderer";
+    private static final String RICH_ITEM_RENDERER = "richItemRenderer";
+    private static final String REEL_ITEM_RENDERER = "reelItemRenderer";
     private static final String SIDEBAR = "sidebar";
     private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
 
@@ -85,10 +88,6 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
      * browse response (the old returns instead a sidebar one).
      * </p>
      *
-     * <p>
-     * This new playlist UI is currently A/B tested.
-     * </p>
-     *
      * @return Whether the playlist response is using only the new playlist design
      */
     private boolean checkIfResponseIsNewPlaylistInterface() {
@@ -327,17 +326,22 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
                 .map(content -> content.getObject("itemSectionRenderer")
                         .getArray("contents")
                         .getObject(0))
-                .filter(contentItemSectionRendererContents ->
-                        contentItemSectionRendererContents.has(PLAYLIST_VIDEO_LIST_RENDERER)
-                                || contentItemSectionRendererContents.has(
-                                "playlistSegmentRenderer"))
+                .filter(content -> content.has(PLAYLIST_VIDEO_LIST_RENDERER)
+                        || content.has(RICH_GRID_RENDERER))
                 .findFirst()
                 .orElse(null);
 
-        if (videoPlaylistObject != null && videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
-            final JsonArray videosArray = videoPlaylistObject
-                    .getObject(PLAYLIST_VIDEO_LIST_RENDERER)
-                    .getArray("contents");
+        if (videoPlaylistObject != null) {
+            final JsonObject renderer;
+            if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
+                renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
+            } else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
+                renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
+            } else {
+                return new InfoItemsPage<>(collector, null);
+            }
+
+            final JsonArray videosArray = renderer.getArray("contents");
             collectStreamsFrom(collector, videosArray);
 
             nextPage = getNextPageFrom(videosArray);
@@ -399,14 +403,26 @@ private Page getNextPageFrom(final JsonArray contents)
     private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
                                     @Nonnull final JsonArray videos) {
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
-
         videos.stream()
                 .filter(JsonObject.class::isInstance)
                 .map(JsonObject.class::cast)
-                .filter(video -> video.has(PLAYLIST_VIDEO_RENDERER))
-                .map(video -> new YoutubeStreamInfoItemExtractor(
-                        video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser))
-                .forEachOrdered(collector::commit);
+                .forEach(video -> {
+                    if (video.has(PLAYLIST_VIDEO_RENDERER)) {
+                        collector.commit(new YoutubeStreamInfoItemExtractor(
+                                video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser));
+                    } else if (video.has(RICH_ITEM_RENDERER)) {
+                        final JsonObject richItemRenderer = video.getObject(RICH_ITEM_RENDERER);
+                        if (richItemRenderer.has("content")) {
+                            final JsonObject richItemRendererContent =
+                                    richItemRenderer.getObject("content");
+                            if (richItemRendererContent.has(REEL_ITEM_RENDERER)) {
+                                collector.commit(new YoutubeReelInfoItemExtractor(
+                                        richItemRendererContent.getObject(REEL_ITEM_RENDERER),
+                                        timeAgoParser));
+                            }
+                        }
+                    }
+                });
     }
 
     @Nonnull

extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java

@@ -56,7 +56,7 @@ public static void assertNotEmpty(String stringToCheck) {
     }
 
     public static void assertNotEmpty(@Nullable String message, String stringToCheck) {
-        assertNotNull(message, stringToCheck);
+        assertNotNull(stringToCheck, message);
         assertFalse(stringToCheck.isEmpty(), message);
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java

@@ -38,9 +38,6 @@ public static void defaultTestListOfItems(StreamingService expectedService, List
 
             if (item instanceof StreamInfoItem) {
                 StreamInfoItem streamInfoItem = (StreamInfoItem) item;
-                assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
-
-                // assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
                 final String uploaderUrl = streamInfoItem.getUploaderUrl();
                 if (!isNullOrEmpty(uploaderUrl)) {
                     assertIsSecureUrl(uploaderUrl);

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -28,6 +28,7 @@
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 
@@ -416,6 +417,120 @@ public void testDescription() throws ParsingException {
         }
     }
 
+    static class ShortsUI implements BasePlaylistExtractorTest {
+
+        private static PlaylistExtractor extractor;
+
+        @BeforeAll
+        static void setUp() throws Exception {
+            YoutubeTestsUtils.ensureStateless();
+            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "shortsUI"));
+            extractor = YouTube.getPlaylistExtractor(
+                    "https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ");
+            extractor.fetchPage();
+        }
+
+        @Test
+        @Override
+        public void testServiceId() throws Exception {
+            assertEquals(YouTube.getServiceId(), extractor.getServiceId());
+        }
+
+        @Test
+        @Override
+        public void testName() throws Exception {
+            assertEquals("Short videos", extractor.getName());
+        }
+
+        @Test
+        @Override
+        public void testId() throws Exception {
+            assertEquals("UUSHBR8-60-B28hp2BmDPdntcQ", extractor.getId());
+        }
+
+        @Test
+        @Override
+        public void testUrl() throws Exception {
+            assertEquals("https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ",
+                    extractor.getUrl());
+        }
+
+        @Test
+        @Override
+        public void testOriginalUrl() throws Exception {
+            assertEquals("https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ",
+                    extractor.getOriginalUrl());
+        }
+
+        @Test
+        @Override
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor);
+        }
+
+        // TODO: enable test when continuations are available
+        @Disabled("Shorts UI doesn't return any continuation, even if when there are more than 100 "
+                + "items: this is a bug on YouTube's side, which is not related to the requirement "
+                + "of a valid visitorData like it is for Shorts channel tab")
+        @Test
+        @Override
+        public void testMoreRelatedItems() throws Exception {
+            defaultTestMoreItems(extractor);
+        }
+
+        @Test
+        @Override
+        public void testThumbnailUrl() throws Exception {
+            final String thumbnailUrl = extractor.getThumbnailUrl();
+            assertIsSecureUrl(thumbnailUrl);
+            ExtractorAsserts.assertContains("yt", thumbnailUrl);
+        }
+
+        @Test
+        @Override
+        public void testBannerUrl() throws Exception {
+            final String thumbnailUrl = extractor.getThumbnailUrl();
+            assertIsSecureUrl(thumbnailUrl);
+            ExtractorAsserts.assertContains("yt", thumbnailUrl);
+        }
+
+        @Test
+        @Override
+        public void testUploaderName() throws Exception {
+            assertEquals("YouTube", extractor.getUploaderName());
+        }
+
+        @Test
+        @Override
+        public void testUploaderAvatarUrl() throws Exception {
+            final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
+            ExtractorAsserts.assertContains("yt", uploaderAvatarUrl);
+        }
+
+        @Test
+        @Override
+        public void testStreamCount() throws Exception {
+            ExtractorAsserts.assertGreater(250, extractor.getStreamCount());
+        }
+
+        @Test
+        @Override
+        public void testUploaderVerified() throws Exception {
+            // YouTube doesn't provide this information for playlists
+            assertFalse(extractor.isUploaderVerified());
+        }
+
+        @Test
+        void getPlaylistType() throws ParsingException {
+            assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
+        }
+
+        @Test
+        void testDescription() throws ParsingException {
+            assertTrue(Utils.isNullOrEmpty(extractor.getDescription().getContent()));
+        }
+    }
+
     public static class ContinuationsTests {
 
         @BeforeAll

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/shortsUI/generated_mock_0.json

@@ -0,0 +1,85 @@
+{
+  "request": {
+    "httpMethod": "GET",
+    "url": "https://www.youtube.com/sw.js",
+    "headers": {
+      "Referer": [
+        "https://www.youtube.com"
+      ],
+      "Origin": [
+        "https://www.youtube.com"
+      ],
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
+      ]
+    },
+    "localization": {
+      "languageCode": "en",
+      "countryCode": "GB"
+    }
+  },
+  "response": {
+    "responseCode": 200,
+    "responseMessage": "",
+    "responseHeaders": {
+      "access-control-allow-credentials": [
+        "true"
+      ],
+      "access-control-allow-origin": [
+        "https://www.youtube.com"
+      ],
+      "alt-svc": [
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
+      ],
+      "cache-control": [
+        "private, max-age\u003d0"
+      ],
+      "content-type": [
+        "text/javascript; charset\u003dutf-8"
+      ],
+      "cross-origin-opener-policy-report-only": [
+        "same-origin; report-to\u003d\"youtube_main\""
+      ],
+      "date": [
+        "Mon, 07 Aug 2023 17:06:40 GMT"
+      ],
+      "expires": [
+        "Mon, 07 Aug 2023 17:06:40 GMT"
+      ],
+      "origin-trial": [
+        "AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
+      ],
+      "p3p": [
+        "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
+      ],
+      "permissions-policy": [
+        "ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
+      ],
+      "report-to": [
+        "{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
+      ],
+      "server": [
+        "ESF"
+      ],
+      "set-cookie": [
+        "YSC\u003d9y_BcrNyhW0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dTue, 10-Nov-2020 17:06:40 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+034; expires\u003dWed, 06-Aug-2025 17:06:40 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+      ],
+      "strict-transport-security": [
+        "max-age\u003d31536000"
+      ],
+      "x-content-type-options": [
+        "nosniff"
+      ],
+      "x-frame-options": [
+        "SAMEORIGIN"
+      ],
+      "x-xss-protection": [
+        "0"
+      ]
+    },
+    "responseBody": "\n  self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n    event.waitUntil(self.skipWaiting());\n  });\n  self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n    event.waitUntil(\n      self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n  });\n  ",
+    "latestUrl": "https://www.youtube.com/sw.js"
+  }
+}