Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[YouTube] Add support for new playlist items data structure #1240

Merged
merged 2 commits into from
Nov 14, 2024
Merged

[YouTube] Add support for new playlist items data structure #1240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f52d226
[YouTube] Move channel verified status check from badges to a method
AudricV Nov 10, 2024
183563c
[YouTube] Add support for playlists lockupViewModels
AudricV Nov 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 6 additions & 21 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;

Expand Down Expand Up @@ -363,27 +364,11 @@ public static boolean isChannelVerified(@Nonnull final ChannelHeader channelHead
final JsonObject pageHeaderViewModel = channelHeader.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL);

final boolean hasCircleOrMusicIcon = pageHeaderViewModel.getObject(TITLE)
.getObject("dynamicTextViewModel")
.getObject("text")
.getArray("attachmentRuns")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(attachmentRun -> attachmentRun.getObject("element")
.getObject("type")
.getObject("imageType")
.getObject("image")
.getArray("sources")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(source -> {
final String imageName = source.getObject("clientResource")
.getString("imageName");
return "CHECK_CIRCLE_FILLED".equals(imageName)
|| "MUSIC_FILLED".equals(imageName);
}));
final boolean hasCircleOrMusicIcon = hasArtistOrVerifiedIconBadgeAttachment(
pageHeaderViewModel.getObject(TITLE)
.getObject("dynamicTextViewModel")
.getObject("text")
.getArray("attachmentRuns"));
if (!hasCircleOrMusicIcon && pageHeaderViewModel.getObject("image")
.has("contentPreviewImageViewModel")) {
// If a pageHeaderRenderer has no object in which a check verified may be
Expand Down
23 changes: 23 additions & 0 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1584,6 +1584,29 @@ public static boolean isVerified(final JsonArray badges) {
return false;
}

public static boolean hasArtistOrVerifiedIconBadgeAttachment(
@Nonnull final JsonArray attachmentRuns) {
return attachmentRuns.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(attachmentRun -> attachmentRun.getObject("element")
.getObject("type")
.getObject("imageType")
.getObject("image")
.getArray("sources")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(source -> {
final String imageName = source.getObject("clientResource")
.getString("imageName");
return "CHECK_CIRCLE_FILLED".equals(imageName)
|| "AUDIO_BADGE".equals(imageName)
|| "MUSIC_FILLED".equals(imageName);
}));

}

/**
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
* playback requests (and also for some clients, in the player request body).
Expand Down
37 changes: 37 additions & 0 deletions .../org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,12 @@ private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector
} else if (item.has("expandedShelfContentsRenderer")) {
return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
.getArray("items"), channelVerifiedStatus, channelName, channelUrl);
} else if (item.has("lockupViewModel")) {
final JsonObject lockupViewModel = item.getObject("lockupViewModel");
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(lockupViewModel.getString("contentType"))) {
commitPlaylistLockup(collector, lockupViewModel, channelVerifiedStatus,
channelName, channelUrl);
}
} else if (item.has("continuationItemRenderer")) {
return Optional.ofNullable(item.getObject("continuationItemRenderer"));
}
Expand Down Expand Up @@ -366,6 +372,37 @@ public boolean isUploaderVerified() {
});
}

private void commitPlaylistLockup(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject playlistLockupViewModel,
@Nonnull final VerifiedStatus channelVerifiedStatus,
@Nullable final String channelName,
@Nullable final String channelUrl) {
collector.commit(
new YoutubeMixOrPlaylistLockupInfoItemExtractor(playlistLockupViewModel) {
@Override
public String getUploaderName() throws ParsingException {
return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
}

@Override
public String getUploaderUrl() throws ParsingException {
return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
}

@Override
public boolean isUploaderVerified() throws ParsingException {
switch (channelVerifiedStatus) {
case VERIFIED:
return true;
case UNVERIFIED:
return false;
default:
return super.isUploaderVerified();
}
}
});
}

private void commitVideo(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final JsonObject jsonObject,
Expand Down
193 changes: 193 additions & 0 deletions ...pe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;

import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Image;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import java.util.List;

import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;

public class YoutubeMixOrPlaylistLockupInfoItemExtractor implements PlaylistInfoItemExtractor {

@Nonnull
private final JsonObject lockupViewModel;
@Nonnull
private final JsonObject thumbnailViewModel;
@Nonnull
private final JsonObject lockupMetadataViewModel;
@Nonnull
private final JsonObject firstMetadataRow;
@Nonnull
private PlaylistInfo.PlaylistType playlistType;

public YoutubeMixOrPlaylistLockupInfoItemExtractor(@Nonnull final JsonObject lockupViewModel) {
this.lockupViewModel = lockupViewModel;
this.thumbnailViewModel = lockupViewModel.getObject("contentImage")
.getObject("collectionThumbnailViewModel")
.getObject("primaryThumbnail")
.getObject("thumbnailViewModel");
this.lockupMetadataViewModel = lockupViewModel.getObject("metadata")
.getObject("lockupMetadataViewModel");
/*
The metadata rows are structured in the following way:
1st part: uploader info, playlist type, playlist updated date
2nd part: space row
3rd element: first video
4th (not always returned for playlists with less than 2 items?): second video
5th element (always returned, but at a different index for playlists with less than 2
items?): Show full playlist

The first metadata row has the following structure:
1st array element: uploader info
2nd element: playlist type (course, playlist, podcast)
3rd element (not always returned): playlist updated date
*/
this.firstMetadataRow = lockupMetadataViewModel.getObject("metadata")
.getObject("contentMetadataViewModel")
.getArray("metadataRows")
.getObject(0);

try {
this.playlistType = extractPlaylistTypeFromPlaylistId(getPlaylistId());
} catch (final ParsingException e) {
// If we cannot extract the playlist type, fall back to the normal one
this.playlistType = PlaylistInfo.PlaylistType.NORMAL;
}
}

@Override
public String getUploaderName() throws ParsingException {
return firstMetadataRow.getArray("metadataParts")
.getObject(0)
.getObject("text")
.getString("content");
}

@Override
public String getUploaderUrl() throws ParsingException {
if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
// If the playlist is a mix, there is no uploader as they are auto-generated
return null;
}

return getUrlFromNavigationEndpoint(
firstMetadataRow.getArray("metadataParts")
.getObject(0)
.getObject("text")
.getArray("commandRuns")
.getObject(0)
.getObject("onTap")
.getObject("innertubeCommand"));
}

@Override
public boolean isUploaderVerified() throws ParsingException {
if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
// If the playlist is a mix, there is no uploader as they are auto-generated
return false;
}

return hasArtistOrVerifiedIconBadgeAttachment(
firstMetadataRow.getArray("metadataParts")
.getObject(0)
.getObject("text")
.getArray("attachmentRuns"));
}

@Override
public long getStreamCount() throws ParsingException {
if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
// If the playlist is a mix, we are not able to get its stream count
return ListExtractor.ITEM_COUNT_INFINITE;
}

try {
return Long.parseLong(Utils.removeNonDigitCharacters(
thumbnailViewModel.getArray("overlays")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(overlay -> overlay.has("thumbnailOverlayBadgeViewModel"))
.findFirst()
.orElseThrow(() -> new ParsingException(
"Could not get thumbnailOverlayBadgeViewModel"))
.getObject("thumbnailOverlayBadgeViewModel")
.getArray("thumbnailBadges")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(badge -> badge.has("thumbnailBadgeViewModel"))
.findFirst()
.orElseThrow(() ->
new ParsingException("Could not get thumbnailBadgeViewModel"))
.getObject("thumbnailBadgeViewModel")
.getString("text")));
} catch (final Exception e) {
throw new ParsingException("Could not get playlist stream count", e);
}
}

@Override
public String getName() throws ParsingException {
return lockupMetadataViewModel.getObject("title")
.getString("content");
}

@Override
public String getUrl() throws ParsingException {
// If the playlist item is a mix, we cannot return just its playlist ID as mix playlists
// are not viewable in playlist pages
// Use directly getUrlFromNavigationEndpoint in this case, which returns the watch URL with
// the mix playlist
if (playlistType == PlaylistInfo.PlaylistType.NORMAL) {
try {
return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(getPlaylistId());
} catch (final Exception ignored) {
}
}

return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext")
.getObject("commandContext")
.getObject("onTap")
.getObject("innertubeCommand"));
}

@Nonnull
@Override
public List<Image> getThumbnails() throws ParsingException {
return getImagesFromThumbnailsArray(thumbnailViewModel.getObject("image")
.getArray("sources"));
}

@Nonnull
@Override
public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
return playlistType;
}

private String getPlaylistId() throws ParsingException {
String id = lockupViewModel.getString("contentId");
if (Utils.isNullOrEmpty(id)) {
id = lockupViewModel.getObject("rendererContext")
.getObject("commandContext")
.getObject("watchEndpoint")
.getString("playlistId");
}

if (Utils.isNullOrEmpty(id)) {
throw new ParsingException("Could not get playlist ID");
}

return id;
}
}
7 changes: 7 additions & 0 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,13 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector,
} else if (item.has("showRenderer")) {
collector.commit(new YoutubeShowRendererInfoItemExtractor(
item.getObject("showRenderer")));
} else if (item.has("lockupViewModel")) {
final JsonObject lockupViewModel = item.getObject("lockupViewModel");
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
lockupViewModel.getString("contentType"))) {
collector.commit(
new YoutubeMixOrPlaylistLockupInfoItemExtractor(lockupViewModel));
}
}
}
}
Expand Down
7 changes: 7 additions & 0 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,13 @@ public MultiInfoItemsCollector getRelatedItems() throws ExtractionException {
} else if (result.has("compactPlaylistRenderer")) {
return new YoutubeMixOrPlaylistInfoItemExtractor(
result.getObject("compactPlaylistRenderer"));
} else if (result.has("lockupViewModel")) {
final JsonObject lockupViewModel = result.getObject("lockupViewModel");
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
lockupViewModel.getString("contentType"))) {
return new YoutubeMixOrPlaylistLockupInfoItemExtractor(
lockupViewModel);
}
}
return null;
})
Expand Down
Loading