Merge pull request #1127 from AudricV/yt_improvements-and-fixes

[YouTube] Make some improvements and fixes

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -448,9 +448,6 @@ private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws Parsin
                             case "playlists":
                                 addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
                                 break;
-                            case "channels":
-                                addNonVideosTab.accept(ChannelTabs.CHANNELS);
-                                break;
                         }
                     }
                 });

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java

@@ -80,10 +80,9 @@ private String getChannelTabsParameters() throws ParsingException {
                 return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
             case ChannelTabs.PLAYLISTS:
                 return "EglwbGF5bGlzdHPyBgQKAkIA";
-            case ChannelTabs.CHANNELS:
-                return "EghjaGFubmVsc_IGBAoCUgA%3D";
+            default:
+                throw new ParsingException("Unsupported channel tab: " + name);
         }
-        throw new ParsingException("Unsupported channel tab: " + name);
     }
 
     @Override
@@ -313,9 +312,6 @@ private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector
         } else if (item.has("gridPlaylistRenderer")) {
             getCommitPlaylistConsumer(collector, channelIds,
                     item.getObject("gridPlaylistRenderer"));
-        } else if (item.has("gridChannelRenderer")) {
-            collector.commit(new YoutubeChannelInfoItemExtractor(
-                    item.getObject("gridChannelRenderer")));
         } else if (item.has("shelfRenderer")) {
             return collectItem(collector, item.getObject("shelfRenderer")
                     .getObject("content"), channelIds);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -54,7 +54,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
         final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
 
-        final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
+        final String url = "https://music.youtube.com/youtubei/v1/search?key="
                 + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
 
         final String params;
@@ -89,20 +89,18 @@ public void onFetchPage(@Nonnull final Downloader downloader)
                         .value("clientVersion", youtubeMusicKeys[2])
                         .value("hl", "en-GB")
                         .value("gl", getExtractorContentCountry().getCountryCode())
-                        .array("experimentIds").end()
-                        .value("experimentsToken", "")
-                        .object("locationInfo").end()
-                        .object("musicAppInfo").end()
+                        .value("platform", "DESKTOP")
+                        .value("utcOffsetMinutes", 0)
                     .end()
-                    .object("capabilities").end()
                     .object("request")
-                        .array("internalExperimentFlags").end()
-                        .object("sessionIndex").end()
+                        .array("internalExperimentFlags")
+                        .end()
+                        .value("useSsl", true)
                     .end()
-                    .object("activePlayers").end()
                     .object("user")
-                        // TO DO: provide a way to enable restricted mode with:
-                        .value("enableSafetyMode", false)
+                        // TODO: provide a way to enable restricted mode with:
+                        //  .value("enableSafetyMode", boolean)
+                        .value("lockedSafetyMode", false)
                     .end()
                 .end()
                 .value("query", getSearchString())
@@ -219,20 +217,18 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
                         .value("clientVersion", youtubeMusicKeys[2])
                         .value("hl", "en-GB")
                         .value("gl", getExtractorContentCountry().getCountryCode())
-                        .array("experimentIds").end()
-                        .value("experimentsToken", "")
+                        .value("platform", "DESKTOP")
                         .value("utcOffsetMinutes", 0)
-                        .object("locationInfo").end()
-                        .object("musicAppInfo").end()
                     .end()
-                    .object("capabilities").end()
                     .object("request")
-                        .array("internalExperimentFlags").end()
-                        .object("sessionIndex").end()
+                        .array("internalExperimentFlags")
+                        .end()
+                        .value("useSsl", true)
                     .end()
-                    .object("activePlayers").end()
                     .object("user")
-                        .value("enableSafetyMode", false)
+                        // TODO: provide a way to enable restricted mode with:
+                        //  .value("enableSafetyMode", boolean)
+                        .value("lockedSafetyMode", false)
                     .end()
                 .end()
             .end().done().getBytes(StandardCharsets.UTF_8);
@@ -310,7 +306,7 @@ private Page getNextPageFrom(final JsonArray continuations)
         final String continuation = nextContinuationData.getString("continuation");
 
         return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
-                + "&continuation=" + continuation + "&alt=json" + "&key="
-                + YoutubeParsingHelper.getYoutubeMusicKey()[0]);
+                + "&continuation=" + continuation + "&key="
+                + YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -6,6 +6,10 @@
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -32,6 +36,7 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
+import java.util.Objects;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -57,28 +62,37 @@
  */
 
 public class YoutubeSearchExtractor extends SearchExtractor {
+
+    @Nullable
+    private final String searchType;
+    private final boolean extractVideoResults;
+    private final boolean extractChannelResults;
+    private final boolean extractPlaylistResults;
+
     private JsonObject initialData;
 
     public YoutubeSearchExtractor(final StreamingService service,
                                   final SearchQueryHandler linkHandler) {
         super(service, linkHandler);
+        final List<String> contentFilters = linkHandler.getContentFilters();
+        searchType = isNullOrEmpty(contentFilters) ? null : contentFilters.get(0);
+        // Save whether we should extract video, channel and playlist results depending on the
+        // requested search type, as YouTube returns sometimes videos inside channel search results
+        // If no search type is provided or ALL filter is requested, extract everything
+        extractVideoResults = searchType == null || ALL.equals(searchType)
+                || VIDEOS.equals(searchType);
+        extractChannelResults = searchType == null || ALL.equals(searchType)
+                || CHANNELS.equals(searchType);
+        extractPlaylistResults = searchType == null || ALL.equals(searchType)
+                || PLAYLISTS.equals(searchType);
     }
 
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
             ExtractionException {
         final String query = super.getSearchString();
         final Localization localization = getExtractorLocalization();
-
-        // Get the search parameter of the request
-        final List<String> contentFilters = super.getLinkHandler().getContentFilters();
-        final String params;
-        if (!isNullOrEmpty(contentFilters)) {
-            final String searchType = contentFilters.get(0);
-            params = getSearchParameter(searchType);
-        } else {
-            params = "";
-        }
+        final String params = getSearchParameter(searchType);
 
         final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
                 getExtractorContentCountry())
@@ -111,18 +125,17 @@ public String getSearchSuggestion() throws ParsingException {
         final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
                 .getObject(0)
                 .getObject("didYouMeanRenderer");
-        final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
-                .getObject(0)
-                .getObject("showingResultsForRenderer");
 
         if (!didYouMeanRenderer.isEmpty()) {
             return JsonUtils.getString(didYouMeanRenderer,
                     "correctedQueryEndpoint.searchEndpoint.query");
-        } else if (showingResultsForRenderer != null) {
-            return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
-        } else {
-            return "";
         }
+
+        return Objects.requireNonNullElse(
+                getTextFromObject(itemSectionRenderer.getArray("contents")
+                        .getObject(0)
+                        .getObject("showingResultsForRenderer")
+                        .getObject("correctedQuery")), "");
     }
 
     @Override
@@ -211,7 +224,7 @@ public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
 
     private void collectStreamsFrom(final MultiInfoItemsCollector collector,
                                     @Nonnull final JsonArray contents)
-            throws NothingFoundException, ParsingException {
+            throws NothingFoundException {
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
         for (final Object content : contents) {
@@ -220,13 +233,13 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector,
                 throw new NothingFoundException(
                         getTextFromObject(item.getObject("backgroundPromoRenderer")
                                 .getObject("bodyText")));
-            } else if (item.has("videoRenderer")) {
+            } else if (extractVideoResults && item.has("videoRenderer")) {
                 collector.commit(new YoutubeStreamInfoItemExtractor(
                         item.getObject("videoRenderer"), timeAgoParser));
-            } else if (item.has("channelRenderer")) {
+            } else if (extractChannelResults && item.has("channelRenderer")) {
                 collector.commit(new YoutubeChannelInfoItemExtractor(
                         item.getObject("channelRenderer")));
-            } else if (item.has("playlistRenderer")) {
+            } else if (extractPlaylistResults && item.has("playlistRenderer")) {
                 collector.commit(new YoutubePlaylistInfoItemExtractor(
                         item.getObject("playlistRenderer")));
             }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -388,57 +388,43 @@ public long getLikeCount() throws ParsingException {
 
         // If ratings are not allowed, there is no like count available
         if (!playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
-            return -1;
+            return -1L;
         }
 
-        String likesString = "";
+        final JsonArray topLevelButtons = getVideoPrimaryInfoRenderer()
+                .getObject("videoActions")
+                .getObject("menuRenderer")
+                .getArray("topLevelButtons");
 
         try {
-            final JsonArray topLevelButtons = getVideoPrimaryInfoRenderer()
-                    .getObject("videoActions")
-                    .getObject("menuRenderer")
-                    .getArray("topLevelButtons");
+            return parseLikeCountFromLikeButtonViewModel(topLevelButtons);
+        } catch (final ParsingException ignored) {
+            // A segmentedLikeDislikeButtonRenderer could be returned instead of a
+            // segmentedLikeDislikeButtonViewModel, so ignore extraction errors relative to
+            // segmentedLikeDislikeButtonViewModel object
+        }
 
-            // Try first with the new video actions buttons data structure
-            JsonObject likeToggleButtonRenderer = topLevelButtons.stream()
-                    .filter(JsonObject.class::isInstance)
-                    .map(JsonObject.class::cast)
-                    .map(button -> button.getObject("segmentedLikeDislikeButtonRenderer")
-                            .getObject("likeButton")
-                            .getObject("toggleButtonRenderer"))
-                    .filter(toggleButtonRenderer -> !isNullOrEmpty(toggleButtonRenderer))
-                    .findFirst()
-                    .orElse(null);
-
-            // Use the old video actions buttons data structure if the new one isn't returned
-            if (likeToggleButtonRenderer == null) {
-                /*
-                In the old video actions buttons data structure, there are 3 ways to detect whether
-                a button is the like button, using its toggleButtonRenderer:
-                - checking whether toggleButtonRenderer.targetId is equal to watch-like;
-                - checking whether toggleButtonRenderer.defaultIcon.iconType is equal to LIKE;
-                - checking whether
-                  toggleButtonRenderer.toggleButtonSupportedData.toggleButtonIdData.id
-                  is equal to TOGGLE_BUTTON_ID_TYPE_LIKE.
-                */
-                likeToggleButtonRenderer = topLevelButtons.stream()
-                        .filter(JsonObject.class::isInstance)
-                        .map(JsonObject.class::cast)
-                        .map(topLevelButton -> topLevelButton.getObject("toggleButtonRenderer"))
-                        .filter(toggleButtonRenderer -> toggleButtonRenderer.getString("targetId")
-                                .equalsIgnoreCase("watch-like")
-                                || toggleButtonRenderer.getObject("defaultIcon")
-                                .getString("iconType")
-                                .equalsIgnoreCase("LIKE")
-                                || toggleButtonRenderer.getObject("toggleButtonSupportedData")
-                                .getObject("toggleButtonIdData")
-                                .getString("id")
-                                .equalsIgnoreCase("TOGGLE_BUTTON_ID_TYPE_LIKE"))
-                        .findFirst()
-                        .orElseThrow(() -> new ParsingException(
-                                "The like button is missing even though ratings are enabled"));
-            }
+        try {
+            return parseLikeCountFromLikeButtonRenderer(topLevelButtons);
+        } catch (final ParsingException e) {
+            throw new ParsingException("Could not get like count", e);
+        }
+    }
 
+    private static long parseLikeCountFromLikeButtonRenderer(
+            @Nonnull final JsonArray topLevelButtons) throws ParsingException {
+        String likesString = null;
+        final JsonObject likeToggleButtonRenderer = topLevelButtons.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .map(button -> button.getObject("segmentedLikeDislikeButtonRenderer")
+                        .getObject("likeButton")
+                        .getObject("toggleButtonRenderer"))
+                .filter(toggleButtonRenderer -> !isNullOrEmpty(toggleButtonRenderer))
+                .findFirst()
+                .orElse(null);
+
+        if (likeToggleButtonRenderer != null) {
             // Use one of the accessibility strings available (this one has the same path as the
             // one used for comments' like count extraction)
             likesString = likeToggleButtonRenderer.getObject("accessibilityData")
@@ -460,23 +446,58 @@ public long getLikeCount() throws ParsingException {
                         .getString("label");
             }
 
-            // If ratings are allowed and the likes string is null, it means that we couldn't
-            // extract the (real) like count from accessibility data
-            if (likesString == null) {
-                throw new ParsingException("Could not get like count from accessibility data");
-            }
-
             // This check only works with English localizations!
-            if (likesString.toLowerCase().contains("no likes")) {
+            if (likesString != null && likesString.toLowerCase().contains("no likes")) {
                 return 0;
             }
+        }
 
-            return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
-        } catch (final NumberFormatException nfe) {
-            throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer",
-                    nfe);
-        } catch (final Exception e) {
-            throw new ParsingException("Could not get like count", e);
+        // If ratings are allowed and the likes string is null, it means that we couldn't extract
+        // the full like count from accessibility data
+        if (likesString == null) {
+            throw new ParsingException("Could not get like count from accessibility data");
+        }
+
+        try {
+            return Long.parseLong(Utils.removeNonDigitCharacters(likesString));
+        } catch (final NumberFormatException e) {
+            throw new ParsingException("Could not parse \"" + likesString + "\" as a long", e);
+        }
+    }
+
+    private static long parseLikeCountFromLikeButtonViewModel(
+            @Nonnull final JsonArray topLevelButtons) throws ParsingException {
+        // Try first with the current video actions buttons data structure
+        final JsonObject likeToggleButtonViewModel = topLevelButtons.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .map(button -> button.getObject("segmentedLikeDislikeButtonViewModel")
+                        .getObject("likeButtonViewModel")
+                        .getObject("likeButtonViewModel")
+                        .getObject("toggleButtonViewModel")
+                        .getObject("toggleButtonViewModel")
+                        .getObject("defaultButtonViewModel")
+                        .getObject("buttonViewModel"))
+                .filter(buttonViewModel -> !isNullOrEmpty(buttonViewModel))
+                .findFirst()
+                .orElse(null);
+
+        if (likeToggleButtonViewModel == null) {
+            throw new ParsingException("Could not find buttonViewModel object");
+        }
+
+        final String accessibilityText = likeToggleButtonViewModel.getString("accessibilityText");
+        if (accessibilityText == null) {
+            throw new ParsingException("Could not find buttonViewModel's accessibilityText string");
+        }
+
+        // The like count is always returned as a number in this element, even for videos with no
+        // likes
+        try {
+            return Long.parseLong(Utils.removeNonDigitCharacters(accessibilityText));
+        } catch (final NumberFormatException e) {
+            throw new ParsingException(
+                    "Could not parse \"" + accessibilityText + "\" as a long", e);
         }
     }