From 4df58f749f7a6da935c9547d664c97cc35318325 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Sat, 26 Mar 2022 18:09:36 +0100 Subject: [PATCH 01/21] Code cleanup --- .../schabi/newpipe/extractor/ServiceList.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 18c617f659..7d1ac3bdce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -37,11 +37,11 @@ private ServiceList() { //no instance } - public static final YoutubeService YouTube; - public static final SoundcloudService SoundCloud; - public static final MediaCCCService MediaCCC; - public static final PeertubeService PeerTube; - public static final BandcampService Bandcamp; + public static final YoutubeService YouTube = new YoutubeService(0); + public static final SoundcloudService SoundCloud = new SoundcloudService(1); + public static final MediaCCCService MediaCCC = new MediaCCCService(2); + public static final PeertubeService PeerTube = new PeertubeService(3); + public static final BandcampService Bandcamp = new BandcampService(4); /** * When creating a new service, put this service in the end of this list, @@ -49,11 +49,11 @@ private ServiceList() { */ private static final List SERVICES = Collections.unmodifiableList( Arrays.asList( - YouTube = new YoutubeService(0), - SoundCloud = new SoundcloudService(1), - MediaCCC = new MediaCCCService(2), - PeerTube = new PeertubeService(3), - Bandcamp = new BandcampService(4) + YouTube, + SoundCloud, + MediaCCC, + PeerTube, + Bandcamp )); /** From b48b593d02285d38c167ae1bbf3ba573eeadd930 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Fri, 15 Apr 2022 14:50:52 +0200 Subject: [PATCH 02/21] Made YT proxyable * allows usage of YouTubeLike services, e.g. invidious * simplified some classes/code * Added support for invidious * Some credits go to @B0pol's PR #555 for that ;) --- .../InstanceBasedStreamingService.java | 9 + .../schabi/newpipe/extractor/ServiceList.java | 2 +- .../newpipe/extractor/StreamingService.java | 17 +- .../extractor/channel/ChannelExtractor.java | 24 +- .../channel/ChannelInfoItemExtractor.java | 4 +- .../comments/CommentsInfoItemExtractor.java | 2 +- .../extractor/instance/AbstractInstance.java | 55 +++ .../newpipe/extractor/instance/Instance.java | 46 +++ .../InstanceMetaDataFetchException.java | 15 + .../newpipe/extractor/kiosk/KioskList.java | 5 +- .../extractor/playlist/PlaylistExtractor.java | 15 +- .../extractor/search/SearchExtractor.java | 38 +- .../services/bandcamp/BandcampService.java | 49 ++- .../extractors/BandcampChannelExtractor.java | 20 -- .../extractors/BandcampPlaylistExtractor.java | 5 - .../BandcampRadioInfoItemExtractor.java | 23 +- .../extractors/BandcampSearchExtractor.java | 26 +- ...campDiscographStreamInfoItemExtractor.java | 9 +- ...ndcampPlaylistStreamInfoItemExtractor.java | 8 +- ...BandcampSearchStreamInfoItemExtractor.java | 8 - .../BandcampStreamInfoItemExtractor.java | 11 - .../services/media_ccc/MediaCCCService.java | 69 ++-- .../MediaCCCConferenceExtractor.java | 24 +- .../MediaCCCLiveStreamKioskExtractor.java | 17 +- .../MediaCCCRecentKioskExtractor.java | 16 - .../extractors/MediaCCCSearchExtractor.java | 26 +- .../MediaCCCConferenceInfoItemExtractor.java | 5 - .../MediaCCCStreamInfoItemExtractor.java | 17 +- .../services/peertube/PeertubeInstance.java | 57 ++- .../services/peertube/PeertubeService.java | 50 +-- .../extractors/PeertubeAccountExtractor.java | 34 +- .../extractors/PeertubeChannelExtractor.java | 19 +- .../extractors/PeertubePlaylistExtractor.java | 19 +- .../extractors/PeertubeSearchExtractor.java | 35 +- .../PeertubeStreamInfoItemExtractor.java | 11 +- .../soundcloud/SoundcloudService.java | 12 +- .../SoundcloudChannelExtractor.java | 25 +- .../extractors/SoundcloudSearchExtractor.java | 31 +- .../SoundcloudStreamInfoItemExtractor.java | 20 +- .../services/youtube/YoutubeLikeInstance.java | 13 + .../youtube/YoutubeLikeStreamingService.java | 47 +++ .../services/youtube/YoutubeService.java | 136 +++---- .../youtube/invidious/InvidiousInstance.java | 84 +++++ .../invidious/InvidiousParsingHelper.java | 156 ++++++++ .../youtube/invidious/InvidiousService.java | 176 +++++++++ .../extractors/InvidiousChannelExtractor.java | 114 ++++++ .../InvidiousChannelInfoItemExtractor.java | 52 +++ .../InvidiousCommentsExtractor.java | 89 +++++ .../InvidiousCommentsInfoItemExtractor.java | 84 +++++ .../extractors/InvidiousFeedExtractor.java | 82 +++++ .../InvidiousMixPlaylistExtractor.java | 94 +++++ .../InvidiousPlaylistExtractor.java | 113 ++++++ .../InvidiousPlaylistInfoItemExtractor.java | 48 +++ .../extractors/InvidiousSearchExtractor.java | 92 +++++ .../extractors/InvidiousStreamExtractor.java | 333 ++++++++++++++++++ .../InvidiousStreamInfoItemExtractor.java | 100 ++++++ .../InvidiousSuggestionExtractor.java | 37 ++ .../InvidiousTrendingExtractor.java | 77 ++++ .../InvidiousChannelLinkHandlerFactory.java | 38 ++ .../InvidiousCommentLinkHandlerFactory.java | 12 + .../InvidiousPlaylistLinkHandlerFactory.java | 26 ++ .../InvidiousSearchQueryHandlerFactory.java | 58 +++ .../InvidiousStreamLinkHandlerFactory.java | 19 + .../InvidiousTrendingLinkHandlerFactory.java | 64 ++++ .../YoutubeCommentsLinkHandlerFactory.java | 50 --- .../YoutubeStreamLinkHandlerFactory.java | 258 -------------- .../youtube/{ => shared}/ItagItem.java | 8 +- .../youtube/shared/YoutubePlaylistHelper.java | 163 +++++++++ .../youtube/shared/YoutubeUrlHelper.java | 89 +++++ .../YoutubeTakeoutSubscriptionExtractor.java} | 23 +- ...YoutubeLikeChannelLinkHandlerFactory.java} | 64 +--- ...YoutubeLikeCommentsLinkHandlerFactory.java | 41 +++ ...outubeLikePlaylistLinkHandlerFactory.java} | 60 ++-- .../YoutubeLikeStreamLinkHandlerFactory.java | 176 +++++++++ .../youtube/youtube/YoutubeDirectService.java | 215 +++++++++++ .../youtube/youtube/YoutubeInstance.java | 18 + .../YoutubeJavaScriptExtractor.java | 2 +- .../{ => youtube}/YoutubeParsingHelper.java | 269 +------------- .../YoutubeThrottlingDecrypter.java | 9 +- .../extractors/YoutubeChannelExtractor.java | 26 +- .../YoutubeChannelInfoItemExtractor.java | 12 +- .../extractors/YoutubeCommentsExtractor.java | 29 +- .../YoutubeCommentsInfoItemExtractor.java | 5 +- .../extractors/YoutubeFeedExtractor.java | 4 +- .../YoutubeFeedInfoItemExtractor.java | 16 +- ...YoutubeMixOrPlaylistInfoItemExtractor.java | 10 +- .../YoutubeMixPlaylistExtractor.java | 50 ++- .../YoutubeMusicSearchExtractor.java | 29 +- .../extractors/YoutubePlaylistExtractor.java | 28 +- .../YoutubePlaylistInfoItemExtractor.java | 10 +- .../extractors/YoutubeSearchExtractor.java | 24 +- .../extractors/YoutubeStreamExtractor.java | 49 ++- .../YoutubeStreamInfoItemExtractor.java | 12 +- .../YoutubeSuggestionExtractor.java | 33 +- .../extractors/YoutubeTrendingExtractor.java | 14 +- .../YoutubeChannelLinkHandlerFactory.java | 51 +++ .../YoutubeCommentsLinkHandlerFactory.java | 17 + .../YoutubePlaylistLinkHandlerFactory.java | 29 ++ .../YoutubeSearchQueryHandlerFactory.java | 9 +- .../YoutubeStreamLinkHandlerFactory.java | 41 +++ .../YoutubeTrendingLinkHandlerFactory.java | 7 +- .../newpipe/extractor/stream/AudioStream.java | 2 +- .../newpipe/extractor/stream/Description.java | 10 +- .../stream/StreamInfoItemExtractor.java | 12 +- .../newpipe/extractor/stream/VideoStream.java | 2 +- .../extractor/utils/DashMpdParser.java | 7 +- .../schabi/newpipe/extractor/utils/Utils.java | 10 + .../youtube/YoutubeChannelExtractorTest.java | 100 +++++- .../YoutubeChannelLinkHandlerFactoryTest.java | 13 +- .../YoutubeChannelLocalizationTest.java | 10 +- .../youtube/YoutubeCommentsExtractorTest.java | 31 +- ...YoutubeCommentsLinkHandlerFactoryTest.java | 12 +- .../youtube/YoutubeFeedExtractorTest.java | 12 +- .../YoutubeJavaScriptExtractorTest.java | 1 + .../youtube/YoutubeKioskExtractorTest.java | 10 +- .../YoutubeMixPlaylistExtractorTest.java | 7 +- .../youtube/YoutubeParsingHelperTest.java | 6 +- .../youtube/YoutubePlaylistExtractorTest.java | 39 +- ...YoutubePlaylistLinkHandlerFactoryTest.java | 9 +- .../services/youtube/YoutubeServiceTest.java | 14 +- .../YoutubeStreamLinkHandlerFactoryTest.java | 17 +- .../YoutubeSuggestionExtractorTest.java | 1 + ...tubeTakeoutSubscriptionExtractorTest.java} | 51 +-- .../services/youtube/YoutubeTestsUtils.java | 3 +- .../YoutubeThrottlingDecrypterTest.java | 11 +- .../youtube/YoutubeTrendingKioskInfoTest.java | 3 +- ...YoutubeTrendingLinkHandlerFactoryTest.java | 12 +- .../YoutubeMusicSearchExtractorTest.java | 3 +- .../search/YoutubeSearchExtractorTest.java | 9 +- .../youtube/search/YoutubeSearchQHTest.java | 8 +- ...utubeStreamExtractorControversialTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 2 +- 132 files changed, 3787 insertions(+), 1614 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/InstanceBasedStreamingService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/instance/AbstractInstance.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/instance/Instance.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/instance/InstanceMetaDataFetchException.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeInstance.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeStreamingService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousMixPlaylistExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSuggestionExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousChannelLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousCommentLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousSearchQueryHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousTrendingLinkHandlerFactory.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => shared}/ItagItem.java (96%) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubePlaylistHelper.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{extractors/YoutubeSubscriptionExtractor.java => shared/extractors/YoutubeTakeoutSubscriptionExtractor.java} (90%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{linkHandler/YoutubeChannelLinkHandlerFactory.java => shared/linkHandler/YoutubeLikeChannelLinkHandlerFactory.java} (51%) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeCommentsLinkHandlerFactory.java rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{linkHandler/YoutubePlaylistLinkHandlerFactory.java => shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java} (63%) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeDirectService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeInstance.java rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/YoutubeJavaScriptExtractor.java (98%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/YoutubeParsingHelper.java (82%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/YoutubeThrottlingDecrypter.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeChannelExtractor.java (93%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeChannelInfoItemExtractor.java (89%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeCommentsExtractor.java (96%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeCommentsInfoItemExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeFeedExtractor.java (94%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeFeedInfoItemExtractor.java (90%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java (82%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeMixPlaylistExtractor.java (87%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeMusicSearchExtractor.java (95%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubePlaylistExtractor.java (90%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubePlaylistInfoItemExtractor.java (84%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeSearchExtractor.java (90%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeStreamExtractor.java (96%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeStreamInfoItemExtractor.java (94%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeSuggestionExtractor.java (71%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/extractors/YoutubeTrendingExtractor.java (92%) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/linkHandler/YoutubeSearchQueryHandlerFactory.java (98%) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => youtube}/linkHandler/YoutubeTrendingLinkHandlerFactory.java (86%) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/{YoutubeSubscriptionExtractorTest.java => YoutubeTakeoutSubscriptionExtractorTest.java} (78%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InstanceBasedStreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InstanceBasedStreamingService.java new file mode 100644 index 0000000000..213307c65b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InstanceBasedStreamingService.java @@ -0,0 +1,9 @@ +package org.schabi.newpipe.extractor; + +import org.schabi.newpipe.extractor.instance.Instance; + +public interface InstanceBasedStreamingService { + I getInstance(); + + void setInstance(I instance); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 7d1ac3bdce..9ddaac4543 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -31,7 +31,7 @@ /** * A list of supported services. */ -@SuppressWarnings({"ConstantName", "InnerAssignment"}) // keep unusual names and inner assignments +// keep unusual names and inner assignments public final class ServiceList { private ServiceList() { //no instance diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 94b8ba2d91..6ae18da8e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -23,10 +23,11 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nullable; import java.util.Collections; import java.util.List; +import javax.annotation.Nullable; + /* * Copyright (C) Christian Schabesberger 2018 * StreamingService.java is part of NewPipe. @@ -97,13 +98,14 @@ public enum LinkType { * If you Implement one do not set id within your implementation of this extractor, instead * set the id when you put the extractor into {@link ServiceList} * All other parameters can be set directly from the overriding constructor. - * @param id the number of the service to identify him within the NewPipe frontend - * @param name the name of the service + * + * @param id the number of the service to identify him within the NewPipe frontend + * @param name the name of the service * @param capabilities the type of media this service can handle */ - public StreamingService(final int id, - final String name, - final List capabilities) { + protected StreamingService(final int id, + final String name, + final List capabilities) { this.serviceId = id; this.serviceInfo = new ServiceInfo(name, capabilities); } @@ -192,9 +194,10 @@ public FeedExtractor getFeedExtractor(final String url) throws ExtractionExcepti /** * Must create a new instance of a KioskList implementation. + * * @return a new KioskList instance */ - public abstract KioskList getKioskList() throws ExtractionException; + public abstract KioskList getKioskList(); /** * Must create a new instance of a ChannelExtractor implementation. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index a045411247..f97262f837 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -35,13 +35,29 @@ public ChannelExtractor(final StreamingService service, final ListLinkHandler li } public abstract String getAvatarUrl() throws ParsingException; + public abstract String getBannerUrl() throws ParsingException; + public abstract String getFeedUrl() throws ParsingException; + public abstract long getSubscriberCount() throws ParsingException; + public abstract String getDescription() throws ParsingException; - public abstract String getParentChannelName() throws ParsingException; - public abstract String getParentChannelUrl() throws ParsingException; - public abstract String getParentChannelAvatarUrl() throws ParsingException; - public abstract boolean isVerified() throws ParsingException; + + public String getParentChannelName() throws ParsingException { + return null; + } + + public String getParentChannelUrl() throws ParsingException { + return null; + } + + public String getParentChannelAvatarUrl() throws ParsingException { + return null; + } + + public boolean isVerified() throws ParsingException { + return false; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java index 6c22c3373e..2a54f15a7e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java @@ -30,5 +30,7 @@ public interface ChannelInfoItemExtractor extends InfoItemExtractor { long getStreamCount() throws ParsingException; - boolean isVerified() throws ParsingException; + default boolean isVerified() throws ParsingException { + return false; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java index 40279eb0d1..36cd7f402f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java @@ -4,7 +4,7 @@ import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeCommentsInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/AbstractInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/AbstractInstance.java new file mode 100644 index 0000000000..f3e3f2b709 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/AbstractInstance.java @@ -0,0 +1,55 @@ +package org.schabi.newpipe.extractor.instance; + +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.URI; +import java.util.Objects; + +import javax.annotation.Nonnull; + +public abstract class AbstractInstance implements Instance { + + protected Boolean valid = null; + + protected final String url; + protected String name; + + protected AbstractInstance(final String url, final String name) { + this.url = removeTrailingSlashes(Objects.requireNonNull(url)); + this.name = name; + } + + @Nonnull + @Override + public String getName() { + return name; + } + + @Nonnull + @Override + public String getUrl() { + return url; + } + + public void setName(final String name) { + this.name = Objects.requireNonNull(name); + } + + @Override + public void fetchMetadata() { + // Default: Do nothing + } + + public static String removeTrailingSlashes(final String input) { + return input.replaceAll("/*$", ""); + } + + public static String tryExtractDomainFromUrl(final String url, final String fallback) { + Objects.requireNonNull(url); + try { + return Utils.removeMAndWWWFromHost(new URI(url).getHost()); + } catch (final Exception ignored) { + return fallback; + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/Instance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/Instance.java new file mode 100644 index 0000000000..703d4bf793 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/Instance.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.instance; + +import javax.annotation.Nonnull; + +public interface Instance { + + /** + * The name of the instance. + *
+ * Note: May only be available after {@link #fetchMetadata()} was called + * + * @return the instance-name + */ + String getName(); + + /** + * The base url of this instance. + *
+ * Note that the url is returned without trailing slashes. + * Use {@link #getUrlWithTrailingSlash()} if you want a trailing slash. + * + * @return base url + */ + @Nonnull + String getUrl(); + + default String getUrlWithTrailingSlash() { + return getUrl() + "/"; + } + + + /** + * Fetch instance metadata. + * + * @throws InstanceMetaDataFetchException + */ + void fetchMetadata(); + + /** + * Returns the service name, e.g. "Invidious" for an invidious instance. + * @return service name or null if the name of the streamingservice should be used + */ + default String getServiceName() { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/InstanceMetaDataFetchException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/InstanceMetaDataFetchException.java new file mode 100644 index 0000000000..7afaa5b4b2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/InstanceMetaDataFetchException.java @@ -0,0 +1,15 @@ +package org.schabi.newpipe.extractor.instance; + +public class InstanceMetaDataFetchException extends RuntimeException { + public InstanceMetaDataFetchException(final String message) { + super(message); + } + + public InstanceMetaDataFetchException(final String message, final Throwable cause) { + super(message, cause); + } + + public InstanceMetaDataFetchException(final Throwable cause) { + super(cause); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index cc9df054c9..3dc3fc4f53 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -50,10 +50,9 @@ public KioskList(final StreamingService service) { public void addKioskEntry(final KioskExtractorFactory extractorFactory, final ListLinkHandlerFactory handlerFactory, - final String id) - throws Exception { + final String id) { if (kioskList.get(id) != null) { - throw new Exception("Kiosk with type " + id + " already exists."); + throw new IllegalArgumentException("Kiosk with type " + id + " already exists."); } kioskList.put(id, new KioskEntry(extractorFactory, handlerFactory)); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index b055c6e30a..2142dc669b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.playlist; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; + import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -8,8 +10,6 @@ import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; - public abstract class PlaylistExtractor extends ListExtractor { public PlaylistExtractor(final StreamingService service, final ListLinkHandler linkHandler) { @@ -17,9 +17,16 @@ public PlaylistExtractor(final StreamingService service, final ListLinkHandler l } public abstract String getUploaderUrl() throws ParsingException; + public abstract String getUploaderName() throws ParsingException; - public abstract String getUploaderAvatarUrl() throws ParsingException; - public abstract boolean isUploaderVerified() throws ParsingException; + + public String getUploaderAvatarUrl() throws ParsingException { + return EMPTY_STRING; + } + + public boolean isUploaderVerified() throws ParsingException { + return false; + } public abstract long getStreamCount() throws ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index 601359acf6..bf8cd77d91 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -7,10 +7,13 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; +import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + public abstract class SearchExtractor extends ListExtractor { public static class NothingFoundException extends ExtractionException { @@ -27,17 +30,6 @@ public String getSearchString() { return getLinkHandler().getSearchString(); } - /** - * The search suggestion provided by the service. - *

- * This method also returns the corrected query if - * {@link SearchExtractor#isCorrectedSearch()} is true. - * - * @return a suggestion to another query, the corrected query, or an empty String. - */ - @Nonnull - public abstract String getSearchSuggestion() throws ParsingException; - @Nonnull @Override public SearchQueryHandler getLinkHandler() { @@ -50,6 +42,19 @@ public String getName() { return getLinkHandler().getSearchString(); } + /** + * The search suggestion provided by the service. + *

+ * This method also returns the corrected query if + * {@link SearchExtractor#isCorrectedSearch()} is true. + * + * @return a suggestion to another query, the corrected query, or an empty String. + */ + @Nonnull + public String getSearchSuggestion() throws ParsingException { + return Utils.EMPTY_STRING; + } + /** * Tell if the search was corrected by the service (if it's not exactly the search you typed). *

@@ -58,15 +63,20 @@ public String getName() { * * @return whether the results comes from a corrected query or not. */ - public abstract boolean isCorrectedSearch() throws ParsingException; + public boolean isCorrectedSearch() throws ParsingException { + return false; + } /** * Meta information about the search query. *

* Example: on YouTube, if you search for "Covid-19", * there is a box with information from the WHO about Covid-19 and a link to the WHO's website. + * * @return additional meta information about the search query */ @Nonnull - public abstract List getMetaInfo() throws ParsingException; + public List getMetaInfo() throws ParsingException { + return Collections.emptyList(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 8b2758796e..a1e178f5b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -97,36 +97,31 @@ public SubscriptionExtractor getSubscriptionExtractor() { } @Override - public KioskList getKioskList() throws ExtractionException { + public KioskList getKioskList() { final KioskList kioskList = new KioskList(this); - try { - kioskList.addKioskEntry( - (streamingService, url, kioskId) -> new BandcampFeaturedExtractor( - BandcampService.this, - new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), - kioskId - ), - new BandcampFeaturedLinkHandlerFactory(), - KIOSK_FEATURED - ); - - kioskList.addKioskEntry( - (streamingService, url, kioskId) -> new BandcampRadioExtractor( - BandcampService.this, - new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL), - kioskId - ), - new BandcampFeaturedLinkHandlerFactory(), - KIOSK_RADIO - ); - - kioskList.setDefaultKiosk(KIOSK_FEATURED); - - } catch (final Exception e) { - throw new ExtractionException(e); - } + kioskList.addKioskEntry( + (streamingService, url, kioskId) -> new BandcampFeaturedExtractor( + BandcampService.this, + new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), + kioskId + ), + new BandcampFeaturedLinkHandlerFactory(), + KIOSK_FEATURED + ); + + kioskList.addKioskEntry( + (streamingService, url, kioskId) -> new BandcampRadioExtractor( + BandcampService.this, + new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL), + kioskId + ), + new BandcampFeaturedLinkHandlerFactory(), + KIOSK_RADIO + ); + + kioskList.setDefaultKiosk(KIOSK_FEATURED); return kioskList; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 80e7ba27ea..b620650f1c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -85,26 +85,6 @@ public String getDescription() { return channelInfo.getString("bio"); } - @Override - public String getParentChannelName() { - return null; - } - - @Override - public String getParentChannelUrl() { - return null; - } - - @Override - public String getParentChannelAvatarUrl() { - return null; - } - - @Override - public boolean isVerified() throws ParsingException { - return false; - } - @Nonnull @Override public InfoItemsPage getInitialPage() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 6bceb9ceb9..12e024eed9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -99,11 +99,6 @@ public String getUploaderAvatarUrl() { .orElse(EMPTY_STRING); } - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Override public long getStreamCount() { return trackInfo.size(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index be6ba0b7ad..cd8b7c2454 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -2,7 +2,11 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; + import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; @@ -10,9 +14,6 @@ import javax.annotation.Nullable; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; - public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { private final JsonObject show; @@ -77,20 +78,4 @@ public String getUploaderName() { public String getUploaderUrl() { return ""; } - - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - - @Override - public boolean isAd() { - return false; - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index af8fafead0..ffc498731b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -2,28 +2,24 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import edu.umd.cs.findbugs.annotations.NonNull; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampSearchStreamInfoItemExtractor; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; -import java.util.Collections; -import java.util.List; + +import javax.annotation.Nonnull; public class BandcampSearchExtractor extends SearchExtractor { @@ -32,23 +28,7 @@ public BandcampSearchExtractor(final StreamingService service, super(service, linkHandler); } - @NonNull - @Override - public String getSearchSuggestion() { - return Utils.EMPTY_STRING; - } - @Override - public boolean isCorrectedSearch() { - return false; - } - - @Nonnull - @Override - public List getMetaInfo() throws ParsingException { - return Collections.emptyList(); - } - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java index 42b3170b3e..5577a8b621 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -1,11 +1,10 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; -import javax.annotation.Nullable; - public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { private final JsonObject discograph; @@ -20,12 +19,6 @@ public String getUploaderName() { return discograph.getString("band_name"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public String getName() { return discograph.getString("title"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index 1b2e7e886c..6738e0cd6e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -3,12 +3,12 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import javax.annotation.Nullable; import java.io.IOException; @@ -56,12 +56,6 @@ public String getUploaderName() { return ""; } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - /** * Each track can have its own cover art. Therefore, unless a substitute is provided, * the thumbnail is extracted using a stream extractor. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index 5e585e7e00..4aa37f0d75 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -4,8 +4,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; -import javax.annotation.Nullable; - public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { private final Element resultInfo; @@ -29,12 +27,6 @@ public String getUploaderName() { } } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public String getName() throws ParsingException { return resultInfo.getElementsByClass("heading").text(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java index fee96fcd97..f23a4cf97a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; @@ -43,14 +42,4 @@ public String getTextualUploadDate() { public DateWrapper getUploadDate() { return null; } - - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - - @Override - public boolean isAd() { - return false; - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 54bfb95304..25e49bbaef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -89,46 +89,41 @@ public SuggestionExtractor getSuggestionExtractor() { } @Override - public KioskList getKioskList() throws ExtractionException { + public KioskList getKioskList() { final KioskList list = new KioskList(this); // add kiosks here e.g.: - try { - list.addKioskEntry( - (streamingService, url, kioskId) -> new MediaCCCConferenceKiosk( - MediaCCCService.this, - new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), - kioskId - ), - new MediaCCCConferencesListLinkHandlerFactory(), - "conferences" - ); - - list.addKioskEntry( - (streamingService, url, kioskId) -> new MediaCCCRecentKiosk( - MediaCCCService.this, - new MediaCCCRecentListLinkHandlerFactory().fromUrl(url), - kioskId - ), - new MediaCCCRecentListLinkHandlerFactory(), - "recent" - ); - - list.addKioskEntry( - (streamingService, url, kioskId) -> new MediaCCCLiveStreamKiosk( - MediaCCCService.this, - new MediaCCCLiveListLinkHandlerFactory().fromUrl(url), - kioskId - ), - new MediaCCCLiveListLinkHandlerFactory(), - "live" - ); - - list.setDefaultKiosk("recent"); - } catch (final Exception e) { - throw new ExtractionException(e); - } - + list.addKioskEntry( + (streamingService, url, kioskId) -> new MediaCCCConferenceKiosk( + MediaCCCService.this, + new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), + kioskId + ), + new MediaCCCConferencesListLinkHandlerFactory(), + "conferences" + ); + + list.addKioskEntry( + (streamingService, url, kioskId) -> new MediaCCCRecentKiosk( + MediaCCCService.this, + new MediaCCCRecentListLinkHandlerFactory().fromUrl(url), + kioskId + ), + new MediaCCCRecentListLinkHandlerFactory(), + "recent" + ); + + list.addKioskEntry( + (streamingService, url, kioskId) -> new MediaCCCLiveStreamKiosk( + MediaCCCService.this, + new MediaCCCLiveListLinkHandlerFactory().fromUrl(url), + kioskId + ), + new MediaCCCLiveListLinkHandlerFactory(), + "live" + ); + + list.setDefaultKiosk("recent"); return list; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 98aa2c2571..289c347f4d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -16,9 +17,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; +import javax.annotation.Nonnull; + public class MediaCCCConferenceExtractor extends ChannelExtractor { private JsonObject conferenceData; @@ -52,26 +54,6 @@ public String getDescription() { return null; } - @Override - public String getParentChannelName() { - return ""; - } - - @Override - public String getParentChannelUrl() { - return ""; - } - - @Override - public String getParentChannelAvatarUrl() { - return ""; - } - - @Override - public boolean isVerified() { - return false; - } - @Nonnull @Override public InfoItemsPage getInitialPage() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java index 2b311fb352..177143ae9a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; @@ -49,11 +50,6 @@ public StreamType getStreamType() throws ParsingException { return isVideo ? StreamType.LIVE_STREAM : StreamType.AUDIO_LIVE_STREAM; } - @Override - public boolean isAd() throws ParsingException { - return false; - } - @Override public long getDuration() throws ParsingException { return 0; @@ -75,17 +71,6 @@ public String getUploaderUrl() throws ParsingException { return "https://media.ccc.de/c/" + conferenceInfo.getString("slug"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Nullable @Override public String getTextualUploadDate() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java index 1e18f8da13..ded86eb94f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java @@ -41,11 +41,6 @@ public StreamType getStreamType() throws ParsingException { return StreamType.VIDEO_STREAM; } - @Override - public boolean isAd() { - return false; - } - @Override public long getDuration() { // duration and length have the same value, see @@ -70,17 +65,6 @@ public String getUploaderUrl() throws ParsingException { .getUrl(); // web URL } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Nullable @Override public String getTextualUploadDate() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index e9c300b8ab..0d98543c50 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -10,7 +10,7 @@ import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; @@ -18,13 +18,11 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; import java.io.IOException; -import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; @@ -45,23 +43,6 @@ public MediaCCCSearchExtractor(final StreamingService service, } } - @Nonnull - @Override - public String getSearchSuggestion() { - return ""; - } - - @Override - public boolean isCorrectedSearch() { - return false; - } - - @Nonnull - @Override - public List getMetaInfo() { - return Collections.emptyList(); - } - @Nonnull @Override public InfoItemsPage getInitialPage() { @@ -141,11 +122,6 @@ public long getStreamCount() { return item.getStreamCount(); } - @Override - public boolean isVerified() { - return false; - } - @Override public String getName() { return item.getName(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java index b69d7a9083..3ced44d9ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -28,11 +28,6 @@ public long getStreamCount() { return ListExtractor.ITEM_COUNT_UNKNOWN; } - @Override - public boolean isVerified() throws ParsingException { - return false; - } - @Override public String getName() throws ParsingException { return conference.getString("title"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index 92f0894b9f..17e76e0271 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper; @@ -21,11 +22,6 @@ public StreamType getStreamType() { return StreamType.VIDEO_STREAM; } - @Override - public boolean isAd() { - return false; - } - @Override public long getDuration() { return event.getInt("length"); @@ -46,17 +42,6 @@ public String getUploaderUrl() { return event.getString("conference_url"); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Nullable @Override public String getTextualUploadDate() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 5b6d64d6ff..c9e9959d42 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -2,60 +2,47 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.instance.AbstractInstance; +import org.schabi.newpipe.extractor.instance.InstanceMetaDataFetchException; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; +public class PeertubeInstance extends AbstractInstance { -public class PeertubeInstance { + public static final String INSTANCE_SERVICE_NAME = "Peertube"; - private final String url; - private String name; - public static final PeertubeInstance DEFAULT_INSTANCE - = new PeertubeInstance("https://framatube.org", "FramaTube"); + public static final PeertubeInstance DEFAULT_INSTANCE = + new PeertubeInstance("https://framatube.org", "FramaTube"); public PeertubeInstance(final String url) { - this.url = url; - this.name = "PeerTube"; + super(url, INSTANCE_SERVICE_NAME); } public PeertubeInstance(final String url, final String name) { - this.url = url; - this.name = name; + super(url, name); } - public String getUrl() { - return url; - } - - public void fetchInstanceMetaData() throws Exception { - final Response response; + @Override + public void fetchMetadata() { + final String urlToCheck = getUrl() + "/api/v1/config"; try { - response = NewPipe.getDownloader().get(url + "/api/v1/config"); - } catch (ReCaptchaException | IOException e) { - throw new Exception("unable to configure instance " + url, e); - } + final Response response = NewPipe.getDownloader().get(urlToCheck); - if (response == null || Utils.isBlank(response.responseBody())) { - throw new Exception("unable to configure instance " + url); - } + if (response == null) { + throw new IllegalStateException("Received no response"); + } + if (Utils.isBlank(response.responseBody())) { + throw new IllegalStateException("Received no response data"); + } - try { final JsonObject json = JsonParser.object().from(response.responseBody()); - this.name = JsonUtils.getString(json, "instance.name"); - } catch (JsonParserException | ParsingException e) { - throw new Exception("unable to parse instance config", e); + this.setName(JsonUtils.getString(json, "instance.name")); + } catch (final Exception e) { + throw new InstanceMetaDataFetchException( + "Unable to fetch trending from " + urlToCheck, e); } } - - public String getName() { - return name; - } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index de25e2799f..7ffb4c709a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -4,6 +4,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; import static java.util.Arrays.asList; +import org.schabi.newpipe.extractor.InstanceBasedStreamingService; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -37,7 +38,8 @@ import java.util.List; -public class PeertubeService extends StreamingService { +public class PeertubeService extends StreamingService + implements InstanceBasedStreamingService { private PeertubeInstance instance; @@ -50,6 +52,21 @@ public PeertubeService(final int id, final PeertubeInstance instance) { this.instance = instance; } + @Override + public PeertubeInstance getInstance() { + return this.instance; + } + + @Override + public void setInstance(final PeertubeInstance instance) { + this.instance = instance; + } + + @Override + public String getBaseUrl() { + return instance.getUrl(); + } + @Override public LinkHandlerFactory getStreamLHFactory() { return PeertubeStreamLinkHandlerFactory.getInstance(); @@ -122,20 +139,7 @@ public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler) } @Override - public String getBaseUrl() { - return instance.getUrl(); - } - - public PeertubeInstance getInstance() { - return this.instance; - } - - public void setInstance(final PeertubeInstance instance) { - this.instance = instance; - } - - @Override - public KioskList getKioskList() throws ExtractionException { + public KioskList getKioskList() { final KioskList.KioskExtractorFactory kioskFactory = (streamingService, url, id) -> new PeertubeTrendingExtractor( PeertubeService.this, @@ -147,16 +151,12 @@ public KioskList getKioskList() throws ExtractionException { // add kiosks here e.g.: final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory(); - try { - list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); - list.addKioskEntry(kioskFactory, h, - PeertubeTrendingLinkHandlerFactory.KIOSK_MOST_LIKED); - list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT); - list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL); - list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); - } catch (final Exception e) { - throw new ExtractionException(e); - } + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + list.addKioskEntry(kioskFactory, h, + PeertubeTrendingLinkHandlerFactory.KIOSK_MOST_LIKED); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL); + list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); return list; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index f63eca52f1..7e8a87f1fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -1,9 +1,16 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -20,14 +27,9 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public class PeertubeAccountExtractor extends ChannelExtractor { private JsonObject json; @@ -99,26 +101,6 @@ public String getDescription() { } } - @Override - public String getParentChannelName() { - return ""; - } - - @Override - public String getParentChannelUrl() { - return ""; - } - - @Override - public String getParentChannelAvatarUrl() { - return ""; - } - - @Override - public boolean isVerified() throws ParsingException { - return false; - } - @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index c5b7bf399d..fbaedd44d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -1,8 +1,15 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -18,14 +25,9 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public class PeertubeChannelExtractor extends ChannelExtractor { private JsonObject json; @@ -93,11 +95,6 @@ public String getParentChannelAvatarUrl() { return baseUrl + value; } - @Override - public boolean isVerified() throws ParsingException { - return false; - } - @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 71648b1ecd..46544b39cd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -1,8 +1,15 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -16,14 +23,9 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public class PeertubePlaylistExtractor extends PlaylistExtractor { private JsonObject playlistInfo; @@ -55,11 +57,6 @@ public String getUploaderAvatarUrl() throws ParsingException { + playlistInfo.getObject("ownerAccount").getObject("avatar").getString("path"); } - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Override public long getStreamCount() { return playlistInfo.getLong("videosLength"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 8be07855d8..12e5aa6c73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -1,10 +1,16 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,23 +18,14 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.List; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - public class PeertubeSearchExtractor extends SearchExtractor { // if we should use PeertubeSepiaStreamInfoItemExtractor @@ -46,24 +43,6 @@ public PeertubeSearchExtractor(final StreamingService service, this.sepia = sepia; } - @Nonnull - @Override - public String getSearchSuggestion() { - return ""; - } - - @Override - public boolean isCorrectedSearch() { - return false; - } - - @Nonnull - @Override - public List getMetaInfo() { - return Collections.emptyList(); - } - - @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { return getPage(new Page(getUrl() + "&" + START_KEY + "=0&" diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index b9c7f4b13e..5f427511a5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -37,11 +38,6 @@ public String getName() throws ParsingException { return JsonUtils.getString(item, "name"); } - @Override - public boolean isAd() { - return false; - } - @Override public long getViewCount() { return item.getLong("views"); @@ -66,11 +62,6 @@ public String getUploaderAvatarUrl() { return null; } - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Override public String getUploaderName() throws ParsingException { return JsonUtils.getString(item, "account.displayName"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index cc4977e42d..8cf56d3f52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -102,7 +102,7 @@ public SoundcloudSuggestionExtractor getSuggestionExtractor() { } @Override - public KioskList getKioskList() throws ExtractionException { + public KioskList getKioskList() { final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) -> new SoundcloudChartsExtractor(SoundcloudService.this, new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id); @@ -111,13 +111,9 @@ public KioskList getKioskList() throws ExtractionException { // add kiosks here e.g.: final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory(); - try { - list.addKioskEntry(chartsFactory, h, "Top 50"); - list.addKioskEntry(chartsFactory, h, "New & hot"); - list.setDefaultKiosk("New & hot"); - } catch (final Exception e) { - throw new ExtractionException(e); - } + list.addKioskEntry(chartsFactory, h, "Top 50"); + list.addKioskEntry(chartsFactory, h, "New & hot"); + list.setDefaultKiosk("New & hot"); return list; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 542666eb18..355958ad0b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -1,8 +1,13 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -14,12 +19,9 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; -import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public class SoundcloudChannelExtractor extends ChannelExtractor { private String userId; @@ -85,21 +87,6 @@ public String getDescription() { return user.getString("description", EMPTY_STRING); } - @Override - public String getParentChannelName() { - return ""; - } - - @Override - public String getParentChannelUrl() { - return ""; - } - - @Override - public String getParentChannelAvatarUrl() { - return ""; - } - @Override public boolean isVerified() throws ParsingException { return user.getBoolean("verified"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 1ae0becad5..a22fdc10ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -1,5 +1,9 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -8,29 +12,23 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemsCollector; -import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.utils.Parser; -import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; -import java.util.Collections; -import java.util.List; import java.util.function.IntUnaryOperator; -import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public class SoundcloudSearchExtractor extends SearchExtractor { private JsonArray initialSearchCollection; @@ -40,23 +38,6 @@ public SoundcloudSearchExtractor(final StreamingService service, super(service, linkHandler); } - @Nonnull - @Override - public String getSearchSuggestion() { - return ""; - } - - @Override - public boolean isCorrectedSearch() { - return false; - } - - @Nonnull - @Override - public List getMetaInfo() { - return Collections.emptyList(); - } - @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index 3647ff09e2..124c1062bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -1,17 +1,16 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import javax.annotation.Nullable; - -import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { protected final JsonObject itemObject; @@ -45,12 +44,6 @@ public String getUploaderUrl() { return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url")); } - @Nullable - @Override - public String getUploaderAvatarUrl() { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return itemObject.getObject("user").getBoolean("verified"); @@ -84,9 +77,4 @@ public String getThumbnailUrl() { public StreamType getStreamType() { return StreamType.AUDIO_STREAM; } - - @Override - public boolean isAd() { - return false; - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeInstance.java new file mode 100644 index 0000000000..6a3af903a3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeInstance.java @@ -0,0 +1,13 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.instance.AbstractInstance; + +public abstract class YoutubeLikeInstance + extends AbstractInstance { + + protected YoutubeLikeInstance(final String url, final String name) { + super(url, name); + } + + public abstract S getNewStreamingService(int id); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeStreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeStreamingService.java new file mode 100644 index 0000000000..2eacfa95f9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeStreamingService.java @@ -0,0 +1,47 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamExtractor; + +import java.util.List; + +import javax.annotation.Nonnull; + +public abstract class YoutubeLikeStreamingService extends StreamingService { + + /** + * Creates a new Streaming service. + * If you Implement one do not set id within your implementation of this extractor, instead + * set the id when you put the extractor into {@link org.schabi.newpipe.extractor.ServiceList} + * All other parameters can be set directly from the overriding constructor. + * + * @param id the number of the service to identify him within the NewPipe frontend + * @param name the name of the service + * @param capabilities the type of media this service can handle + */ + protected YoutubeLikeStreamingService( + final int id, + final String name, + final List capabilities) { + super(id, name, capabilities); + } + + @Override + public abstract StreamExtractor getStreamExtractor(final LinkHandler linkHandler); + + @Override + public abstract ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler); + + @Override + public abstract PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler); + + @Nonnull + @Override + public abstract FeedExtractor getFeedExtractor(final String url) throws ExtractionException; +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index a15d3ed012..f637d5e713 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -6,6 +6,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; import static java.util.Arrays.asList; +import org.schabi.newpipe.extractor.InstanceBasedStreamingService; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -22,28 +23,13 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeInstance; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.util.List; +import java.util.Objects; import javax.annotation.Nonnull; @@ -67,153 +53,125 @@ * along with NewPipe. If not, see . */ -public class YoutubeService extends StreamingService { +public class YoutubeService extends StreamingService + implements InstanceBasedStreamingService> { + + @Nonnull + protected YoutubeLikeInstance instance; + @Nonnull + protected YoutubeLikeStreamingService subService; public YoutubeService(final int id) { + this(id, YoutubeInstance.YOUTUBE); + } + + public YoutubeService( + final int id, + final YoutubeLikeInstance instance) { super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS)); + this.setInstance(instance); + } + + @Override + @Nonnull + public YoutubeLikeInstance getInstance() { + return instance; + } + + @Override + public void setInstance(final YoutubeLikeInstance instance) { + this.instance = Objects.requireNonNull(instance); + subService = Objects.requireNonNull(instance.getNewStreamingService(getServiceId())); } @Override public String getBaseUrl() { - return "https://youtube.com"; + return subService.getBaseUrl(); } @Override public LinkHandlerFactory getStreamLHFactory() { - return YoutubeStreamLinkHandlerFactory.getInstance(); + return subService.getStreamLHFactory(); } @Override public ListLinkHandlerFactory getChannelLHFactory() { - return YoutubeChannelLinkHandlerFactory.getInstance(); + return subService.getChannelLHFactory(); } @Override public ListLinkHandlerFactory getPlaylistLHFactory() { - return YoutubePlaylistLinkHandlerFactory.getInstance(); + return subService.getPlaylistLHFactory(); } @Override public SearchQueryHandlerFactory getSearchQHFactory() { - return YoutubeSearchQueryHandlerFactory.getInstance(); + return subService.getSearchQHFactory(); } @Override public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) { - return new YoutubeStreamExtractor(this, linkHandler); + return subService.getStreamExtractor(linkHandler); } @Override public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) { - return new YoutubeChannelExtractor(this, linkHandler); + return subService.getChannelExtractor(linkHandler); } @Override public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { - if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId()) - && !YoutubeParsingHelper.isYoutubeMusicMixId(linkHandler.getId())) { - return new YoutubeMixPlaylistExtractor(this, linkHandler); - } else { - return new YoutubePlaylistExtractor(this, linkHandler); - } + return subService.getPlaylistExtractor(linkHandler); } @Override public SearchExtractor getSearchExtractor(final SearchQueryHandler query) { - final List contentFilters = query.getContentFilters(); - - if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("music_")) { - return new YoutubeMusicSearchExtractor(this, query); - } else { - return new YoutubeSearchExtractor(this, query); - } + return subService.getSearchExtractor(query); } @Override public SuggestionExtractor getSuggestionExtractor() { - return new YoutubeSuggestionExtractor(this); + return subService.getSuggestionExtractor(); } @Override - public KioskList getKioskList() throws ExtractionException { - final KioskList list = new KioskList(this); - - // add kiosks here e.g.: - try { - list.addKioskEntry( - (streamingService, url, id) -> new YoutubeTrendingExtractor( - YoutubeService.this, - new YoutubeTrendingLinkHandlerFactory().fromUrl(url), - id - ), - new YoutubeTrendingLinkHandlerFactory(), - "Trending" - ); - list.setDefaultKiosk("Trending"); - } catch (final Exception e) { - throw new ExtractionException(e); - } - - return list; + public KioskList getKioskList() { + return subService.getKioskList(); } @Override public SubscriptionExtractor getSubscriptionExtractor() { - return new YoutubeSubscriptionExtractor(this); + return subService.getSubscriptionExtractor(); } @Nonnull @Override public FeedExtractor getFeedExtractor(final String channelUrl) throws ExtractionException { - return new YoutubeFeedExtractor(this, getChannelLHFactory().fromUrl(channelUrl)); + return subService.getFeedExtractor(channelUrl); } @Override public ListLinkHandlerFactory getCommentsLHFactory() { - return YoutubeCommentsLinkHandlerFactory.getInstance(); + return subService.getCommentsLHFactory(); } @Override public CommentsExtractor getCommentsExtractor(final ListLinkHandler urlIdHandler) throws ExtractionException { - return new YoutubeCommentsExtractor(this, urlIdHandler); + return subService.getCommentsExtractor(urlIdHandler); } /*////////////////////////////////////////////////////////////////////////// // Localization //////////////////////////////////////////////////////////////////////////*/ - // https://www.youtube.com/picker_ajax?action_language_json=1 - private static final List SUPPORTED_LANGUAGES = Localization.listFrom( - "en-GB" - /*"af", "am", "ar", "az", "be", "bg", "bn", "bs", "ca", "cs", "da", "de", - "el", "en", "en-GB", "es", "es-419", "es-US", "et", "eu", "fa", "fi", "fil", "fr", - "fr-CA", "gl", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "iw", "ja", - "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", "mn", - "mr", "ms", "my", "ne", "nl", "no", "pa", "pl", "pt", "pt-PT", "ro", "ru", - "si", "sk", "sl", "sq", "sr", "sr-Latn", "sv", "sw", "ta", "te", "th", "tr", - "uk", "ur", "uz", "vi", "zh-CN", "zh-HK", "zh-TW", "zu"*/ - ); - - // https://www.youtube.com/picker_ajax?action_country_json=1 - private static final List SUPPORTED_COUNTRIES = ContentCountry.listFrom( - "DZ", "AR", "AU", "AT", "AZ", "BH", "BD", "BY", "BE", "BO", "BA", "BR", "BG", "CA", - "CL", "CO", "CR", "HR", "CY", "CZ", "DK", "DO", "EC", "EG", "SV", "EE", "FI", "FR", - "GE", "DE", "GH", "GR", "GT", "HN", "HK", "HU", "IS", "IN", "ID", "IQ", "IE", "IL", - "IT", "JM", "JP", "JO", "KZ", "KE", "KW", "LV", "LB", "LY", "LI", "LT", "LU", "MY", - "MT", "MX", "ME", "MA", "NP", "NL", "NZ", "NI", "NG", "MK", "NO", "OM", "PK", "PA", - "PG", "PY", "PE", "PH", "PL", "PT", "PR", "QA", "RO", "RU", "SA", "SN", "RS", "SG", - "SK", "SI", "ZA", "KR", "ES", "LK", "SE", "CH", "TW", "TZ", "TH", "TN", "TR", "UG", - "UA", "AE", "GB", "US", "UY", "VE", "VN", "YE", "ZW" - ); - @Override public List getSupportedLocalizations() { - return SUPPORTED_LANGUAGES; + return subService.getSupportedLocalizations(); } @Override public List getSupportedCountries() { - return SUPPORTED_COUNTRIES; + return subService.getSupportedCountries(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java new file mode 100644 index 0000000000..713a502ba2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java @@ -0,0 +1,84 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.instance.InstanceMetaDataFetchException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeLikeInstance; +import org.schabi.newpipe.extractor.utils.Utils; + +public class InvidiousInstance extends YoutubeLikeInstance { + + public InvidiousInstance(final String url) { + super(url, tryExtractDomainFromUrl(url, "Invidious instance" + url.hashCode())); + } + + public InvidiousInstance(final String url, final String name) { + super(url, name); + } + + @Override + public void fetchMetadata() { + checkStatsEndpoint(); + checkTrendingEndpoint(); + } + + private void checkStatsEndpoint() { + checkEndpoint("/api/v1/stats?fields=software,error", r -> { + final JsonObject json = JsonParser.object().from(r.responseBody()); + if (!json.has("software") + && !"Statistics are not enabled.".equals(json.getString("error"))) { + throw new IllegalStateException("Could not get stats"); + } + }); + } + + private void checkTrendingEndpoint() { + checkEndpoint("/api/v1/trending", r -> { + final JsonArray json = JsonParser.array().from(r.responseBody()); + if (json.isEmpty()) { + throw new IllegalStateException("No data returned"); + } + }); + } + + private void checkEndpoint( + final String subUrlToCheck, + final ResponseCheck responseCheck + ) { + final String urlToCheck = getUrl() + subUrlToCheck; + try { + final Response response = NewPipe.getDownloader().get(urlToCheck); + + if (response == null) { + throw new IllegalStateException("Received no response"); + } + if (Utils.isBlank(response.responseBody())) { + throw new IllegalStateException("Received no response data"); + } + + responseCheck.check(response); + } catch (final Exception e) { + throw new InstanceMetaDataFetchException( + "Unable to fetch trending from " + urlToCheck, e); + } + } + + @Override + public InvidiousService getNewStreamingService(final int id) { + return new InvidiousService(id, this); + } + + @Override + public String getServiceName() { + return "Invidious"; + } + + @FunctionalInterface + public interface ResponseCheck { + void check(final Response r) throws Exception; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java new file mode 100644 index 0000000000..daca29af4b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java @@ -0,0 +1,156 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious; + +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.localization.DateWrapper; + +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; + +import javax.annotation.Nonnull; + +public final class InvidiousParsingHelper { + private InvidiousParsingHelper() { + // No impl pls + } + + /** + * Get valid JsonObject + *

+ * Checks status code and handle JSON parsing + * + * @param response the response got from the service + * @param apiUrl the url used to call the service + * @return the JsonObject + * @throws ExtractionException if the HTTP code indicate an error or the json parsing went wrong + */ + public static JsonObject getValidJsonObjectFromResponse( + final Response response, + final String apiUrl + ) throws ExtractionException { + final String responseBody = getValidResponseBody(response, apiUrl); + + try { + return JsonParser.object().from(responseBody); + } catch (final JsonParserException e) { + throw new ExtractionException("Could not parse json from page \"" + apiUrl + "\"", e); + } + } + + /** + * Get valid Response body + *

+ * Checks status code and handle JSON parsing + * + * @param response the response got from the service + * @param apiUrl the url used to call the service + * @return the response body + * @throws ExtractionException if the HTTP code indicate an error or the json parsing went wrong + */ + public static String getValidResponseBody( + final Response response, + final String apiUrl + ) throws ExtractionException { + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Could not get page " + apiUrl + + " (HTTP " + response.responseCode() + " : " + response.responseMessage()); + } else if (response.responseCode() >= 400) { + throw new ExtractionException("Could not get page " + apiUrl + + " (HTTP " + response.responseCode() + " : " + response.responseMessage()); + } + + return response.responseBody(); + } + + /** + * Get valid Response body + *

+ * Checks status code and handle JSON parsing + * + * @param response the response got from the service + * @param apiUrl the url used to call the service + * @return the JsonArray + * @throws ExtractionException if the HTTP code indicate an error or the json parsing went wrong + */ + public static JsonArray getValidJsonArrayFromResponse( + final Response response, + final String apiUrl + ) throws ExtractionException { + final String responseBody = getValidResponseBody(response, apiUrl); + + try { + return JsonParser.array().from(responseBody); + } catch (final JsonParserException e) { + throw new ExtractionException("Could not parse json from page \"" + apiUrl + "\"", e); + } + } + + public static DateWrapper getUploadDateFromEpochTime(final long epochTime) { + return new DateWrapper( + OffsetDateTime.ofInstant(Instant.ofEpochMilli(epochTime), ZoneOffset.UTC)); + } + + public static String getUid(@Nonnull final String id) { + if (id.startsWith("user/")) { + return id.substring(5); + } else if (id.startsWith("channel/")) { + return id.substring(8); + } + + return id; + } + + public static Page getPage(final String url, final int page) { + return new Page(url + "?page=" + page, String.valueOf(page)); + } + + /** + * Get thumbnail URL at a reasonable quality + * + * @param thumbnails an array of thumbnails + * @return a thumbnail URL at a reasonable quality + */ + public static String getThumbnailUrl(final JsonArray thumbnails) { + String url = ""; + + if (thumbnails.isEmpty()) { + return url; + } else if (thumbnails.size() == 1) { + url = thumbnails.getObject(0).getString("url"); + } else { + url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + for (int i = 1; i < thumbnails.size(); i++) { + final JsonObject thumbnail = thumbnails.getObject(i); + final String quality = thumbnail.getString("quality"); + if ("high".equals(quality)) { + url = thumbnail.getString("url"); + break; + } + } + } + return fixThumbnailUrl(url); + } + + public static String fixThumbnailUrl(String thumbnailUrl) { + if (thumbnailUrl.startsWith(HTTP) || thumbnailUrl.startsWith(HTTPS)) { + return thumbnailUrl; + } + + if (thumbnailUrl.startsWith("//")) { + thumbnailUrl = thumbnailUrl.substring(2); + } + + return HTTPS + thumbnailUrl; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java new file mode 100644 index 0000000000..19952b2be4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java @@ -0,0 +1,176 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious; + +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; +import static java.util.Arrays.asList; + +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.localization.ContentCountry; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.YoutubeLikeStreamingService; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousCommentsExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousSuggestionExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler.InvidiousChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler.InvidiousCommentLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler.InvidiousPlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler.InvidiousSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler.InvidiousStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler.InvidiousTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubePlaylistHelper; +import org.schabi.newpipe.extractor.services.youtube.shared.extractors.YoutubeTakeoutSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeDirectService; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +import java.util.List; + +import javax.annotation.Nonnull; + +public class InvidiousService extends YoutubeLikeStreamingService { + protected InvidiousInstance instance; + + public InvidiousService(final int id) { + super(id, "Invidious", asList(AUDIO, VIDEO, LIVE, COMMENTS)); + } + + public InvidiousService(final int id, final InvidiousInstance instance) { + super(id, "Invidious", asList(AUDIO, VIDEO, LIVE, COMMENTS)); + this.instance = instance; + } + + @Override + public String getBaseUrl() { + return instance.getUrl(); + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return new InvidiousStreamLinkHandlerFactory(this); + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + return new InvidiousChannelLinkHandlerFactory(this); + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + return new InvidiousPlaylistLinkHandlerFactory(this); + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return new InvidiousSearchQueryHandlerFactory(this); + } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return new InvidiousCommentLinkHandlerFactory(this); + } + + @Override + public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) { + return new InvidiousSearchExtractor(this, queryHandler); + } + + @Override + public SuggestionExtractor getSuggestionExtractor() { + return new InvidiousSuggestionExtractor(this); + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + // Invidious provides also exports + return new YoutubeTakeoutSubscriptionExtractor(this); + } + + @Override + public KioskList getKioskList() { + + final KioskList.KioskExtractorFactory kioskFactory = ((streamingService, url, id) -> + new InvidiousTrendingExtractor( + InvidiousService.this, + new InvidiousTrendingLinkHandlerFactory(InvidiousService.this).fromId(id), + id + ) + ); + + final KioskList list = new KioskList(this); + + final InvidiousTrendingLinkHandlerFactory lhf = + new InvidiousTrendingLinkHandlerFactory(InvidiousService.this); + list.addKioskEntry(kioskFactory, lhf, InvidiousTrendingLinkHandlerFactory.KIOSK_POPULAR); + list.addKioskEntry(kioskFactory, lhf, InvidiousTrendingLinkHandlerFactory.KIOSK_TRENDING); + list.setDefaultKiosk(InvidiousTrendingLinkHandlerFactory.KIOSK_TRENDING); + + return list; + } + + @Override + public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) { + return new InvidiousChannelExtractor(this, linkHandler); + } + + @Override + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { + return YoutubePlaylistHelper.isYoutubeMixId(linkHandler.getId()) + ? new InvidiousMixPlaylistExtractor(this, linkHandler) + : new InvidiousPlaylistExtractor(this, linkHandler); + } + + @Nonnull + @Override + public FeedExtractor getFeedExtractor(final String url) throws ExtractionException { + return new InvidiousFeedExtractor(this, getChannelLHFactory().fromUrl(url)); + } + + @Override + public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) { + return new InvidiousStreamExtractor(this, linkHandler); + } + + @Override + public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler) { + return new InvidiousCommentsExtractor(this, linkHandler); + } + + public InvidiousInstance getInstance() { + return instance; + } + + public void setInstance(final InvidiousInstance instance) { + this.instance = instance; + } + + @Override + public List getSupportedLocalizations() { + return YoutubeDirectService.SUPPORTED_LANGUAGES; + } + + @Override + public List getSupportedCountries() { + return YoutubeDirectService.SUPPORTED_COUNTRIES; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java new file mode 100644 index 0000000000..1165aac5db --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java @@ -0,0 +1,114 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getUid; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import java.io.IOException; + +import javax.annotation.Nonnull; + + +public class InvidiousChannelExtractor extends ChannelExtractor { + + private final String baseUrl; + private JsonObject json; + + public InvidiousChannelExtractor( + final InvidiousService service, + final ListLinkHandler linkHandler + ) { + super(service, linkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return getPage(getPage(1)); + } + + protected Page getPage(final int page) { + return InvidiousParsingHelper.getPage( + baseUrl + "/api/v1/channels/videos/" + json.getString("authorId"), + page + ); + } + + @Override + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + final Response rp = NewPipe.getDownloader().get(page.getUrl()); + final JsonArray array = + InvidiousParsingHelper.getValidJsonArrayFromResponse(rp, page.getUrl()); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + array.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(o -> new InvidiousStreamInfoItemExtractor(o, baseUrl)) + .forEach(collector::commit); + + final Page nextPage = array.size() < 59 + // max number of items per page + // with Second it is 29 but next Page logic is not implemented + ? null + : getPage(Integer.parseInt(page.getId()) + 1); + + return new InfoItemsPage<>(collector, nextPage); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/channels/" + getUid(getId()) + + "?fields=author,description,subCount,authorThumbnails,authorBanners,authorId" + + "®ion=" + getExtractorContentCountry().getCountryCode(); + + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Override + public String getAvatarUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + + @Override + public String getBannerUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorBanners")); + } + + @Override + public String getFeedUrl() { + return baseUrl + "/feed/channel/" + json.getString("authorId"); + } + + @Override + public long getSubscriberCount() { + return json.getNumber("subCount").longValue(); + } + + @Override + public String getDescription() { + return json.getString("description"); + } + + @Nonnull + @Override + public String getName() { + return json.getString("author"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelInfoItemExtractor.java new file mode 100644 index 0000000000..d7efc17b3b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelInfoItemExtractor.java @@ -0,0 +1,52 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; + +public class InvidiousChannelInfoItemExtractor implements ChannelInfoItemExtractor { + + private final String baseUrl; + private final JsonObject json; + + public InvidiousChannelInfoItemExtractor( + final JsonObject json, + final String baseUrl + ) { + this.json = json; + this.baseUrl = baseUrl; + } + + @Override + public String getName() { + return json.getString("author"); + } + + @Override + public String getUrl() { + return baseUrl + json.getString("authorUrl"); + } + + @Override + public String getThumbnailUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + + @Override + public String getDescription() throws ParsingException { + // Note: descriptionHtml is also available + return json.getString("description"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + return json.getLong("subCount"); + } + + @Override + public long getStreamCount() throws ParsingException { + return json.getLong("videoCount"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java new file mode 100644 index 0000000000..d1aad79d92 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java @@ -0,0 +1,89 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; + +import java.io.IOException; + +import javax.annotation.Nonnull; + +public class InvidiousCommentsExtractor extends CommentsExtractor { + + private final String baseUrl; + private JsonObject json; + + public InvidiousCommentsExtractor( + final InvidiousService service, + final ListLinkHandler uiHandler + ) { + super(service, uiHandler); + baseUrl = service.getInstance().getUrl(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException { + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + + collectStreamsFrom(collector, json.getArray("comments"), getUrl()); + + return new InfoItemsPage<>(collector, getNextPage()); + } + + @Override + public InfoItemsPage getPage( + final Page page + ) throws IOException, ExtractionException { + final Downloader dl = NewPipe.getDownloader(); + final Response response = dl.get(page.getUrl()); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, page.getUrl()); + + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, json.getArray("comments"), page.getUrl()); + return new InfoItemsPage<>(collector, getNextPage()); + } + + + public Page getNextPage() throws ParsingException { + return new Page(baseUrl + "/api/v1/comments/" + getId() + + "?continuation=" + json.getString("continuation")); + } + + @Override + public void onFetchPage( + @Nonnull final Downloader downloader + ) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/comments/" + getId(); + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + + } + + private void collectStreamsFrom( + final CommentsInfoItemsCollector collector, + final JsonArray entries, + final String url + ) { + entries.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(commentObj -> new InvidiousCommentsInfoItemExtractor(commentObj, url)) + .forEach(collector::commit); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java new file mode 100644 index 0000000000..1569b042d9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java @@ -0,0 +1,84 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; + +import javax.annotation.Nullable; + +public class InvidiousCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + + private final String url; + private final JsonObject json; + + public InvidiousCommentsInfoItemExtractor(final JsonObject json, final String url) { + this.json = json; + this.url = url; + } + + @Override + public int getLikeCount() { + return json.getNumber("likeCount").intValue(); + } + + @Override + public String getCommentText() { + return json.getString("content"); + } + + @Override + public String getTextualUploadDate() { + return json.getString("publishedText"); + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return InvidiousParsingHelper.getUploadDateFromEpochTime( + json.getNumber("published").longValue()); + } + + @Override + public String getCommentId() { + return json.getString("commentId"); + } + + @Override + public String getUploaderUrl() { + return json.getString("authorUrl"); + } + + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public String getUploaderAvatarUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + + @Override + public boolean isHeartedByUploader() throws ParsingException { + return json.has("creatorHeart"); + } + + @Override + public String getName() throws ParsingException { + return json.getString("author"); + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java new file mode 100644 index 0000000000..ee17e7d33f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java @@ -0,0 +1,82 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getValidResponseBody; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeFeedInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import java.io.IOException; + +import javax.annotation.Nonnull; + +public class InvidiousFeedExtractor extends FeedExtractor { + + private final String baseUrl; + private Document document; + + public InvidiousFeedExtractor( + final InvidiousService service, + final ListLinkHandler listLinkHandler + ) { + super(service, listLinkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() { + final Elements entries = document.select("feed > entry"); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + for (final Element entryElement : entries) { + // no need for InvidiousFeedInfoItemExtractor, it's exactly the same structure + collector.commit(new YoutubeFeedInfoItemExtractor(entryElement)); + } + + return new InfoItemsPage<>(collector, null); + + } + + @Override + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + return null; + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String feedUrl = baseUrl + "/feed/channel/" + getLinkHandler().getId(); + final Response response = downloader.get(feedUrl); + document = Jsoup.parse(getValidResponseBody(response, feedUrl)); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return document.select("feed > author > name").first().text(); + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return document.select("feed > author > uri").first().text(); + } + + @Nonnull + @Override + public String getId() throws ParsingException { + return document.getElementsByTag("yt:channelId").first().text(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousMixPlaylistExtractor.java new file mode 100644 index 0000000000..253f7d3b29 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousMixPlaylistExtractor.java @@ -0,0 +1,94 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.io.IOException; + +import javax.annotation.Nonnull; + +public class InvidiousMixPlaylistExtractor extends PlaylistExtractor { + + private JsonObject json; + private final String baseUrl; + + public InvidiousMixPlaylistExtractor( + final InvidiousService service, + final ListLinkHandler linkHandler + ) { + super(service, linkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public void onFetchPage( + @Nonnull final Downloader downloader + ) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/mixes/" + getId(); + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return json.getString("title"); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + json.getArray("videos").stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(o -> new InvidiousStreamInfoItemExtractor(o, baseUrl)) + .forEach(collector::commit); + + return new InfoItemsPage<>(collector, null); + } + + @Override + public InfoItemsPage getPage( + final Page page + ) throws IOException, ExtractionException { + return InfoItemsPage.emptyPage(); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return YoutubeMixPlaylistExtractor + .getThumbnailUrlFromPlaylistId(json.getString("mixId")); + } + + @Override + public String getUploaderUrl() throws ParsingException { + return Utils.EMPTY_STRING; + } + + @Override + public String getUploaderName() throws ParsingException { + return "YouTube"; + } + + @Override + public long getStreamCount() throws ParsingException { + return ListExtractor.ITEM_COUNT_INFINITE; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java new file mode 100644 index 0000000000..311d6db0aa --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java @@ -0,0 +1,113 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import java.io.IOException; + +import javax.annotation.Nonnull; + +public class InvidiousPlaylistExtractor extends PlaylistExtractor { + + private JsonObject json; + private final String baseUrl; + + public InvidiousPlaylistExtractor( + final InvidiousService service, + final ListLinkHandler linkHandler) { + super(service, linkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Nonnull + @Override + public String getThumbnailUrl() { + final JsonArray thumbnails = json.getArray("authorThumbnails"); + return InvidiousParsingHelper.getThumbnailUrl(thumbnails); + } + + @Override + public String getUploaderUrl() { + return baseUrl + "/channel/" + json.getString("authorId"); + } + + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public String getUploaderAvatarUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + + @Override + public long getStreamCount() { + final Number number = json.getNumber("videoCount"); + return number == null ? -1 : number.longValue(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return getPage(getPage(1)); + } + + @Override + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + if (Integer.parseInt(page.getId()) != 1) { + final Response rp = NewPipe.getDownloader().get(page.getUrl()); + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(rp, page.getUrl()); + } + + final JsonArray videos = json.getArray("videos"); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + videos.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(commentObj -> new InvidiousStreamInfoItemExtractor(commentObj, baseUrl)) + .forEach(collector::commit); + + final Page nextPage = videos.size() < 99 + // max number of items per page + ? null + : getPage(Integer.parseInt(page.getId()) + 1); + + + return new InfoItemsPage<>(collector, nextPage); + } + + public Page getPage(final int page) throws ParsingException { + return InvidiousParsingHelper.getPage(baseUrl + "/api/v1/playlists/" + + getId(), page); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/playlists/" + getId(); + + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return json.getString("title"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistInfoItemExtractor.java new file mode 100644 index 0000000000..e84a74e17b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistInfoItemExtractor.java @@ -0,0 +1,48 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; + +public class InvidiousPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + + private final String baseUrl; + private final JsonObject json; + + public InvidiousPlaylistInfoItemExtractor( + final JsonObject json, + final String baseUrl + ) { + this.json = json; + this.baseUrl = baseUrl; + } + + @Override + public String getName() { + return json.getString("title"); + } + + @Override + public String getUrl() { + return baseUrl + "/playlist?list=" + json.getString("playlistId"); + } + + @Override + public String getThumbnailUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("videos") + .getObject(0) + .getArray("videoThumbnails") + ); + } + + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public long getStreamCount() { + return json.getLong("videoCount"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java new file mode 100644 index 0000000000..c316b8df48 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java @@ -0,0 +1,92 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; + +import java.io.IOException; + +import javax.annotation.Nonnull; + +/** + * Search extractor for Invidious. + *
+ * API-Docs: https://docs.invidious.io/api/#get-apiv1search + */ +public class InvidiousSearchExtractor extends SearchExtractor { + + private final String baseUrl; + private JsonArray results; + + public InvidiousSearchExtractor( + final InvidiousService service, + final SearchQueryHandler linkHandler + ) { + super(service, linkHandler); + baseUrl = service.getInstance().getUrl(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return getPage(new Page(getUrl())); + } + + @Override + public InfoItemsPage getPage( + final Page page + ) throws IOException, ExtractionException { + int pageNumber = 1; + try { + if (page.getId() != null) { + pageNumber = Integer.parseInt(page.getId()); + } + } catch (final NumberFormatException ignored) { + // Use default + } + + final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); + results.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .forEach(o -> collectStreamsFrom(collector, o)); + + final Page nextPage = new Page(getUrl() + "&page=" + (pageNumber + 1)); + return new InfoItemsPage<>(collector, nextPage); + } + + @Override + public void onFetchPage( + @Nonnull final Downloader downloader + ) throws IOException, ExtractionException { + final Response response = downloader.get(getUrl()); + results = InvidiousParsingHelper.getValidJsonArrayFromResponse(response, getUrl()); + } + + private void collectStreamsFrom(final MultiInfoItemsCollector collector, final JsonObject json) { + final String type = json.getString("type"); + + switch (type) { + case "video": + collector.commit(new InvidiousStreamInfoItemExtractor(json, baseUrl)); + break; + case "playlist": + collector.commit(new InvidiousPlaylistInfoItemExtractor(json, baseUrl)); + break; + case "channel": + collector.commit(new InvidiousChannelInfoItemExtractor(json, baseUrl)); + break; + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java new file mode 100644 index 0000000000..17949e82e6 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java @@ -0,0 +1,333 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import static org.schabi.newpipe.extractor.utils.Utils.isBlank; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.services.youtube.shared.ItagItem; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class InvidiousStreamExtractor extends StreamExtractor { + + private final String baseUrl; + private JsonObject json; + + public InvidiousStreamExtractor(final InvidiousService service, final LinkHandler linkHandler) { + super(service, linkHandler); + baseUrl = service.getInstance().getUrl(); + } + + @Override + public void onFetchPage( + @Nonnull final Downloader downloader + ) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/videos/" + getId() + + "?region=" + getExtractorContentCountry().getCountryCode(); + + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Nullable + @Override + public String getTextualUploadDate() { + // Note: Depends on instance localization + return json.getString("publishedText"); + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return InvidiousParsingHelper.getUploadDateFromEpochTime( + json.getNumber("published").longValue()); + } + + @Nonnull + @Override + public String getThumbnailUrl() { + final JsonArray thumbnail = json.getArray("authorThumbnails"); + return InvidiousParsingHelper.getThumbnailUrl(thumbnail); + } + + @Nonnull + @Override + public Description getDescription() { + final String descriptionHtml = json.getString("descriptionHtml"); + if (!isBlank(descriptionHtml) || "

".equals(descriptionHtml)) { + return new Description(descriptionHtml, Description.HTML); + } + + return new Description(json.getString("description"), Description.PLAIN_TEXT); + } + + @Override + public int getAgeLimit() { + final boolean isFamilyFriendly = json.getBoolean("isFamilyFriendly"); + return isFamilyFriendly ? NO_AGE_LIMIT : 18; + } + + @Override + public long getLength() { + return json.getNumber("lengthSeconds").longValue(); + } + + @Override + public long getTimeStamp() throws ParsingException { + return getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + // from YouTubeStreamExtractor + } + + @Override + public long getViewCount() { + return json.getNumber("viewCount").longValue(); + } + + @Override + public long getLikeCount() { + return json.getNumber("likeCount").longValue(); + } + + @Override + public long getDislikeCount() { + return json.getNumber("dislikeCount").longValue(); + } + + @Nonnull + @Override + public String getUploaderUrl() { + return baseUrl + json.getString("authorUrl"); + } + + @Nonnull + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Nonnull + @Override + public String getUploaderAvatarUrl() { + final JsonArray avatars = json.getArray("authorThumbnails"); + return InvidiousParsingHelper.getThumbnailUrl(avatars); + } + + @Nonnull + @Override + public String getDashMpdUrl() { + return baseUrl + json.getString("dashUrl"); + } + + @Nonnull + @Override + public String getHlsUrl() { + final String hlsUrl = json.getString("hlsUrl"); + return hlsUrl != null ? hlsUrl : ""; + } + + @Override + public List getAudioStreams() throws ExtractionException { + return getStreams( + getItags("adaptiveFormats", ItagItem.ItagType.AUDIO), + AudioStream::new + ); + } + + @Override + public List getVideoStreams() throws ExtractionException { + return getStreams( + getItags("formatStreams", ItagItem.ItagType.VIDEO), + (url, itag) -> new VideoStream(url, false, itag) + ); + } + + @Override + public List getVideoOnlyStreams() throws ExtractionException { + return getStreams( + getItags("adaptiveFormats", ItagItem.ItagType.VIDEO_ONLY), + (url, itag) -> new VideoStream(url, true, itag) + ); + } + + private List getStreams( + final Map itags, + final BiFunction newStreamCreator + ) throws ExtractionException { + final List streams = new ArrayList<>(); + try { + for (final Map.Entry entry : itags.entrySet()) { + final T videoStream = newStreamCreator.apply(entry.getKey(), entry.getValue()); + if (!Stream.containSimilarStream(videoStream, streams)) { + streams.add(videoStream); + } + } + } catch (final Exception e) { + throw new ParsingException("Could not get video only streams", e); + } + + return streams; + } + + private Map getItags( + final String streamingDataKey, + final ItagItem.ItagType itagTypeWanted + ) throws ParsingException { + final JsonArray formats = json.getArray(streamingDataKey); + + final Map urlAndItags = new LinkedHashMap<>(); + for (final Object o : formats) { + final JsonObject formatData = (JsonObject) o; + final int itag = Integer.parseInt(formatData.getString("itag")); + + if (ItagItem.isSupported(itag)) { + final ItagItem itagItem = ItagItem.getItag(itag); + if (itagItem.itagType == itagTypeWanted) { + urlAndItags.put(formatData.getString("url"), itagItem); + } + } + } + + return urlAndItags; + } + + @Nonnull + @Override + public List getSubtitlesDefault() { + return getSubtitles(MediaFormat.VTT); + } + + @Nonnull + @Override + public List getSubtitles(final MediaFormat format) { + return json.getArray("captions").stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(obj -> obj.has("language_code")) // Prevent NPE below + .map(obj -> { + final String languageCode = obj.getString("language_code"); + return new SubtitlesStream( + format, + languageCode, + baseUrl + obj.getString("url"), + languageCode.contains("(auto-generated)")); + }) + .collect(Collectors.toList()); + } + + @Override + public StreamType getStreamType() { + if (json.getBoolean("liveNow")) { + return StreamType.LIVE_STREAM; + } + return StreamType.VIDEO_STREAM; + } + + @Nullable + @Override + public StreamInfoItemsCollector getRelatedItems() throws IOException, ExtractionException { + try { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + json.getArray("recommendedVideos").stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .forEach(o -> new InvidiousStreamInfoItemExtractor(o, baseUrl)); + + return collector; + } catch (final Exception e) { + throw new ParsingException("Could not get related videos", e); + } + } + + @Nonnull + @Override + public Privacy getPrivacy() { + return json.getBoolean("isListed") ? Privacy.PUBLIC : Privacy.UNLISTED; + } + + @Nonnull + @Override + public String getCategory() { + return json.getString("genre"); + } + + @Nonnull + @Override + public List getTags() { + return JsonUtils.getStringListFromJsonArray(json.getArray("keywords")); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return json.getString("title"); + } + + @Nonnull + @Override + public List getFrames() throws ExtractionException { + return json.getArray("storyboards").stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(storyboard -> { + final int durationPerFrames = storyboard.getInt("interval"); + if (durationPerFrames == 0) { + return null; + } + + final String templateUrl = storyboard.getString("templateUrl"); + final List urls = + IntStream.range(0, storyboard.getInt("storyboardCount")) + .mapToObj(i -> templateUrl.replace("M$M", "M" + i)) + .collect(Collectors.toList()); + + return new Frameset( + urls, + storyboard.getInt("width"), + storyboard.getInt("height"), + storyboard.getInt("count"), + durationPerFrames, + storyboard.getInt("storyboardWidth"), + storyboard.getInt("storyboardHeight") + ); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java new file mode 100644 index 0000000000..5f968c129c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java @@ -0,0 +1,100 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getUploadDateFromEpochTime; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; + + +public class InvidiousStreamInfoItemExtractor implements StreamInfoItemExtractor { + + private final String baseUrl; + private final JsonObject json; + + public InvidiousStreamInfoItemExtractor( + final JsonObject json, + final String baseUrl + ) { + this.json = json; + this.baseUrl = baseUrl; + } + + @Override + public StreamType getStreamType() { + if (json.getBoolean("liveNow")) { + return StreamType.LIVE_STREAM; + } + return StreamType.VIDEO_STREAM; + } + + @Override + public boolean isAd() throws ParsingException { + return json.getBoolean("premium") + || json.getBoolean("paid"); + } + + @Override + public long getDuration() { + final Number number = json.getNumber("lengthSeconds"); + return number == null ? -1 : number.longValue(); + } + + @Override + public long getViewCount() { + return json.getLong("viewCount", -1); + } + + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public String getUploaderUrl() { + final String authorUrl = json.getString("authorUrl"); + if (authorUrl != null) { + return baseUrl + authorUrl; + } + return baseUrl + "/channel/" + json.getString("authorId"); + } + + @Nullable + @Override + public String getTextualUploadDate() { + return json.getString("publishedText"); + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + final Number epochTime = json.getNumber("published"); + if (epochTime != null) { + return getUploadDateFromEpochTime(epochTime.longValue()); + } + + return null; + } + + @Override + public String getName() { + return json.getString("title"); + } + + @Override + public String getUrl() throws ParsingException { + return baseUrl + "/watch?v=" + json.getString("videoId"); + } + + @Override + public String getThumbnailUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("videoThumbnails")); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSuggestionExtractor.java new file mode 100644 index 0000000000..1aff861280 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSuggestionExtractor.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import java.io.IOException; +import java.util.List; + +public class InvidiousSuggestionExtractor extends SuggestionExtractor { + + private final String baseUrl; + + public InvidiousSuggestionExtractor(final InvidiousService service) { + super(service); + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public List suggestionList(final String query) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/search/suggestions?q=" + query; + final Downloader dl = NewPipe.getDownloader(); + final Response response = dl.get(apiUrl); + + final JsonObject json = + InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + + return JsonUtils.getStringListFromJsonArray(json.getArray("suggestions")); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java new file mode 100644 index 0000000000..b8bb28eef0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java @@ -0,0 +1,77 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler.InvidiousTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import java.io.IOException; + +import javax.annotation.Nonnull; + +public class InvidiousTrendingExtractor extends KioskExtractor { + + private final String baseUrl; + private JsonArray videos; + + public InvidiousTrendingExtractor( + final InvidiousService streamingService, + final ListLinkHandler linkHandler, + final String kioskId + ) { + super(streamingService, linkHandler, kioskId); + baseUrl = streamingService.getInstance().getUrl(); + } + + @Override + public void onFetchPage( + @Nonnull final Downloader downloader + ) throws IOException, ExtractionException { + + String apiUrl = getUrl(); + if (getId().equals(InvidiousTrendingLinkHandlerFactory.KIOSK_TRENDING)) { + // The trending endpoint accepts regions + // see https://docs.invidious.io/api/#get-apiv1trending + apiUrl += "?region=" + getExtractorContentCountry(); + } + + final Response response = downloader.get(apiUrl); + videos = InvidiousParsingHelper.getValidJsonArrayFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + videos.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(o -> new InvidiousStreamInfoItemExtractor(o, baseUrl)) + .forEach(collector::commit); + + return new InfoItemsPage<>(collector, null); + } + + @Override + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + return InfoItemsPage.emptyPage(); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return getId(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousChannelLinkHandlerFactory.java new file mode 100644 index 0000000000..ade00ddda3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousChannelLinkHandlerFactory.java @@ -0,0 +1,38 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeChannelLinkHandlerFactory; + +import java.util.List; + + +public class InvidiousChannelLinkHandlerFactory extends YoutubeLikeChannelLinkHandlerFactory { + + protected final String baseUrl; + + public InvidiousChannelLinkHandlerFactory(final InvidiousService service) { + super(); + baseUrl = service.getInstance().getUrl(); + } + + /** + * Returns URL to channel from an ID + * + * @param id Channel ID including e.g. 'channel/' + * @return URL to channel + */ + @Override + public String getUrl(final String id, + final List contentFilters, + final String searchFilter) { + return baseUrl + "/" + id; + } + + @Override + public String getId(final String url) throws ParsingException { + final String id = super.getId(url); + + return "channel/" + id.split("/")[1]; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousCommentLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousCommentLinkHandlerFactory.java new file mode 100644 index 0000000000..82758a365c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousCommentLinkHandlerFactory.java @@ -0,0 +1,12 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeStreamLinkHandlerFactory; + +public class InvidiousCommentLinkHandlerFactory extends YoutubeLikeCommentsLinkHandlerFactory { + + public InvidiousCommentLinkHandlerFactory(final InvidiousService service) { + super((YoutubeLikeStreamLinkHandlerFactory) service.getStreamLHFactory()); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java new file mode 100644 index 0000000000..7bf1f5a629 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java @@ -0,0 +1,26 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikePlaylistLinkHandlerFactory; + +import java.util.List; + +public class InvidiousPlaylistLinkHandlerFactory extends YoutubeLikePlaylistLinkHandlerFactory { + + protected final String baseUrl; + + public InvidiousPlaylistLinkHandlerFactory(final InvidiousService service) { + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public String getUrl(final String id, final List contentFilters, + final String sortFilter) { + return baseUrl +"/playlist?list=" + id; + } + + @Override + protected String getMixUrl(final String videoID, final String listID) { + return baseUrl + "/watch?v=" + videoID + "&list=" + listID; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousSearchQueryHandlerFactory.java new file mode 100644 index 0000000000..95c6061d20 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousSearchQueryHandlerFactory.java @@ -0,0 +1,58 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; + +import java.util.List; + +public class InvidiousSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + + public static final String ALL = "all"; + public static final String VIDEOS = "videos"; + public static final String CHANNELS = "channels"; + public static final String PLAYLISTS = "playlists"; + + protected final String baseUrl; + + public InvidiousSearchQueryHandlerFactory(final InvidiousService service) { + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public String getUrl( + final String query, + final List contentFilter, + final String sortFilter + ) throws ParsingException { + final String url = baseUrl + "/api/v1/search?q=" + query; + + if (!contentFilter.isEmpty()) { + switch (contentFilter.get(0)) { + case VIDEOS: + return url; // + "&type=video" it's the default type provided by Invidious + case CHANNELS: + return url + "&type=channel"; + case PLAYLISTS: + return url + "&type=playlist"; + case ALL: + default: + break; + } + } + + return url + "&type=all"; + } + + + @Override + public String[] getAvailableContentFilter() { + return new String[]{ + ALL, + VIDEOS, + CHANNELS, + PLAYLISTS + }; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java new file mode 100644 index 0000000000..fc50f4be0f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java @@ -0,0 +1,19 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeStreamLinkHandlerFactory; + +public class InvidiousStreamLinkHandlerFactory extends YoutubeLikeStreamLinkHandlerFactory { + + protected final String baseUrl; + + public InvidiousStreamLinkHandlerFactory(final InvidiousService service) { + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public String getUrl(final String id) throws ParsingException { + return baseUrl + "/watch?v=" + id; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousTrendingLinkHandlerFactory.java new file mode 100644 index 0000000000..c8fef271a0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousTrendingLinkHandlerFactory.java @@ -0,0 +1,64 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public final class InvidiousTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + public static final Map KIOSK_MAP; + public static final String KIOSK_TRENDING = "Trending"; + public static final String KIOSK_POPULAR = "Popular"; + + static { + final Map map = new HashMap<>(); + // https://docs.invidious.io/api/#get-apiv1trending + map.put(KIOSK_TRENDING, "%s/api/v1/trending"); + // https://docs.invidious.io/api/#get-apiv1popular + map.put(KIOSK_POPULAR, "%s/api/v1/popular"); + + KIOSK_MAP = Collections.unmodifiableMap(map); + } + + protected final String baseUrl; + + public InvidiousTrendingLinkHandlerFactory(final InvidiousService service) { + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public String getUrl(final String id, + final List contentFilters, + final String sortFilter) { + return getUrl(id, contentFilters, sortFilter, baseUrl); + } + + @Override + public String getUrl(final String id, + final List contentFilters, + final String sortFilter, + final String baseUrl) { + return String.format(KIOSK_MAP.get(id), baseUrl); + } + + @Override + public String getId(final String url) throws ParsingException { + final String cleanUrl = url.replace(baseUrl, "%s"); + if (cleanUrl.contains("/trending")) { + return KIOSK_TRENDING; + } else if (cleanUrl.contains("/popular")) { + return KIOSK_POPULAR; + } + throw new ParsingException("no id found for this url"); + } + + @Override + public boolean onAcceptUrl(final String url) { + return url.contains("/trending") + || url.contains("/popular"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java deleted file mode 100644 index a15801b229..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; - -import org.schabi.newpipe.extractor.exceptions.FoundAdException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; - -import java.util.List; - -public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { - - private static final YoutubeCommentsLinkHandlerFactory INSTANCE - = new YoutubeCommentsLinkHandlerFactory(); - - private YoutubeCommentsLinkHandlerFactory() { - } - - public static YoutubeCommentsLinkHandlerFactory getInstance() { - return INSTANCE; - } - - @Override - public String getUrl(final String id) { - return "https://www.youtube.com/watch?v=" + id; - } - - @Override - public String getId(final String urlString) throws ParsingException, IllegalArgumentException { - // we need the same id, avoids duplicate code - return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString); - } - - @Override - public boolean onAcceptUrl(final String url) throws FoundAdException { - try { - getId(url); - return true; - } catch (final FoundAdException fe) { - throw fe; - } catch (final ParsingException e) { - return false; - } - } - - @Override - public String getUrl(final String id, - final List contentFilter, - final String sortFilter) throws ParsingException { - return getUrl(id); - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java deleted file mode 100644 index d302575306..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ /dev/null @@ -1,258 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isHooktubeURL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isY2ubeURL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeServiceURL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL; - -import org.schabi.newpipe.extractor.exceptions.FoundAdException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.utils.Utils; - -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.annotation.Nullable; - -/* - * Created by Christian Schabesberger on 02.02.16. - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeStreamLinkHandlerFactory.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { - - private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN - = Pattern.compile("^([a-zA-Z0-9_-]{11})"); - private static final YoutubeStreamLinkHandlerFactory INSTANCE - = new YoutubeStreamLinkHandlerFactory(); - private static final List SUBPATHS - = Arrays.asList("embed/", "shorts/", "watch/", "v/", "w/"); - - private YoutubeStreamLinkHandlerFactory() { - } - - public static YoutubeStreamLinkHandlerFactory getInstance() { - return INSTANCE; - } - - @Nullable - private static String extractId(@Nullable final String id) { - if (id != null) { - final Matcher m = YOUTUBE_VIDEO_ID_REGEX_PATTERN.matcher(id); - return m.find() ? m.group(1) : null; - } - return null; - } - - private static String assertIsId(@Nullable final String id) throws ParsingException { - final String extractedId = extractId(id); - if (extractedId != null) { - return extractedId; - } else { - throw new ParsingException("The given string is not a Youtube-Video-ID"); - } - } - - @Override - public String getUrl(final String id) { - return "https://www.youtube.com/watch?v=" + id; - } - - @SuppressWarnings("AvoidNestedBlocks") - @Override - public String getId(final String theUrlString) - throws ParsingException, IllegalArgumentException { - String urlString = theUrlString; - try { - final URI uri = new URI(urlString); - final String scheme = uri.getScheme(); - - if (scheme != null - && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) { - final String schemeSpecificPart = uri.getSchemeSpecificPart(); - if (schemeSpecificPart.startsWith("//")) { - final String extractedId = extractId(schemeSpecificPart.substring(2)); - if (extractedId != null) { - return extractedId; - } - - urlString = "https:" + schemeSpecificPart; - } else { - return assertIsId(schemeSpecificPart); - } - } - } catch (final URISyntaxException ignored) { - } - - final URL url; - try { - url = Utils.stringToURL(urlString); - } catch (final MalformedURLException e) { - throw new IllegalArgumentException("The given URL is not valid"); - } - - final String host = url.getHost(); - String path = url.getPath(); - // remove leading "/" of URL-path if URL-path is given - if (!path.isEmpty()) { - path = path.substring(1); - } - - if (!Utils.isHTTP(url) || !(isYoutubeURL(url) || isYoutubeServiceURL(url) - || isHooktubeURL(url) || isInvidioURL(url) || isY2ubeURL(url))) { - if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) { - throw new FoundAdException("Error found ad: " + urlString); - } - - throw new ParsingException("The url is not a Youtube-URL"); - } - - if (YoutubePlaylistLinkHandlerFactory.getInstance().acceptUrl(urlString)) { - throw new ParsingException("Error no suitable url: " + urlString); - } - - // Using uppercase instead of lowercase, because toLowercase replaces some unicode - // characters with their lowercase ASCII equivalent. Using toLowercase could result in - // faultily matching unicode urls. - switch (host.toUpperCase()) { - case "WWW.YOUTUBE-NOCOOKIE.COM": { - if (path.startsWith("embed/")) { - return assertIsId(path.substring(6)); - } - break; - } - - case "YOUTUBE.COM": - case "WWW.YOUTUBE.COM": - case "M.YOUTUBE.COM": - case "MUSIC.YOUTUBE.COM": { - if (path.equals("attribution_link")) { - final String uQueryValue = Utils.getQueryValue(url, "u"); - - final URL decodedURL; - try { - decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue); - } catch (final MalformedURLException e) { - throw new ParsingException("Error no suitable url: " + urlString); - } - - final String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); - return assertIsId(viewQueryValue); - } - - final String maybeId = getIdFromSubpathsInPath(path); - if (maybeId != null) { - return maybeId; - } - - final String viewQueryValue = Utils.getQueryValue(url, "v"); - return assertIsId(viewQueryValue); - } - - case "Y2U.BE": - case "YOUTU.BE": { - final String viewQueryValue = Utils.getQueryValue(url, "v"); - if (viewQueryValue != null) { - return assertIsId(viewQueryValue); - } - - return assertIsId(path); - } - - case "HOOKTUBE.COM": - case "INVIDIO.US": - case "DEV.INVIDIO.US": - case "WWW.INVIDIO.US": - case "REDIRECT.INVIDIOUS.IO": - case "INVIDIOUS.SNOPYTA.ORG": - case "YEWTU.BE": - case "TUBE.CONNECT.CAFE": - case "TUBUS.EDUVID.ORG": - case "INVIDIOUS.KAVIN.ROCKS": - case "INVIDIOUS-US.KAVIN.ROCKS": - case "PIPED.KAVIN.ROCKS": - case "INVIDIOUS.SITE": - case "VID.MINT.LGBT": - case "INVIDIOU.SITE": - case "INVIDIOUS.FDN.FR": - case "INVIDIOUS.048596.XYZ": - case "INVIDIOUS.ZEE.LI": - case "VID.PUFFYAN.US": - case "YTPRIVATE.COM": - case "INVIDIOUS.NAMAZSO.EU": - case "INVIDIOUS.SILKKY.CLOUD": - case "INVIDIOUS.EXONIP.DE": - case "INV.RIVERSIDE.ROCKS": - case "INVIDIOUS.BLAMEFRAN.NET": - case "INVIDIOUS.MOOMOO.ME": - case "YTB.TROM.TF": - case "YT.CYBERHOST.UK": - case "Y.COM.CM": { // code-block for hooktube.com and Invidious instances - if (path.equals("watch")) { - final String viewQueryValue = Utils.getQueryValue(url, "v"); - if (viewQueryValue != null) { - return assertIsId(viewQueryValue); - } - } - final String maybeId = getIdFromSubpathsInPath(path); - if (maybeId != null) { - return maybeId; - } - - final String viewQueryValue = Utils.getQueryValue(url, "v"); - if (viewQueryValue != null) { - return assertIsId(viewQueryValue); - } - - return assertIsId(path); - } - } - - throw new ParsingException("Error no suitable url: " + urlString); - } - - @Override - public boolean onAcceptUrl(final String url) throws FoundAdException { - try { - getId(url); - return true; - } catch (final FoundAdException fe) { - throw fe; - } catch (final ParsingException e) { - return false; - } - } - - private String getIdFromSubpathsInPath(final String path) throws ParsingException { - for (final String subpath : SUBPATHS) { - if (path.startsWith(subpath)) { - final String id = path.substring(subpath.length()); - return assertIsId(id); - } - } - return null; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/ItagItem.java similarity index 96% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/ItagItem.java index 55d3082a23..645d10565a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/ItagItem.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.shared; import static org.schabi.newpipe.extractor.MediaFormat.M4A; import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4; @@ -6,9 +6,9 @@ import static org.schabi.newpipe.extractor.MediaFormat.WEBMA; import static org.schabi.newpipe.extractor.MediaFormat.WEBMA_OPUS; import static org.schabi.newpipe.extractor.MediaFormat.v3GPP; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY; +import static org.schabi.newpipe.extractor.services.youtube.shared.ItagItem.ItagType.AUDIO; +import static org.schabi.newpipe.extractor.services.youtube.shared.ItagItem.ItagType.VIDEO; +import static org.schabi.newpipe.extractor.services.youtube.shared.ItagItem.ItagType.VIDEO_ONLY; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubePlaylistHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubePlaylistHelper.java new file mode 100644 index 0000000000..2920a91552 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubePlaylistHelper.java @@ -0,0 +1,163 @@ +package org.schabi.newpipe.extractor.services.youtube.shared; + +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfo; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.MalformedURLException; + +import javax.annotation.Nonnull; + +public final class YoutubePlaylistHelper { + private YoutubePlaylistHelper() { + // No impl + } + + + /** + * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) + * Ids from a YouTube Mix start with "RD" + * + * @param playlistId the playlist id + * @return Whether given id belongs to a YouTube Mix + */ + public static boolean isYoutubeMixId(@Nonnull final String playlistId) { + return playlistId.startsWith("RD") + && !isYoutubeMusicMixId(playlistId); + } + + /** + * Checks if the given playlist id is a YouTube My Mix (auto-generated playlist) + * Ids from a YouTube My Mix start with "RDMM" + * + * @param playlistId the playlist id + * @return Whether given id belongs to a YouTube My Mix + */ + public static boolean isYoutubeMyMixId(@Nonnull final String playlistId) { + return playlistId.startsWith("RDMM"); + } + + /** + * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) + * Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK" + * + * @param playlistId the playlist id + * @return Whether given id belongs to a YouTube Music Mix + */ + public static boolean isYoutubeMusicMixId(@Nonnull final String playlistId) { + return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK"); + } + + /** + * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) + * Ids from a YouTube channel Mix start with "RDCM" + * + * @return Whether given id belongs to a YouTube Channel Mix + */ + public static boolean isYoutubeChannelMixId(@Nonnull final String playlistId) { + return playlistId.startsWith("RDCM"); + } + + /** + * Checks if the given playlist id is a YouTube Genre Mix (auto-generated playlist) + * Ids from a YouTube Genre Mix start with "RDGMEM" + * + * @return Whether given id belongs to a YouTube Genre Mix + */ + public static boolean isYoutubeGenreMixId(@Nonnull final String playlistId) { + return playlistId.startsWith("RDGMEM"); + } + + /** + * @param playlistId the playlist id to parse + * @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistId (mix playlist + * types included) + * @throws ParsingException if the playlistId is null or empty, if the playlistId is not a mix, + * if it is a mix but it's not based on a specific stream (this is the + * case for channel or genre mixes) + */ + @Nonnull + public static String extractVideoIdFromMixId(final String playlistId) + throws ParsingException { + if (isNullOrEmpty(playlistId)) { + throw new ParsingException("Video id could not be determined from empty playlist id"); + + } else if (isYoutubeMyMixId(playlistId)) { + return playlistId.substring(4); + + } else if (isYoutubeMusicMixId(playlistId)) { + return playlistId.substring(6); + + } else if (isYoutubeChannelMixId(playlistId)) { + // Channel mixes are of the form RMCM{channelId}, so videoId can't be determined + throw new ParsingException("Video id could not be determined from channel mix id: " + + playlistId); + + } else if (isYoutubeGenreMixId(playlistId)) { + // Genre mixes are of the form RDGMEM{garbage}, so videoId can't be determined + throw new ParsingException("Video id could not be determined from genre mix id: " + + playlistId); + + } else if (isYoutubeMixId(playlistId)) { // normal mix + if (playlistId.length() != 13) { + // Stream YouTube mixes are of the form RD{videoId}, but if videoId is not exactly + // 11 characters then it can't be a video id, hence we are dealing with a different + // type of mix (e.g. genre mixes handled above, of the form RDGMEM{garbage}) + throw new ParsingException("Video id could not be determined from mix id: " + + playlistId); + } + return playlistId.substring(2); + + } else { // not a mix + throw new ParsingException("Video id could not be determined from playlist id: " + + playlistId); + } + } + + /** + * @param playlistId the playlist id to parse + * @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistId (mix playlist + * types included) + * @throws ParsingException if the playlistId is null or empty + */ + @Nonnull + public static PlaylistInfo.PlaylistType extractPlaylistTypeFromPlaylistId( + final String playlistId) throws ParsingException { + if (isNullOrEmpty(playlistId)) { + throw new ParsingException("Could not extract playlist type from empty playlist id"); + } else if (isYoutubeMusicMixId(playlistId)) { + return PlaylistInfo.PlaylistType.MIX_MUSIC; + } else if (isYoutubeChannelMixId(playlistId)) { + return PlaylistInfo.PlaylistType.MIX_CHANNEL; + } else if (isYoutubeGenreMixId(playlistId)) { + return PlaylistInfo.PlaylistType.MIX_GENRE; + } else if (isYoutubeMixId(playlistId)) { // normal mix + // Either a normal mix based on a stream, or a "my mix" (still based on a stream). + // NOTE: if YouTube introduces even more types of mixes that still start with RD, + // they will default to this, even though they might not be based on a stream. + return PlaylistInfo.PlaylistType.MIX_STREAM; + } else { + // not a known type of mix: just consider it a normal playlist + return PlaylistInfo.PlaylistType.NORMAL; + } + } + + /** + * @param playlistUrl the playlist url to parse + * @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistUrl's list param + * (mix playlist types included) + * @throws ParsingException if the playlistUrl is malformed, if has no list param or if the list + * param is empty + */ + public static PlaylistInfo.PlaylistType extractPlaylistTypeFromPlaylistUrl( + final String playlistUrl) throws ParsingException { + try { + return extractPlaylistTypeFromPlaylistId( + Utils.getQueryValue(Utils.stringToURL(playlistUrl), "list")); + } catch (final MalformedURLException e) { + throw new ParsingException("Could not extract playlist type from malformed url", e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java new file mode 100644 index 0000000000..46ce44e380 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java @@ -0,0 +1,89 @@ +package org.schabi.newpipe.extractor.services.youtube.shared; + +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import javax.annotation.Nonnull; + +public final class YoutubeUrlHelper { + private YoutubeUrlHelper() { + // No impl + } + + public static boolean isGoogleURL(final String url) { + final String cachedUrl = extractCachedUrlIfNeeded(url); + try { + final URL u = new URL(cachedUrl); + final String host = Utils.removeMAndWWWFromHost(u.getHost()); + return host.startsWith("google."); + } catch (final MalformedURLException e) { + return false; + } + } + + public static boolean isYoutubeURL(@Nonnull final URL url) { + final String host = Utils.removeMAndWWWFromHost(url.getHost()); + return host.equalsIgnoreCase("youtube.com") + || host.equalsIgnoreCase("music.youtube.com"); + } + + public static boolean isYoutubeServiceURL(@Nonnull final URL url) { + final String host = Utils.removeMAndWWWFromHost(url.getHost()); + return host.equalsIgnoreCase("youtube-nocookie.com") + || host.equalsIgnoreCase("youtu.be"); + } + + public static boolean isHooktubeURL(@Nonnull final URL url) { + final String host = Utils.removeMAndWWWFromHost(url.getHost()); + return host.equalsIgnoreCase("hooktube.com"); + } + + public static boolean isInvidioURL(@Nonnull final URL url) { + // Valid, working instances as of 2022-03 + final Set invInstances = new HashSet<>(Arrays.asList( + "invidious.snopyta.org", + "yewtu.be", + "tube.connect.cafe", + "tubus.eduvid.org", + "invidious.kavin.rocks", + "invidious-us.kavin.rocks", + "piped.kavin.rocks", // Not sure why this is here - it's not an invidious instance + "invidious.fdn.fr", + "invidious.zee.li", + "vid.puffyan.us", + "invidious.namazso.eu", + "inv.riverside.rocks", + "ytb.trom.tf", + "invidio.xamh.de", + "y.com.cm" + )); + final String host = Utils.removeMAndWWWFromUrl(url.getHost()).toLowerCase(); + return invInstances.contains(host); + } + + public static boolean isY2ubeURL(@Nonnull final URL url) { + return Utils.removeMAndWWWFromUrl(url.getHost()).equalsIgnoreCase("y2u.be"); + } + + /** + * Sometimes, YouTube provides URLs which use Google's cache. They look like + * {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL} + * + * @param url the URL which might refer to the Google's webcache + * @return the URL which is referring to the original site + */ + public static String extractCachedUrlIfNeeded(final String url) { + if (url == null) { + return null; + } + if (url.contains("webcache.googleusercontent.com")) { + return url.split("cache:")[1]; + } + return url; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/extractors/YoutubeTakeoutSubscriptionExtractor.java similarity index 90% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/extractors/YoutubeTakeoutSubscriptionExtractor.java index 403151f075..1731c9ac3d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/extractors/YoutubeTakeoutSubscriptionExtractor.java @@ -1,4 +1,6 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.shared.extractors; + +import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -6,9 +8,10 @@ import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.services.youtube.YoutubeService; +import org.schabi.newpipe.extractor.services.youtube.YoutubeLikeStreamingService; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.BufferedReader; import java.io.IOException; @@ -22,15 +25,13 @@ import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM; - /** * Extract subscriptions from a Google takeout export */ -public class YoutubeSubscriptionExtractor extends SubscriptionExtractor { +public class YoutubeTakeoutSubscriptionExtractor extends SubscriptionExtractor { private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/"; - public YoutubeSubscriptionExtractor(final YoutubeService youtubeService) { + public YoutubeTakeoutSubscriptionExtractor(final YoutubeLikeStreamingService youtubeService) { super(youtubeService, Collections.singletonList(INPUT_STREAM)); } @@ -101,7 +102,7 @@ public List fromJsonInputStream(@Nonnull final InputStream con public List fromZipInputStream(@Nonnull final InputStream contentInputStream) throws ExtractionException { - try (ZipInputStream zipInputStream = new ZipInputStream(contentInputStream)) { + try (final ZipInputStream zipInputStream = new ZipInputStream(contentInputStream)) { ZipEntry zipEntry; while ((zipEntry = zipInputStream.getNextEntry()) != null) { if (zipEntry.getName().toLowerCase().endsWith(".csv")) { @@ -110,7 +111,7 @@ public List fromZipInputStream(@Nonnull final InputStream cont // Return it only if it has items (it exits early if it's the wrong file // format), otherwise try the next file - if (csvItems.size() > 0) { + if (!csvItems.isEmpty()) { return csvItems; } } catch (final ExtractionException e) { @@ -142,7 +143,7 @@ public List fromCsvInputStream(@Nonnull final InputStream cont int currentLine = 0; String line = ""; - try (BufferedReader br = new BufferedReader(new InputStreamReader(contentInputStream))) { + try (final BufferedReader br = new BufferedReader(new InputStreamReader(contentInputStream))) { final List subscriptionItems = new ArrayList<>(); // ignore header and skip first line @@ -154,7 +155,7 @@ public List fromCsvInputStream(@Nonnull final InputStream cont // Exit early if we've read the first few lines and we haven't added any items // It's likely we're in the wrong file - if (currentLine > 5 && subscriptionItems.size() == 0) { + if (currentLine > 5 && subscriptionItems.isEmpty()) { break; } @@ -179,7 +180,7 @@ public List fromCsvInputStream(@Nonnull final InputStream cont // Channel URL from second entry final String channelUrl = line .substring(i1 + 1, i2) - .replace("http://", "https://"); + .replace(Utils.HTTP, Utils.HTTPS); if (!channelUrl.startsWith(BASE_CHANNEL_URL)) { continue; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeChannelLinkHandlerFactory.java similarity index 51% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeChannelLinkHandlerFactory.java index eed04bf418..dadda5799e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeChannelLinkHandlerFactory.java @@ -1,62 +1,18 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; +package org.schabi.newpipe.extractor.services.youtube.shared.linkHandler; -import java.util.regex.Pattern; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.net.URL; -import java.util.List; - -/* - * Created by Christian Schabesberger on 25.07.16. - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeChannelLinkHandlerFactory.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { +import java.util.regex.Pattern; - private static final YoutubeChannelLinkHandlerFactory INSTANCE - = new YoutubeChannelLinkHandlerFactory(); +public abstract class YoutubeLikeChannelLinkHandlerFactory extends ListLinkHandlerFactory { - private static final Pattern EXCLUDED_SEGMENTS = + protected static final Pattern EXCLUDED_SEGMENTS = Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); - private YoutubeChannelLinkHandlerFactory() { - } - - public static YoutubeChannelLinkHandlerFactory getInstance() { - return INSTANCE; - } - - /** - * Returns URL to channel from an ID - * - * @param id Channel ID including e.g. 'channel/' - * @return URL to channel - */ - @Override - public String getUrl(final String id, - final List contentFilters, - final String searchFilter) { - return "https://www.youtube.com/" + id; - } - /** * Returns true if path conform to * custom short channel URLs like youtube.com/yourcustomname @@ -64,7 +20,7 @@ public String getUrl(final String id, * @param splitPath path segments array * @return true - if value conform to short channel URL, false - not */ - private boolean isCustomShortChannelUrl(final String[] splitPath) { + protected boolean isCustomShortChannelUrl(final String[] splitPath) { return splitPath.length == 1 && !EXCLUDED_SEGMENTS.matcher(splitPath[0]).matches(); } @@ -74,9 +30,9 @@ public String getId(final String url) throws ParsingException { final URL urlObj = Utils.stringToURL(url); String path = urlObj.getPath(); - if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) - || YoutubeParsingHelper.isInvidioURL(urlObj) - || YoutubeParsingHelper.isHooktubeURL(urlObj))) { + if (!Utils.isHTTP(urlObj) || !(YoutubeUrlHelper.isYoutubeURL(urlObj) + || YoutubeUrlHelper.isInvidioURL(urlObj) + || YoutubeUrlHelper.isHooktubeURL(urlObj))) { throw new ParsingException("the URL given is not a Youtube-URL"); } @@ -99,7 +55,7 @@ public String getId(final String url) throws ParsingException { final String id = splitPath[1]; if (id == null || !id.matches("[A-Za-z0-9_-]+")) { - throw new ParsingException("The given id is not a Youtube-Video-ID"); + throw new ParsingException("The given id is not a Youtube-Channel-ID"); } return splitPath[0] + "/" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeCommentsLinkHandlerFactory.java new file mode 100644 index 0000000000..db15ceba5c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeCommentsLinkHandlerFactory.java @@ -0,0 +1,41 @@ +package org.schabi.newpipe.extractor.services.youtube.shared.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import java.util.List; + +public abstract class YoutubeLikeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { + + protected final YoutubeLikeStreamLinkHandlerFactory streamLinkHandlerFactory; + + protected YoutubeLikeCommentsLinkHandlerFactory( + final YoutubeLikeStreamLinkHandlerFactory streamLinkHandlerFactory + ) { + this.streamLinkHandlerFactory = streamLinkHandlerFactory; + } + + @Override + public String getUrl(final String id) throws ParsingException { + return streamLinkHandlerFactory.getUrl(id); + } + + @Override + public String getId(final String urlString) throws ParsingException, IllegalArgumentException { + // we need the same id, avoids duplicate code + return streamLinkHandlerFactory.getId(urlString); + } + + @Override + public boolean onAcceptUrl(final String url) throws FoundAdException { + return streamLinkHandlerFactory.onAcceptUrl(url); + } + + @Override + public String getUrl(final String id, + final List contentFilter, + final String sortFilter) throws ParsingException { + return getUrl(id); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java similarity index 63% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java index fb47d95c02..152947ea3c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java @@ -1,61 +1,46 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; +package org.schabi.newpipe.extractor.services.youtube.shared.linkHandler; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.List; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubePlaylistHelper; +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; import org.schabi.newpipe.extractor.utils.Utils; -public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { - - private static final YoutubePlaylistLinkHandlerFactory INSTANCE = - new YoutubePlaylistLinkHandlerFactory(); - - private YoutubePlaylistLinkHandlerFactory() { - } - - public static YoutubePlaylistLinkHandlerFactory getInstance() { - return INSTANCE; - } +import java.net.MalformedURLException; +import java.net.URL; - @Override - public String getUrl(final String id, final List contentFilters, - final String sortFilter) { - return "https://www.youtube.com/playlist?list=" + id; - } +public abstract class YoutubeLikePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(final String url) throws ParsingException { try { final URL urlObj = Utils.stringToURL(url); - if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) - || YoutubeParsingHelper.isInvidioURL(urlObj))) { - throw new ParsingException("the url given is not a YouTube-URL"); + if (!Utils.isHTTP(urlObj) || !(YoutubeUrlHelper.isYoutubeURL(urlObj) + || YoutubeUrlHelper.isInvidioURL(urlObj))) { + throw new ParsingException("The url given is not a YouTube-URL"); } final String path = urlObj.getPath(); if (!path.equals("/watch") && !path.equals("/playlist")) { - throw new ParsingException("the url given is neither a video nor a playlist URL"); + throw new ParsingException("The url given is neither a video nor a playlist URL"); } final String listID = Utils.getQueryValue(urlObj, "list"); if (listID == null) { - throw new ParsingException("the URL given does not include a playlist"); + throw new ParsingException("The URL given does not include a playlist"); } if (!listID.matches("[a-zA-Z0-9_-]{10,}")) { throw new ParsingException( - "the list-ID given in the URL does not match the list pattern"); + "The list-ID given in the URL does not match the list pattern"); } - if (YoutubeParsingHelper.isYoutubeChannelMixId(listID) + if (YoutubePlaylistHelper.isYoutubeChannelMixId(listID) && Utils.getQueryValue(urlObj, "v") == null) { // Video id can't be determined from the channel mix id. // See YoutubeParsingHelper#extractVideoIdFromMixId @@ -80,9 +65,19 @@ public boolean onAcceptUrl(final String url) { return true; } + /** + * Returns the url for the mix-playlist. + *
+ * Example for Youtube: {@code https://youtube.com/watch?v=videoId&list=playlistId} + * @param videoID id of the video + * @param listID id of the list + * @return url for the mix-playlist + */ + protected abstract String getMixUrl(final String videoID, final String listID); + /** * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is - * like {@code https://youtube.com/watch?v=videoId&list=playlistId} + * like see {@link #getMixUrl(String, String)} *

Otherwise use super

*/ @Override @@ -90,13 +85,12 @@ public ListLinkHandler fromUrl(final String url) throws ParsingException { try { final URL urlObj = Utils.stringToURL(url); final String listID = Utils.getQueryValue(urlObj, "list"); - if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { + if (listID != null && YoutubePlaylistHelper.isYoutubeMixId(listID)) { String videoID = Utils.getQueryValue(urlObj, "v"); if (videoID == null) { - videoID = YoutubeParsingHelper.extractVideoIdFromMixId(listID); + videoID = YoutubePlaylistHelper.extractVideoIdFromMixId(listID); } - final String newUrl = "https://www.youtube.com/watch?v=" + videoID - + "&list=" + listID; + final String newUrl = getMixUrl(videoID, listID); return new ListLinkHandler(new LinkHandler(url, newUrl, listID)); } } catch (final MalformedURLException exception) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java new file mode 100644 index 0000000000..de1e946929 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java @@ -0,0 +1,176 @@ +package org.schabi.newpipe.extractor.services.youtube.shared.linkHandler; + +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isHooktubeURL; +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isInvidioURL; +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isY2ubeURL; +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isYoutubeServiceURL; +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isYoutubeURL; + +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +public abstract class YoutubeLikeStreamLinkHandlerFactory extends LinkHandlerFactory { + + protected static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN + = Pattern.compile("^([a-zA-Z0-9_-]{11})"); + protected static final List SUBPATHS + = Arrays.asList("embed/", "shorts/", "watch/", "v/", "w/"); + + @Nullable + private static String extractId(@Nullable final String id) { + if (id != null) { + final Matcher m = YOUTUBE_VIDEO_ID_REGEX_PATTERN.matcher(id); + return m.find() ? m.group(1) : null; + } + return null; + } + + private static String assertIsId(@Nullable final String id) throws ParsingException { + final String extractedId = extractId(id); + if (extractedId != null) { + return extractedId; + } + throw new ParsingException("The given string is not a Youtube-Video-ID"); + } + + @Override + public String getId(final String theUrlString) + throws ParsingException, IllegalArgumentException { + String urlString = theUrlString; + try { + final URI uri = new URI(urlString); + final String scheme = uri.getScheme(); + + if (scheme != null + && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) { + final String schemeSpecificPart = uri.getSchemeSpecificPart(); + if (!schemeSpecificPart.startsWith("//")) { + return assertIsId(schemeSpecificPart); + } + final String extractedId = extractId(schemeSpecificPart.substring(2)); + if (extractedId != null) { + return extractedId; + } + + urlString = "https:" + schemeSpecificPart; + } + } catch (final URISyntaxException ignored) { + } + + final URL url; + try { + url = Utils.stringToURL(urlString); + } catch (final MalformedURLException e) { + throw new IllegalArgumentException("The given URL is not valid"); + } + + final String host = url.getHost(); + String path = url.getPath(); + // remove leading "/" of URL-path if URL-path is given + if (!path.isEmpty()) { + path = path.substring(1); + } + + if (!Utils.isHTTP(url) + || !(isYoutubeURL(url) + || isYoutubeServiceURL(url) + || isHooktubeURL(url) + || isInvidioURL(url) + || isY2ubeURL(url)) + ) { + if ("googleads.g.doubleclick.net".equalsIgnoreCase(host)) { + throw new FoundAdException("Error found ad: " + urlString); + } + + throw new ParsingException("The url is not a Youtube-URL"); + } + + if (YoutubePlaylistLinkHandlerFactory.getInstance().acceptUrl(urlString)) { + throw new ParsingException("Can't find handler for url: " + urlString); + } + + if (isYoutubeURL(url) || isInvidioURL(url) || isHooktubeURL(url)) { + if (isYoutubeURL(url) + && "attribution_link".equalsIgnoreCase(path)) { + final String uQueryValue = Utils.getQueryValue(url, "u"); + + final URL decodedURL; + try { + decodedURL = Utils.stringToURL("https://www.youtube.com" + uQueryValue); + } catch (final MalformedURLException e) { + throw new ParsingException("Url is invalid: " + urlString, e); + } + + final String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); + return assertIsId(viewQueryValue); + + } else if ((isInvidioURL(url) || isHooktubeURL(url)) + && path.equalsIgnoreCase("watch")) { + final String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsId(viewQueryValue); + } + } + + final String maybeId = getIdFromSubpathsInPath(path); + if (maybeId != null) { + return maybeId; + } + + final String viewQueryValue = Utils.getQueryValue(url, "v"); + return assertIsId(viewQueryValue); + + } else if (isYoutubeServiceURL(url) || isY2ubeURL(url)) { + if ("youtube-nocookie.com".equalsIgnoreCase( + Utils.removeMAndWWWFromHost(url.getHost()))) { + if (!path.startsWith("embed/")) { + throw new ParsingException("Invalid url: " + urlString); + } + return assertIsId(path.substring(6)); + } + final String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsId(viewQueryValue); + } + + return assertIsId(path); + } + + throw new ParsingException("Error no suitable url: " + urlString); + } + + @Override + public boolean onAcceptUrl(final String url) throws FoundAdException { + try { + getId(url); + return true; + } catch (final FoundAdException fe) { + throw fe; + } catch (final ParsingException e) { + return false; + } + } + + private String getIdFromSubpathsInPath(final String path) throws ParsingException { + for (final String subpath : SUBPATHS) { + if (path.startsWith(subpath)) { + return assertIsId(path.substring(subpath.length())); + } + } + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeDirectService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeDirectService.java new file mode 100644 index 0000000000..9a6d4abeac --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeDirectService.java @@ -0,0 +1,215 @@ +package org.schabi.newpipe.extractor.services.youtube.youtube; + +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; +import static java.util.Arrays.asList; + +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.localization.ContentCountry; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.YoutubeLikeStreamingService; +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubePlaylistHelper; +import org.schabi.newpipe.extractor.services.youtube.shared.extractors.YoutubeTakeoutSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeMusicSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubePlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeSuggestionExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +import java.util.List; + +import javax.annotation.Nonnull; + +/* + * Created by Christian Schabesberger on 23.08.15. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeService.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class YoutubeDirectService extends YoutubeLikeStreamingService { + + public YoutubeDirectService(final int id) { + super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS)); + } + + @Override + public String getBaseUrl() { + return "https://youtube.com"; + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return YoutubeStreamLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + return YoutubeChannelLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + return YoutubePlaylistLinkHandlerFactory.getInstance(); + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return YoutubeSearchQueryHandlerFactory.getInstance(); + } + + @Override + public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) { + return new YoutubeStreamExtractor(this, linkHandler); + } + + @Override + public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) { + return new YoutubeChannelExtractor(this, linkHandler); + } + + @Override + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { + if (YoutubePlaylistHelper.isYoutubeMixId(linkHandler.getId())) { + return new YoutubeMixPlaylistExtractor(this, linkHandler); + } else { + return new YoutubePlaylistExtractor(this, linkHandler); + } + } + + @Override + public SearchExtractor getSearchExtractor(final SearchQueryHandler query) { + final List contentFilters = query.getContentFilters(); + + if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("music_")) { + return new YoutubeMusicSearchExtractor(this, query); + } else { + return new YoutubeSearchExtractor(this, query); + } + } + + @Override + public SuggestionExtractor getSuggestionExtractor() { + return new YoutubeSuggestionExtractor(this); + } + + @Override + public KioskList getKioskList() { + final KioskList list = new KioskList(this); + + // add kiosks here e.g.: + list.addKioskEntry( + (streamingService, url, id) -> new YoutubeTrendingExtractor( + YoutubeDirectService.this, + new YoutubeTrendingLinkHandlerFactory().fromUrl(url), + id + ), + new YoutubeTrendingLinkHandlerFactory(), + "Trending" + ); + list.setDefaultKiosk("Trending"); + + return list; + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + return new YoutubeTakeoutSubscriptionExtractor(this); + } + + @Nonnull + @Override + public FeedExtractor getFeedExtractor(final String channelUrl) throws ExtractionException { + return new YoutubeFeedExtractor(this, getChannelLHFactory().fromUrl(channelUrl)); + } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return YoutubeCommentsLinkHandlerFactory.getInstance(); + } + + @Override + public CommentsExtractor getCommentsExtractor(final ListLinkHandler urlIdHandler) + throws ExtractionException { + return new YoutubeCommentsExtractor(this, urlIdHandler); + } + + /*////////////////////////////////////////////////////////////////////////// + // Localization + //////////////////////////////////////////////////////////////////////////*/ + + // https://www.youtube.com/picker_ajax?action_language_json=1 + public static final List SUPPORTED_LANGUAGES = Localization.listFrom( + "en-GB" + /*"af", "am", "ar", "az", "be", "bg", "bn", "bs", "ca", "cs", "da", "de", + "el", "en", "en-GB", "es", "es-419", "es-US", "et", "eu", "fa", "fi", "fil", "fr", + "fr-CA", "gl", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "iw", "ja", + "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", "mn", + "mr", "ms", "my", "ne", "nl", "no", "pa", "pl", "pt", "pt-PT", "ro", "ru", + "si", "sk", "sl", "sq", "sr", "sr-Latn", "sv", "sw", "ta", "te", "th", "tr", + "uk", "ur", "uz", "vi", "zh-CN", "zh-HK", "zh-TW", "zu"*/ + ); + + // https://www.youtube.com/picker_ajax?action_country_json=1 + public static final List SUPPORTED_COUNTRIES = ContentCountry.listFrom( + "DZ", "AR", "AU", "AT", "AZ", "BH", "BD", "BY", "BE", "BO", "BA", "BR", "BG", "CA", + "CL", "CO", "CR", "HR", "CY", "CZ", "DK", "DO", "EC", "EG", "SV", "EE", "FI", "FR", + "GE", "DE", "GH", "GR", "GT", "HN", "HK", "HU", "IS", "IN", "ID", "IQ", "IE", "IL", + "IT", "JM", "JP", "JO", "KZ", "KE", "KW", "LV", "LB", "LY", "LI", "LT", "LU", "MY", + "MT", "MX", "ME", "MA", "NP", "NL", "NZ", "NI", "NG", "MK", "NO", "OM", "PK", "PA", + "PG", "PY", "PE", "PH", "PL", "PT", "PR", "QA", "RO", "RU", "SA", "SN", "RS", "SG", + "SK", "SI", "ZA", "KR", "ES", "LK", "SE", "CH", "TW", "TZ", "TH", "TN", "TR", "UG", + "UA", "AE", "GB", "US", "UY", "VE", "VN", "YE", "ZW" + ); + + @Override + public List getSupportedLocalizations() { + return SUPPORTED_LANGUAGES; + } + + @Override + public List getSupportedCountries() { + return SUPPORTED_COUNTRIES; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeInstance.java new file mode 100644 index 0000000000..ed98987f1c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeInstance.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.extractor.services.youtube.youtube; + +import org.schabi.newpipe.extractor.services.youtube.YoutubeLikeInstance; + +public class YoutubeInstance extends YoutubeLikeInstance { + + public static final YoutubeInstance YOUTUBE = + new YoutubeInstance("https://youtube.com", "YouTube"); + + protected YoutubeInstance(final String url, final String name) { + super(url, name); + } + + @Override + public YoutubeDirectService getNewStreamingService(final int id) { + return new YoutubeDirectService(id); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeJavaScriptExtractor.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeJavaScriptExtractor.java index 5813cf09df..0ebae536be 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeJavaScriptExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.youtube; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeParsingHelper.java similarity index 82% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeParsingHelper.java index d1df0cb8dd..ff8b66ca8b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeParsingHelper.java @@ -1,24 +1,4 @@ -/* - * Created by Christian Schabesberger on 02.03.16. - * - * Copyright (C) Christian Schabesberger 2016 - * YoutubeParsingHelper.java is part of NewPipe Extractor. - * - * NewPipe Extractor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe Extractor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe Extractor. If not, see . - */ - -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.youtube; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; @@ -44,7 +24,7 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; -import org.schabi.newpipe.extractor.playlist.PlaylistInfo; +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; @@ -190,7 +170,6 @@ private YoutubeParsingHelper() { private static String[] youtubeMusicKey; private static boolean keyAndVersionExtracted = false; - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private static Optional hardcodedClientVersionAndKeyValid = Optional.empty(); private static final String[] INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES = @@ -247,74 +226,6 @@ private YoutubeParsingHelper() { "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; - private static boolean isGoogleURL(final String url) { - final String cachedUrl = extractCachedUrlIfNeeded(url); - try { - final URL u = new URL(cachedUrl); - final String host = u.getHost(); - return host.startsWith("google.") - || host.startsWith("m.google.") - || host.startsWith("www.google."); - } catch (final MalformedURLException e) { - return false; - } - } - - public static boolean isYoutubeURL(@Nonnull final URL url) { - final String host = url.getHost(); - return host.equalsIgnoreCase("youtube.com") - || host.equalsIgnoreCase("www.youtube.com") - || host.equalsIgnoreCase("m.youtube.com") - || host.equalsIgnoreCase("music.youtube.com"); - } - - public static boolean isYoutubeServiceURL(@Nonnull final URL url) { - final String host = url.getHost(); - return host.equalsIgnoreCase("www.youtube-nocookie.com") - || host.equalsIgnoreCase("youtu.be"); - } - - public static boolean isHooktubeURL(@Nonnull final URL url) { - final String host = url.getHost(); - return host.equalsIgnoreCase("hooktube.com"); - } - - public static boolean isInvidioURL(@Nonnull final URL url) { - final String host = url.getHost(); - return host.equalsIgnoreCase("invidio.us") - || host.equalsIgnoreCase("dev.invidio.us") - || host.equalsIgnoreCase("www.invidio.us") - || host.equalsIgnoreCase("redirect.invidious.io") - || host.equalsIgnoreCase("invidious.snopyta.org") - || host.equalsIgnoreCase("yewtu.be") - || host.equalsIgnoreCase("tube.connect.cafe") - || host.equalsIgnoreCase("tubus.eduvid.org") - || host.equalsIgnoreCase("invidious.kavin.rocks") - || host.equalsIgnoreCase("invidious-us.kavin.rocks") - || host.equalsIgnoreCase("piped.kavin.rocks") - || host.equalsIgnoreCase("invidious.site") - || host.equalsIgnoreCase("vid.mint.lgbt") - || host.equalsIgnoreCase("invidiou.site") - || host.equalsIgnoreCase("invidious.fdn.fr") - || host.equalsIgnoreCase("invidious.048596.xyz") - || host.equalsIgnoreCase("invidious.zee.li") - || host.equalsIgnoreCase("vid.puffyan.us") - || host.equalsIgnoreCase("ytprivate.com") - || host.equalsIgnoreCase("invidious.namazso.eu") - || host.equalsIgnoreCase("invidious.silkky.cloud") - || host.equalsIgnoreCase("invidious.exonip.de") - || host.equalsIgnoreCase("inv.riverside.rocks") - || host.equalsIgnoreCase("invidious.blamefran.net") - || host.equalsIgnoreCase("invidious.moomoo.me") - || host.equalsIgnoreCase("ytb.trom.tf") - || host.equalsIgnoreCase("yt.cyberhost.uk") - || host.equalsIgnoreCase("y.com.cm"); - } - - public static boolean isY2ubeURL(@Nonnull final URL url) { - return url.getHost().equalsIgnoreCase("y2u.be"); - } - /** * Parses the duration string of the video expecting ":" or "." as separators * @@ -411,152 +322,7 @@ public static OffsetDateTime parseDateFrom(final String textualUploadDate) } } - /** - * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) - * Ids from a YouTube Mix start with "RD" - * - * @param playlistId the playlist id - * @return Whether given id belongs to a YouTube Mix - */ - public static boolean isYoutubeMixId(@Nonnull final String playlistId) { - return playlistId.startsWith("RD") - && !isYoutubeMusicMixId(playlistId); - } - - /** - * Checks if the given playlist id is a YouTube My Mix (auto-generated playlist) - * Ids from a YouTube My Mix start with "RDMM" - * - * @param playlistId the playlist id - * @return Whether given id belongs to a YouTube My Mix - */ - public static boolean isYoutubeMyMixId(@Nonnull final String playlistId) { - return playlistId.startsWith("RDMM"); - } - - /** - * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) - * Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK" - * - * @param playlistId the playlist id - * @return Whether given id belongs to a YouTube Music Mix - */ - public static boolean isYoutubeMusicMixId(@Nonnull final String playlistId) { - return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK"); - } - - /** - * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) - * Ids from a YouTube channel Mix start with "RDCM" - * - * @return Whether given id belongs to a YouTube Channel Mix - */ - public static boolean isYoutubeChannelMixId(@Nonnull final String playlistId) { - return playlistId.startsWith("RDCM"); - } - - /** - * Checks if the given playlist id is a YouTube Genre Mix (auto-generated playlist) - * Ids from a YouTube Genre Mix start with "RDGMEM" - * - * @return Whether given id belongs to a YouTube Genre Mix - */ - public static boolean isYoutubeGenreMixId(@Nonnull final String playlistId) { - return playlistId.startsWith("RDGMEM"); - } - - /** - * @param playlistId the playlist id to parse - * @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistId (mix playlist - * types included) - * @throws ParsingException if the playlistId is null or empty, if the playlistId is not a mix, - * if it is a mix but it's not based on a specific stream (this is the - * case for channel or genre mixes) - */ - @Nonnull - public static String extractVideoIdFromMixId(final String playlistId) - throws ParsingException { - if (isNullOrEmpty(playlistId)) { - throw new ParsingException("Video id could not be determined from empty playlist id"); - - } else if (isYoutubeMyMixId(playlistId)) { - return playlistId.substring(4); - - } else if (isYoutubeMusicMixId(playlistId)) { - return playlistId.substring(6); - - } else if (isYoutubeChannelMixId(playlistId)) { - // Channel mixes are of the form RMCM{channelId}, so videoId can't be determined - throw new ParsingException("Video id could not be determined from channel mix id: " - + playlistId); - - } else if (isYoutubeGenreMixId(playlistId)) { - // Genre mixes are of the form RDGMEM{garbage}, so videoId can't be determined - throw new ParsingException("Video id could not be determined from genre mix id: " - + playlistId); - - } else if (isYoutubeMixId(playlistId)) { // normal mix - if (playlistId.length() != 13) { - // Stream YouTube mixes are of the form RD{videoId}, but if videoId is not exactly - // 11 characters then it can't be a video id, hence we are dealing with a different - // type of mix (e.g. genre mixes handled above, of the form RDGMEM{garbage}) - throw new ParsingException("Video id could not be determined from mix id: " - + playlistId); - } - return playlistId.substring(2); - - } else { // not a mix - throw new ParsingException("Video id could not be determined from playlist id: " - + playlistId); - } - } - - /** - * @param playlistId the playlist id to parse - * @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistId (mix playlist - * types included) - * @throws ParsingException if the playlistId is null or empty - */ - @Nonnull - public static PlaylistInfo.PlaylistType extractPlaylistTypeFromPlaylistId( - final String playlistId) throws ParsingException { - if (isNullOrEmpty(playlistId)) { - throw new ParsingException("Could not extract playlist type from empty playlist id"); - } else if (isYoutubeMusicMixId(playlistId)) { - return PlaylistInfo.PlaylistType.MIX_MUSIC; - } else if (isYoutubeChannelMixId(playlistId)) { - return PlaylistInfo.PlaylistType.MIX_CHANNEL; - } else if (isYoutubeGenreMixId(playlistId)) { - return PlaylistInfo.PlaylistType.MIX_GENRE; - } else if (isYoutubeMixId(playlistId)) { // normal mix - // Either a normal mix based on a stream, or a "my mix" (still based on a stream). - // NOTE: if YouTube introduces even more types of mixes that still start with RD, - // they will default to this, even though they might not be based on a stream. - return PlaylistInfo.PlaylistType.MIX_STREAM; - } else { - // not a known type of mix: just consider it a normal playlist - return PlaylistInfo.PlaylistType.NORMAL; - } - } - - /** - * @param playlistUrl the playlist url to parse - * @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistUrl's list param - * (mix playlist types included) - * @throws ParsingException if the playlistUrl is malformed, if has no list param or if the list - * param is empty - */ - public static PlaylistInfo.PlaylistType extractPlaylistTypeFromPlaylistUrl( - final String playlistUrl) throws ParsingException { - try { - return extractPlaylistTypeFromPlaylistId( - Utils.getQueryValue(Utils.stringToURL(playlistUrl), "list")); - } catch (final MalformedURLException e) { - throw new ParsingException("Could not extract playlist type from malformed url", e); - } - } - - private static JsonObject getInitialData(final String html) throws ParsingException { + public static JsonObject getInitialData(final String html) throws ParsingException { try { return JsonParser.object().from(getStringResultFromRegexArray(html, INITIAL_DATA_REGEXES, 1)); @@ -974,8 +740,8 @@ public static String getTextFromObject(final JsonObject textObject, final boolea String text = textBuilder.toString(); if (html) { - text = text.replaceAll("\\n", "
"); - text = text.replaceAll(" {2}", "  "); + text = text.replace("\\n", "
") + .replace(" ", "  "); } return text; @@ -1005,7 +771,7 @@ public static String fixThumbnailUrl(@Nonnull final String thumbnailUrl) { if (result.startsWith(HTTP)) { result = Utils.replaceHttpWithHttps(result); } else if (!result.startsWith(HTTPS)) { - result = "https://" + result; + result = HTTPS + result; } return result; @@ -1445,7 +1211,7 @@ private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelC final String metaInfoLinkUrl = YoutubeParsingHelper.getUrlFromNavigationEndpoint( infoPanelContentRenderer.getObject("sourceEndpoint")); try { - metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded( + metaInfo.addUrl(new URL(Objects.requireNonNull(YoutubeUrlHelper.extractCachedUrlIfNeeded( metaInfoLinkUrl)))); } catch (final NullPointerException | MalformedURLException e) { throw new ParsingException("Could not get metadata info URL", e); @@ -1483,7 +1249,7 @@ private static MetaInfo getClarificationRendererContent( try { final String url = YoutubeParsingHelper.getUrlFromNavigationEndpoint(actionButton .getObject("command")); - metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url)))); + metaInfo.addUrl(new URL(Objects.requireNonNull(YoutubeUrlHelper.extractCachedUrlIfNeeded(url)))); } catch (final NullPointerException | MalformedURLException e) { throw new ParsingException("Could not get metadata info URL", e); } @@ -1501,7 +1267,7 @@ private static MetaInfo getClarificationRendererContent( final String url = getUrlFromNavigationEndpoint(clarificationRenderer .getObject("secondaryEndpoint")); // Ignore Google URLs, because those point to a Google search about "Covid-19" - if (url != null && !isGoogleURL(url)) { + if (url != null && !YoutubeUrlHelper.isGoogleURL(url)) { try { metaInfo.addUrl(new URL(url)); final String description = getTextFromObject(clarificationRenderer @@ -1516,23 +1282,6 @@ private static MetaInfo getClarificationRendererContent( return metaInfo; } - /** - * Sometimes, YouTube provides URLs which use Google's cache. They look like - * {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL} - * - * @param url the URL which might refer to the Google's webcache - * @return the URL which is referring to the original site - */ - public static String extractCachedUrlIfNeeded(final String url) { - if (url == null) { - return null; - } - if (url.contains("webcache.googleusercontent.com")) { - return url.split("cache:")[1]; - } - return url; - } - public static boolean isVerified(final JsonArray badges) { if (Utils.isNullOrEmpty(badges)) { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeThrottlingDecrypter.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeThrottlingDecrypter.java index 2158bb3221..8b8f13e1e8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeThrottlingDecrypter.java @@ -1,16 +1,17 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.youtube; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.JavaScript; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.StringUtils; -import javax.annotation.Nonnull; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nonnull; + /** *

* YouTube's media is protected with a cipher, @@ -40,8 +41,8 @@ public class YoutubeThrottlingDecrypter { "\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]+)(?:\\[(\\d+)])?\\([a-zA-Z0-9]\\)"); private static final Map N_PARAMS_CACHE = new HashMap<>(); - @SuppressWarnings("StaticVariableName") private static String FUNCTION; - @SuppressWarnings("StaticVariableName") private static String FUNCTION_NAME; + private static String FUNCTION; + private static String FUNCTION_NAME; private final String functionName; private final String function; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeChannelExtractor.java similarity index 93% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeChannelExtractor.java index 2318ddc179..213d8c0874 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeChannelExtractor.java @@ -1,14 +1,14 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.addClientInfoHeaders; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -28,8 +28,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeChannelInfoItemExtractor.java similarity index 89% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 1193de76a6..13fd91c545 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,17 +1,17 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; - /* * Created by Christian Schabesberger on 12.02.17. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsExtractor.java similarity index 96% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsExtractor.java index e24819526f..fd4c0619e3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsExtractor.java @@ -1,17 +1,12 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -25,9 +20,14 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class YoutubeCommentsExtractor extends CommentsExtractor { @@ -42,7 +42,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { * If the method or another one that is depending on disabled comments * is now called again, the method execution can avoid unnecessary calls */ - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private Optional optCommentsDisabled = Optional.empty(); public YoutubeCommentsExtractor( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsInfoItemExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index f6d86f7c76..a9f64c6237 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -1,6 +1,6 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import com.grack.nanojson.JsonArray; @@ -226,6 +226,7 @@ public boolean isPinned() throws ParsingException { return getCommentRenderer().has("pinnedCommentBadge"); } + @Override public boolean isUploaderVerified() throws ParsingException { return getCommentRenderer().has("authorCommentBadge"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeFeedExtractor.java similarity index 94% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeFeedExtractor.java index e3369d1ecf..266f02e16d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeFeedExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -13,7 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeFeedInfoItemExtractor.java similarity index 90% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeFeedInfoItemExtractor.java index a79586e8f4..2779ba5485 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -6,10 +6,11 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import javax.annotation.Nullable; import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; +import javax.annotation.Nullable; + public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { private final Element entryElement; @@ -24,11 +25,6 @@ public StreamType getStreamType() { return StreamType.VIDEO_STREAM; } - @Override - public boolean isAd() { - return false; - } - @Override public long getDuration() { // Not available when fetching through the feed endpoint. @@ -51,12 +47,6 @@ public String getUploaderUrl() { return entryElement.select("author > uri").first().text(); } - @Nullable - @Override - public String getUploaderAvatarUrl() throws ParsingException { - return null; - } - @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java similarity index 82% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java index 4ed4288720..75b23bcdc6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java @@ -1,8 +1,8 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubePlaylistHelper.extractPlaylistTypeFromPlaylistUrl; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; @@ -11,7 +11,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import javax.annotation.Nonnull; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixPlaylistExtractor.java similarity index 87% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixPlaylistExtractor.java index 49225330be..f730082707 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,13 +1,13 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubePlaylistHelper.extractPlaylistTypeFromPlaylistId; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.addClientInfoHeaders; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.extractCookieValue; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -30,7 +30,8 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubePlaylistHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -142,17 +143,6 @@ public String getUploaderName() { return "YouTube"; } - @Override - public String getUploaderAvatarUrl() { - // YouTube mixes are auto-generated by YouTube - return EMPTY_STRING; - } - - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Override public long getStreamCount() { // Auto-generated playlists always start with 25 videos and are endless @@ -247,19 +237,19 @@ private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collecto } @Nonnull - private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) - throws ParsingException { - return getThumbnailUrlFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId)); + @Override + public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException { + return extractPlaylistTypeFromPlaylistId(playlistData.getString("playlistId")); } @Nonnull - private String getThumbnailUrlFromVideoId(final String videoId) { - return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + public static String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) + throws ParsingException { + return getThumbnailUrlFromVideoId(YoutubePlaylistHelper.extractVideoIdFromMixId(playlistId)); } @Nonnull - @Override - public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException { - return extractPlaylistTypeFromPlaylistId(playlistData.getString("playlistId")); + static String getThumbnailUrlFromVideoId(final String videoId) { + return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMusicSearchExtractor.java similarity index 95% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMusicSearchExtractor.java index bcbabbcb75..98f5faa1f2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -1,15 +1,15 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -22,6 +22,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -31,9 +32,8 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -280,7 +280,6 @@ public InfoItemsPage getPage(final Page page) return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } - @SuppressWarnings("MethodLength") private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector, @Nonnull final JsonArray videos) { final TimeAgoParser timeAgoParser = getTimeAgoParser(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubePlaylistExtractor.java similarity index 90% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubePlaylistExtractor.java index 85c7d8f4ea..5a1d842aa8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubePlaylistExtractor.java @@ -1,16 +1,16 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubePlaylistHelper.extractPlaylistTypeFromPlaylistUrl; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.addClientInfoHeaders; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -29,7 +29,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubePlaylistInfoItemExtractor.java similarity index 84% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 14895aac07..362786c546 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -1,15 +1,15 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; - public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private final JsonObject playlistInfoItem; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeSearchExtractor.java similarity index 90% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeSearchExtractor.java index de937cb021..bba006c11b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeSearchExtractor.java @@ -1,13 +1,13 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -28,9 +29,8 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeStreamExtractor.java similarity index 96% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeStreamExtractor.java index 46676ff3c7..901dda601b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeStreamExtractor.java @@ -1,20 +1,20 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createDesktopPlayerBody; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.CPN; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.RACY_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.VIDEO_ID; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.createDesktopPlayerBody; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.generateTParameter; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getJsonIosPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -44,11 +44,11 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.shared.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeJavaScriptExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeThrottlingDecrypter; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Frameset; @@ -1375,11 +1375,6 @@ public String getLicence() throws ParsingException { : "YouTube licence"; } - @Override - public Locale getLanguageInfo() { - return null; - } - @Nonnull @Override public List getTags() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeStreamInfoItemExtractor.java similarity index 94% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeStreamInfoItemExtractor.java index f3ecc74a05..fc063c19c9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,8 +1,8 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -12,8 +12,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeSuggestionExtractor.java similarity index 71% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeSuggestionExtractor.java index 70789d4575..c5b3df7818 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeSuggestionExtractor.java @@ -1,6 +1,6 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.addCookieHeader; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import com.grack.nanojson.JsonArray; @@ -9,17 +9,17 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.io.IOException; import java.net.URLEncoder; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; /* * Created by Christian Schabesberger on 28.09.16. @@ -49,9 +49,6 @@ public YoutubeSuggestionExtractor(final StreamingService service) { @Override public List suggestionList(final String query) throws IOException, ExtractionException { - final Downloader dl = NewPipe.getDownloader(); - final List suggestions = new ArrayList<>(); - final String url = "https://suggestqueries.google.com/complete/search" + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml + "&jsonp=" + "JP" @@ -62,23 +59,17 @@ public List suggestionList(final String query) throws IOException, Extra final Map> headers = new HashMap<>(); addCookieHeader(headers); - String response = dl.get(url, headers, getExtractorLocalization()).responseBody(); + String response = NewPipe.getDownloader() + .get(url, headers, getExtractorLocalization()).responseBody(); // trim JSONP part "JP(...)" response = response.substring(3, response.length() - 1); try { - final JsonArray collection = JsonParser.array().from(response).getArray(1); - for (final Object suggestion : collection) { - if (!(suggestion instanceof JsonArray)) { - continue; - } - final String suggestionStr = ((JsonArray) suggestion).getString(0); - if (suggestionStr == null) { - continue; - } - suggestions.add(suggestionStr); - } - - return suggestions; + return JsonParser.array().from(response).getArray(1).stream() + .filter(JsonArray.class::isInstance) + .map(JsonArray.class::cast) + .map(arr -> arr.getString(0)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeTrendingExtractor.java similarity index 92% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeTrendingExtractor.java index a82b8d5dbd..f39b620518 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeTrendingExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; +package org.schabi.newpipe.extractor.services.youtube.youtube.extractors; /* * Created by Christian Schabesberger on 12.08.17. @@ -20,6 +20,12 @@ * along with NewPipe. If not, see . */ +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getTextAtKey; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; @@ -39,12 +45,6 @@ import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - public class YoutubeTrendingExtractor extends KioskExtractor { private JsonObject initialData; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java new file mode 100644 index 0000000000..ce296bd0e4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -0,0 +1,51 @@ +package org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeChannelLinkHandlerFactory; + +import java.util.List; + +/* + * Created by Christian Schabesberger on 25.07.16. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeChannelLinkHandlerFactory.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public final class YoutubeChannelLinkHandlerFactory extends YoutubeLikeChannelLinkHandlerFactory { + + public static final YoutubeChannelLinkHandlerFactory INSTANCE + = new YoutubeChannelLinkHandlerFactory(); + + private YoutubeChannelLinkHandlerFactory() { + } + + public static YoutubeChannelLinkHandlerFactory getInstance() { + return INSTANCE; + } + + /** + * Returns URL to channel from an ID + * + * @param id Channel ID including e.g. 'channel/' + * @return URL to channel + */ + @Override + public String getUrl(final String id, + final List contentFilters, + final String searchFilter) { + return "https://www.youtube.com/" + id; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java new file mode 100644 index 0000000000..c14606ceb3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java @@ -0,0 +1,17 @@ +package org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeCommentsLinkHandlerFactory; + +public final class YoutubeCommentsLinkHandlerFactory extends YoutubeLikeCommentsLinkHandlerFactory { + + private static final YoutubeCommentsLinkHandlerFactory INSTANCE + = new YoutubeCommentsLinkHandlerFactory(); + + private YoutubeCommentsLinkHandlerFactory() { + super(YoutubeStreamLinkHandlerFactory.getInstance()); + } + + public static YoutubeCommentsLinkHandlerFactory getInstance() { + return INSTANCE; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java new file mode 100644 index 0000000000..f40f935606 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -0,0 +1,29 @@ +package org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikePlaylistLinkHandlerFactory; + +import java.util.List; + +public final class YoutubePlaylistLinkHandlerFactory extends YoutubeLikePlaylistLinkHandlerFactory { + + private static final YoutubePlaylistLinkHandlerFactory INSTANCE = + new YoutubePlaylistLinkHandlerFactory(); + + private YoutubePlaylistLinkHandlerFactory() { + } + + public static YoutubePlaylistLinkHandlerFactory getInstance() { + return INSTANCE; + } + + @Override + public String getUrl(final String id, final List contentFilters, + final String sortFilter) { + return "https://www.youtube.com/playlist?list=" + id; + } + + @Override + protected String getMixUrl(final String videoID, final String listID) { + return "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 798a2347c6..4b08e6da40 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -1,15 +1,16 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; +package org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler; + +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; -import javax.annotation.Nonnull; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.List; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java new file mode 100644 index 0000000000..fb593bb935 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -0,0 +1,41 @@ +package org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeStreamLinkHandlerFactory; + +/* + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeStreamLinkHandlerFactory.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public final class YoutubeStreamLinkHandlerFactory extends YoutubeLikeStreamLinkHandlerFactory { + + private static final YoutubeStreamLinkHandlerFactory INSTANCE + = new YoutubeStreamLinkHandlerFactory(); + + private YoutubeStreamLinkHandlerFactory() { + } + + public static YoutubeStreamLinkHandlerFactory getInstance() { + return INSTANCE; + } + + @Override + public String getUrl(final String id) { + return "https://www.youtube.com/watch?v=" + id; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java similarity index 86% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index 3cc21748b4..1949db5ba6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; +package org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler; /* * Created by Christian Schabesberger on 12.08.17. @@ -20,8 +20,8 @@ * along with NewPipe. If not, see . */ -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL; +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isInvidioURL; +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isYoutubeURL; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; @@ -32,6 +32,7 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + @Override public String getUrl(final String id, final List contentFilters, final String sortFilter) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java index c1cf2e0e1b..750d03110c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java @@ -21,7 +21,7 @@ */ import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.shared.ItagItem; public class AudioStream extends Stream { private final int averageBitrate; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java index 5a6f94b7ff..d139085806 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.stream; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; + import java.io.Serializable; import java.util.Objects; -import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; - public class Description implements Serializable { public static final int HTML = 1; @@ -17,11 +17,7 @@ public class Description implements Serializable { public Description(final String content, final int type) { this.type = type; - if (content == null) { - this.content = ""; - } else { - this.content = content; - } + this.content = content == null ? "" : content; } public String getContent() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index 131719d8c3..e013c75f11 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -43,7 +43,9 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * @return {@code true} if the stream is an ad. * @throws ParsingException thrown if there is an error in the extraction */ - boolean isAd() throws ParsingException; + default boolean isAd() throws ParsingException { + return false; + } /** * Get the stream duration in seconds @@ -78,7 +80,9 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * @throws ParsingException if there is an error in the extraction */ @Nullable - String getUploaderAvatarUrl() throws ParsingException; + default String getUploaderAvatarUrl() throws ParsingException { + return null; + } /** * Whether the uploader has been verified by the service's provider. @@ -87,7 +91,9 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { * @return whether the uploader has been verified by the service's provider * @throws ParsingException */ - boolean isUploaderVerified() throws ParsingException; + default boolean isUploaderVerified() throws ParsingException { + return false; + } /** * The original textual date provided by the service. Should be used as a fallback if diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java index 9e6b4eb2be..d4cb96831e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java @@ -21,7 +21,7 @@ */ import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.shared.ItagItem; public class VideoStream extends Stream { public final String resolution; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index b1acabc75b..47f73a4ffe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.shared.ItagItem; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamInfo; @@ -15,14 +15,15 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + /* * Created by Christian Schabesberger on 02.02.16. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 0f1ee49fa7..14b5691e01 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -200,6 +200,16 @@ public static boolean isHTTP(@Nonnull final URL url) { return setsNoPort || usesDefaultPort; } + public static String removeMAndWWWFromHost(final String host) { + if (host.startsWith("m.")) { + return host.replace("m.", ""); + } + if (host.startsWith("www.")) { + return host.replace("www.", ""); + } + return host; + } + public static String removeMAndWWWFromUrl(final String url) { if (M_PATTERN.matcher(url).find()) { return url.replace("m.", ""); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index aee006f58b..4fa72b20e7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -24,9 +24,11 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeChannelExtractor; import java.io.IOException; +import java.util.Random; /** * Test for {@link ChannelExtractor} @@ -64,7 +66,7 @@ public void accountTerminatedTOSFetch() throws Exception { final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ"); - AccountTerminatedException ex = + final AccountTerminatedException ex = assertThrows(AccountTerminatedException.class, extractor::fetchPage); assertEquals(AccountTerminatedException.Reason.VIOLATION, ex.getReason()); } @@ -75,7 +77,7 @@ public void accountTerminatedCommunityFetch() throws Exception { final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/UC0AuOxCr9TZ0TtEgL1zpIgA"); - AccountTerminatedException ex = + final AccountTerminatedException ex = assertThrows(AccountTerminatedException.class, extractor::fetchPage); assertEquals(AccountTerminatedException.Reason.VIOLATION, ex.getReason()); } @@ -87,7 +89,7 @@ public void accountTerminatedHateFetch() throws Exception { final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/UCPWXIOPK-9myzek6jHR5yrg"); - AccountTerminatedException ex = + final AccountTerminatedException ex = assertThrows(AccountTerminatedException.class, extractor::fetchPage); assertEquals(AccountTerminatedException.Reason.VIOLATION, ex.getReason()); } @@ -99,7 +101,7 @@ public void accountTerminatedBullyFetch() throws Exception { final ChannelExtractor extractor = YouTube.getChannelExtractor("https://youtube.com/channel/UCB1o7_gbFp2PLsamWxFenBg"); - AccountTerminatedException ex = + final AccountTerminatedException ex = assertThrows(AccountTerminatedException.class, extractor::fetchPage); assertEquals(AccountTerminatedException.Reason.VIOLATION, ex.getReason()); } @@ -112,7 +114,7 @@ public void accountTerminatedSpamFetch() throws Exception { final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/UCoaO4U_p7G7AwalqSbGCZOA"); - AccountTerminatedException ex = + final AccountTerminatedException ex = assertThrows(AccountTerminatedException.class, extractor::fetchPage); assertEquals(AccountTerminatedException.Reason.VIOLATION, ex.getReason()); } @@ -124,7 +126,7 @@ public void accountTerminatedCopyrightFetch() throws Exception { final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/UCI4i4RgFT5ilfMpna4Z_Y8w"); - AccountTerminatedException ex = + final AccountTerminatedException ex = assertThrows(AccountTerminatedException.class, extractor::fetchPage); assertEquals(AccountTerminatedException.Reason.VIOLATION, ex.getReason()); } @@ -163,26 +165,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { assertEquals("Gronkh", extractor.getName()); } + @Override @Test public void testId() throws Exception { assertEquals("UCYJ61XIK64sp6ZFFS8sctxw", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("http://www.youtube.com/user/Gronkh", extractor.getOriginalUrl()); @@ -192,11 +199,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() throws Exception { defaultTestMoreItems(extractor); @@ -206,30 +215,35 @@ public void testMoreRelatedItems() throws Exception { // ChannelExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testDescription() throws Exception { assertContains("Ungebremster Spieltrieb seit 1896.", extractor.getDescription()); } + @Override @Test public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); + final String avatarUrl = extractor.getAvatarUrl(); assertIsSecureUrl(avatarUrl); ExtractorAsserts.assertContains("yt3", avatarUrl); } + @Override @Test public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); + final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); ExtractorAsserts.assertContains("yt3", bannerUrl); } + @Override @Test public void testFeedUrl() throws Exception { assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCYJ61XIK64sp6ZFFS8sctxw", extractor.getFeedUrl()); } + @Override @Test public void testSubscriberCount() throws Exception { ExtractorAsserts.assertGreaterOrEqual(4_900_000, extractor.getSubscriberCount()); @@ -259,26 +273,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { assertEquals("Vsauce", extractor.getName()); } + @Override @Test public void testId() throws Exception { assertEquals("UC6nSFpj9HTCZ5t-N3Rm3-HA", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/user/Vsauce", extractor.getOriginalUrl()); @@ -288,11 +307,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() throws Exception { defaultTestMoreItems(extractor); @@ -302,35 +323,41 @@ public void testMoreRelatedItems() throws Exception { // ChannelExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testDescription() throws Exception { assertContains("Our World is Amazing. \n\nQuestions? Ideas? Tweet me:", extractor.getDescription()); } + @Override @Test public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); + final String avatarUrl = extractor.getAvatarUrl(); assertIsSecureUrl(avatarUrl); ExtractorAsserts.assertContains("yt3", avatarUrl); } + @Override @Test public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); + final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); ExtractorAsserts.assertContains("yt3", bannerUrl); } + @Override @Test public void testFeedUrl() throws Exception { assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC6nSFpj9HTCZ5t-N3Rm3-HA", extractor.getFeedUrl()); } + @Override @Test public void testSubscriberCount() throws Exception { ExtractorAsserts.assertGreaterOrEqual(17_000_000, extractor.getSubscriberCount()); } + @Override @Test public void testVerified() throws Exception { assertTrue(extractor.isVerified()); @@ -354,26 +381,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { assertTrue(extractor.getName().startsWith("Kurzgesagt")); } + @Override @Test public void testId() throws Exception { assertEquals("UCsXVk37bltHxD1rDPwtNM8Q", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getOriginalUrl()); @@ -383,11 +415,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() throws Exception { defaultTestMoreItems(extractor); @@ -397,6 +431,7 @@ public void testMoreRelatedItems() throws Exception { // ChannelExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testDescription() throws Exception { ExtractorAsserts.assertContains("science", extractor.getDescription()); @@ -405,30 +440,35 @@ public void testDescription() throws Exception { //assertTrue(description, description.contains("Currently we make one animation video per month")); } + @Override @Test public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); + final String avatarUrl = extractor.getAvatarUrl(); assertIsSecureUrl(avatarUrl); ExtractorAsserts.assertContains("yt3", avatarUrl); } + @Override @Test public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); + final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); ExtractorAsserts.assertContains("yt3", bannerUrl); } + @Override @Test public void testFeedUrl() throws Exception { assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q", extractor.getFeedUrl()); } + @Override @Test public void testSubscriberCount() throws Exception { ExtractorAsserts.assertGreaterOrEqual(17_000_000, extractor.getSubscriberCount()); } + @Override @Test public void testVerified() throws Exception { assertTrue(extractor.isVerified()); @@ -471,26 +511,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { assertEquals("Captain Disillusion", extractor.getName()); } + @Override @Test public void testId() throws Exception { assertEquals("UCEOXxzW2vU0P-0THehuIIeg", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/user/CaptainDisillusion/videos", extractor.getOriginalUrl()); @@ -500,11 +545,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() throws Exception { defaultTestMoreItems(extractor); @@ -514,35 +561,41 @@ public void testMoreRelatedItems() throws Exception { // ChannelExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testDescription() throws Exception { ExtractorAsserts.assertContains("In a world where", extractor.getDescription()); } + @Override @Test public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); + final String avatarUrl = extractor.getAvatarUrl(); assertIsSecureUrl(avatarUrl); ExtractorAsserts.assertContains("yt3", avatarUrl); } + @Override @Test public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); + final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); ExtractorAsserts.assertContains("yt3", bannerUrl); } + @Override @Test public void testFeedUrl() throws Exception { assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCEOXxzW2vU0P-0THehuIIeg", extractor.getFeedUrl()); } + @Override @Test public void testSubscriberCount() throws Exception { ExtractorAsserts.assertGreaterOrEqual(2_000_000, extractor.getSubscriberCount()); } + @Override @Test public void testVerified() throws Exception { assertTrue(extractor.isVerified()); @@ -565,26 +618,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { assertEquals("random channel", extractor.getName()); } + @Override @Test public void testId() throws Exception { assertEquals("UCUaQMQS9lY5lit3vurpXQ6w", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getOriginalUrl()); @@ -594,11 +652,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() { try { @@ -614,35 +674,41 @@ public void testMoreRelatedItems() { // ChannelExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testDescription() throws Exception { ExtractorAsserts.assertContains("Hey there iu will upoload a load of pranks onto this channel", extractor.getDescription()); } + @Override @Test public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); + final String avatarUrl = extractor.getAvatarUrl(); assertIsSecureUrl(avatarUrl); ExtractorAsserts.assertContains("yt3", avatarUrl); } + @Override @Test public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); + final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); ExtractorAsserts.assertContains("yt3", bannerUrl); } + @Override @Test public void testFeedUrl() throws Exception { assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCUaQMQS9lY5lit3vurpXQ6w", extractor.getFeedUrl()); } + @Override @Test public void testSubscriberCount() throws Exception { ExtractorAsserts.assertGreaterOrEqual(50, extractor.getSubscriberCount()); } + @Override @Test public void testVerified() throws Exception { assertFalse(extractor.isVerified()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index ae9130b590..f8f57012ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -1,18 +1,19 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; /** - * Test for {@link YoutubeChannelLinkHandlerFactory} + * Test for {@link YoutubeLikeChannelLinkHandlerFactory} */ public class YoutubeChannelLinkHandlerFactoryTest { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java index 533ad7ee94..be3378fc9d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.time.format.DateTimeFormatter; @@ -18,6 +19,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Random; /** * A class that tests multiple channels and ranges of "time ago". @@ -44,10 +46,10 @@ private void testLocalizationsFor(final String channelUrl) throws Exception { // final List supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr")); final Map> results = new LinkedHashMap<>(); - for (Localization currentLocalization : supportedLocalizations) { + for (final Localization currentLocalization : supportedLocalizations) { if (DEBUG) System.out.println("Testing localization = " + currentLocalization); - ListExtractor.InfoItemsPage itemsPage; + final ListExtractor.InfoItemsPage itemsPage; try { final ChannelExtractor extractor = YouTube.getChannelExtractor(channelUrl); extractor.forceLocalization(currentLocalization); @@ -67,7 +69,7 @@ private void testLocalizationsFor(final String channelUrl) throws Exception { + "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount(); final DateWrapper uploadDate = item.getUploadDate(); if (uploadDate != null) { - String dateAsText = dateTimeFormatter.format(uploadDate.offsetDateTime()); + final String dateAsText = dateTimeFormatter.format(uploadDate.offsetDateTime()); debugMessage += "\n:::: " + item.getTextualUploadDate() + "\n:::: " + dateAsText; } @@ -83,7 +85,7 @@ private void testLocalizationsFor(final String channelUrl) throws Exception { final List referenceList = results.get(Localization.DEFAULT); boolean someFail = false; - for (Map.Entry> currentResultEntry : results.entrySet()) { + for (final Map.Entry> currentResultEntry : results.entrySet()) { if (currentResultEntry.getKey().equals(Localization.DEFAULT)) { continue; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index b0bb644747..64f77a3a53 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -17,7 +17,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.DefaultTests; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -49,7 +50,7 @@ public void testGetComments() throws IOException, ExtractionException { assertTrue(getCommentsHelper(extractor)); } - private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException { + private boolean getCommentsHelper(final YoutubeCommentsExtractor extractor) throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); boolean result = findInComments(comments, commentContent); @@ -66,7 +67,7 @@ public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionExce assertTrue(getCommentsFromCommentsInfoHelper(url)); } - private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { + private boolean getCommentsFromCommentsInfoHelper(final String url) throws IOException, ExtractionException { final CommentsInfo commentsInfo = CommentsInfo.getInfo(url); assertEquals("Comments", commentsInfo.getName()); @@ -84,10 +85,10 @@ private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException @Test public void testGetCommentsAllData() throws IOException, ExtractionException { - InfoItemsPage comments = extractor.getInitialPage(); + final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); @@ -102,12 +103,12 @@ public void testGetCommentsAllData() throws IOException, ExtractionException { } } - private boolean findInComments(InfoItemsPage comments, String comment) { + private boolean findInComments(final InfoItemsPage comments, final String comment) { return findInComments(comments.getItems(), comment); } - private boolean findInComments(List comments, String comment) { - for (CommentsInfoItem c : comments) { + private boolean findInComments(final List comments, final String comment) { + for (final CommentsInfoItem c : comments) { if (c.getCommentText().contains(comment)) { return true; } @@ -137,7 +138,7 @@ public void testGetCommentsAllData() throws IOException, ExtractionException { final InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); @@ -179,7 +180,7 @@ public void testGetCommentsAllData() throws IOException, ExtractionException { boolean heartedByUploader = false; - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); @@ -219,7 +220,7 @@ public void testGetCommentsAllData() throws IOException, ExtractionException { DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { + for (final CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); @@ -260,7 +261,7 @@ public void testGetCommentsFirst() throws IOException, ExtractionException { DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - CommentsInfoItem pinnedComment = comments.getItems().get(0); + final CommentsInfoItem pinnedComment = comments.getItems().get(0); assertTrue(pinnedComment.isPinned(), "First comment isn't pinned"); assertTrue(pinnedComment.getLikeCount() > 0, "The first pinned comment has no likes"); @@ -293,7 +294,7 @@ public void testGetCommentsFirst() throws IOException, ExtractionException { DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - CommentsInfoItem pinnedComment = comments.getItems().get(0); + final CommentsInfoItem pinnedComment = comments.getItems().get(0); assertTrue(pinnedComment.isPinned(), "First comment isn't pinned"); assertFalse(Utils.isBlank(pinnedComment.getTextualLikeCount()), "The first pinned comment has no vote count"); @@ -319,11 +320,11 @@ public void testGetCommentsFirstReplies() throws IOException, ExtractionExceptio DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - CommentsInfoItem firstComment = comments.getItems().get(0); + final CommentsInfoItem firstComment = comments.getItems().get(0); assertTrue(firstComment.isPinned(), "First comment isn't pinned"); - InfoItemsPage replies = extractor.getPage(firstComment.getReplies()); + final InfoItemsPage replies = extractor.getPage(firstComment.getReplies()); assertEquals("First", replies.getItems().get(0).getCommentText(), "First reply comment did not match"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java index 3e4d7bdf42..365254547a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java @@ -1,16 +1,16 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; public class YoutubeCommentsLinkHandlerFactoryTest { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java index 4169b0cdf3..cce1fa2aa6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java @@ -14,7 +14,8 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeFeedExtractor; import java.io.IOException; @@ -38,26 +39,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() { assertTrue(extractor.getName().startsWith("Kurzgesagt")); } + @Override @Test public void testId() { assertEquals("UCsXVk37bltHxD1rDPwtNM8Q", extractor.getId()); } + @Override @Test public void testUrl() { assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/user/Kurzgesagt", extractor.getOriginalUrl()); @@ -67,11 +73,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() throws Exception { assertNoMoreItems(extractor); @@ -87,7 +95,7 @@ public static void setUp() throws IOException { @Test void AccountTerminatedFetch() throws Exception { - YoutubeFeedExtractor extractor = (YoutubeFeedExtractor) YouTube + final YoutubeFeedExtractor extractor = (YoutubeFeedExtractor) YouTube .getFeedExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ"); assertThrows(ContentNotAvailableException.class, extractor::fetchPage); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java index 1433623ee1..1e1c2ba7df 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractorTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeJavaScriptExtractor; import java.io.IOException; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java index 1b4c0368f3..40cd23648d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java @@ -11,7 +11,8 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeTrendingExtractor; public class YoutubeKioskExtractorTest { @@ -32,26 +33,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { assertEquals("Trending", extractor.getName()); } + @Override @Test public void testId() throws Exception { assertEquals("Trending", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/feed/trending", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/feed/trending", extractor.getOriginalUrl()); @@ -61,11 +67,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() throws Exception { assertNoMoreItems(extractor); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index b562d6594d..0df0744b7c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -6,7 +6,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import com.grack.nanojson.JsonWriter; @@ -21,7 +24,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 1fa89e94ee..4f06d05158 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -9,6 +9,8 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import java.io.IOException; @@ -44,8 +46,8 @@ public void testParseDurationString() throws ParsingException { @Test public void testConvertFromGoogleCacheUrl() { assertEquals("https://mohfw.gov.in/", - YoutubeParsingHelper.extractCachedUrlIfNeeded("https://webcache.googleusercontent.com/search?q=cache:https://mohfw.gov.in/")); + YoutubeUrlHelper.extractCachedUrlIfNeeded("https://webcache.googleusercontent.com/search?q=cache:https://mohfw.gov.in/")); assertEquals("https://www.infektionsschutz.de/coronavirus-sars-cov-2.html", - YoutubeParsingHelper.extractCachedUrlIfNeeded("https://www.infektionsschutz.de/coronavirus-sars-cov-2.html")); + YoutubeUrlHelper.extractCachedUrlIfNeeded("https://www.infektionsschutz.de/coronavirus-sars-cov-2.html")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 10a7a5bade..89ed9f3702 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -25,7 +25,8 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; @@ -75,26 +76,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { assertTrue(extractor.getName().startsWith("Pop Music Playlist")); } + @Override @Test public void testId() throws Exception { assertEquals("PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getOriginalUrl()); @@ -104,11 +110,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() throws Exception { defaultTestMoreItems(extractor); @@ -118,6 +126,7 @@ public void testMoreRelatedItems() throws Exception { // PlaylistExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testThumbnailUrl() throws Exception { final String thumbnailUrl = extractor.getThumbnailUrl(); @@ -125,6 +134,7 @@ public void testThumbnailUrl() throws Exception { ExtractorAsserts.assertContains("yt", thumbnailUrl); } + @Override @Disabled @Test public void testBannerUrl() throws ParsingException { @@ -138,18 +148,21 @@ public void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); } + @Override @Test public void testUploaderName() throws Exception { final String uploaderName = extractor.getUploaderName(); ExtractorAsserts.assertContains("Just Hits", uploaderName); } + @Override @Test public void testUploaderAvatarUrl() throws Exception { final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); } + @Override @Test public void testStreamCount() throws Exception { ExtractorAsserts.assertGreater(100, extractor.getStreamCount()); @@ -192,27 +205,32 @@ public void testGetPageInNewExtractor() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { final String name = extractor.getName(); assertEquals("I Wanna Rock Super Gigantic Playlist 1: Hardrock, AOR, Metal and more !!! 5000 music videos !!!", name); } + @Override @Test public void testId() throws Exception { assertEquals("PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/playlist?list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/watch?v=8SbUC-UaAxE&list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj", extractor.getOriginalUrl()); @@ -222,11 +240,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Test public void testMoreRelatedItems() throws Exception { ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor); @@ -242,6 +262,7 @@ public void testMoreRelatedItems() throws Exception { // PlaylistExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testThumbnailUrl() throws Exception { final String thumbnailUrl = extractor.getThumbnailUrl(); @@ -249,6 +270,7 @@ public void testThumbnailUrl() throws Exception { ExtractorAsserts.assertContains("yt", thumbnailUrl); } + @Override @Disabled @Test public void testBannerUrl() throws ParsingException { @@ -262,17 +284,20 @@ public void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCHSPWoY1J5fbDVbcnyeqwdw", extractor.getUploaderUrl()); } + @Override @Test public void testUploaderName() throws Exception { assertEquals("Tomas Nilsson TOMPA571", extractor.getUploaderName()); } + @Override @Test public void testUploaderAvatarUrl() throws Exception { final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); } + @Override @Test public void testStreamCount() throws Exception { ExtractorAsserts.assertGreater(100, extractor.getStreamCount()); @@ -305,26 +330,31 @@ public static void setUp() throws Exception { // Extractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testServiceId() { assertEquals(YouTube.getServiceId(), extractor.getServiceId()); } + @Override @Test public void testName() throws Exception { assertTrue(extractor.getName().startsWith("Anatomy & Physiology")); } + @Override @Test public void testId() throws Exception { assertEquals("PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getId()); } + @Override @Test public void testUrl() throws ParsingException { assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getUrl()); } + @Override @Test public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getOriginalUrl()); @@ -334,11 +364,13 @@ public void testOriginalUrl() throws ParsingException { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } + @Override @Disabled @Test public void testMoreRelatedItems() throws Exception { @@ -349,6 +381,7 @@ public void testMoreRelatedItems() throws Exception { // PlaylistExtractor //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testThumbnailUrl() throws Exception { final String thumbnailUrl = extractor.getThumbnailUrl(); @@ -356,6 +389,7 @@ public void testThumbnailUrl() throws Exception { ExtractorAsserts.assertContains("yt", thumbnailUrl); } + @Override @Disabled @Test public void testBannerUrl() throws ParsingException { @@ -369,18 +403,21 @@ public void testUploaderUrl() throws Exception { assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl()); } + @Override @Test public void testUploaderName() throws Exception { final String uploaderName = extractor.getUploaderName(); ExtractorAsserts.assertContains("CrashCourse", uploaderName); } + @Override @Test public void testUploaderAvatarUrl() throws Exception { final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl(); ExtractorAsserts.assertContains("yt", uploaderAvatarUrl); } + @Override @Test public void testStreamCount() throws Exception { // We are not able to extract the stream count of YouTube learning playlists diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index c1fd3c88f2..23883b74f3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -1,13 +1,16 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; - -import static org.junit.jupiter.api.Assertions.*; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; /** * Test for {@link YoutubePlaylistLinkHandlerFactory} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index 8247506bd0..abcac78531 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -20,6 +20,11 @@ * along with NewPipe. If not, see . */ +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; @@ -27,13 +32,8 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubePlaylistExtractor; /** * Test for {@link YoutubeService} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 848c4ed508..00bba32639 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -1,23 +1,26 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; - /** * Test for {@link YoutubeStreamLinkHandlerFactory} */ public class YoutubeStreamLinkHandlerFactoryTest { - private static String AD_URL = "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew"; + private static final String AD_URL = "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew"; private static YoutubeStreamLinkHandlerFactory linkHandler; @BeforeAll @@ -38,17 +41,17 @@ public void getIdForAd() throws ParsingException { @Test public void getIdForInvalidUrls() { - List invalidUrls = new ArrayList<>(50); + final List invalidUrls = new ArrayList<>(50); invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d"); invalidUrls.add("https://www.youtube.com/watchjZViOEv90d"); invalidUrls.add("https://www.youtube.com/"); invalidUrls.add("https://www.youtube.com/channel/UCBR8-60-B28hp2BmDPdntcQ"); invalidUrls.add("https://invidio.us/channel/UCBR8-60-B28hp2BmDPdntcQ"); - for (String invalidUrl : invalidUrls) { + for (final String invalidUrl : invalidUrls) { Throwable exception = null; try { linkHandler.fromUrl(invalidUrl).getId(); - } catch (ParsingException e) { + } catch (final ParsingException e) { exception = e; } if (exception == null) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java index 38c9253718..d598d6ff10 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java @@ -29,6 +29,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.io.IOException; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTakeoutSubscriptionExtractorTest.java similarity index 78% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTakeoutSubscriptionExtractorTest.java index f5003383c4..de7cab13e5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTakeoutSubscriptionExtractorTest.java @@ -1,12 +1,20 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.schabi.newpipe.FileUtils.resolveTestResource; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.shared.extractors.YoutubeTakeoutSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeInstance; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; @@ -16,29 +24,26 @@ import java.util.Arrays; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; -import static org.schabi.newpipe.FileUtils.resolveTestResource; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; - /** - * Test for {@link YoutubeSubscriptionExtractor} + * Test for {@link YoutubeTakeoutSubscriptionExtractor} */ -public class YoutubeSubscriptionExtractorTest { +public class YoutubeTakeoutSubscriptionExtractorTest { - private static YoutubeSubscriptionExtractor subscriptionExtractor; + private static YoutubeTakeoutSubscriptionExtractor subscriptionExtractor; private static LinkHandlerFactory urlHandler; @BeforeAll public static void setupClass() { //Doesn't make network requests NewPipe.init(DownloaderTestImpl.getInstance()); - subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube); + subscriptionExtractor = new YoutubeTakeoutSubscriptionExtractor( + YoutubeInstance.YOUTUBE.getNewStreamingService(0)); urlHandler = ServiceList.YouTube.getChannelLHFactory(); } @Test - public void testFromInputStream() throws Exception { + void testFromInputStream() throws Exception { final List subscriptionItems = subscriptionExtractor.fromInputStream( new FileInputStream(resolveTestResource("youtube_takeout_import_test.json"))); assertEquals(7, subscriptionItems.size()); @@ -52,14 +57,14 @@ public void testFromInputStream() throws Exception { } @Test - public void testEmptySourceException() throws Exception { + void testEmptySourceException() throws Exception { final List items = subscriptionExtractor.fromInputStream( new ByteArrayInputStream("[]".getBytes(UTF_8))); assertTrue(items.isEmpty()); } @Test - public void testSubscriptionWithEmptyTitleInSource() throws Exception { + void testSubscriptionWithEmptyTitleInSource() throws Exception { final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]"; final List items = subscriptionExtractor.fromInputStream( new ByteArrayInputStream(source.getBytes(UTF_8))); @@ -71,7 +76,7 @@ public void testSubscriptionWithEmptyTitleInSource() throws Exception { } @Test - public void testSubscriptionWithInvalidUrlInSource() throws Exception { + void testSubscriptionWithInvalidUrlInSource() throws Exception { final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," + "{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]"; final List items = subscriptionExtractor.fromInputStream( @@ -84,8 +89,8 @@ public void testSubscriptionWithInvalidUrlInSource() throws Exception { } @Test - public void testInvalidSourceException() { - List invalidList = Arrays.asList( + void testInvalidSourceException() { + final List invalidList = Arrays.asList( "", "", "{\"a\":\"b\"}", @@ -97,13 +102,13 @@ public void testInvalidSourceException() { "\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28", "gibberish"); - for (String invalidContent : invalidList) { + for (final String invalidContent : invalidList) { try { - byte[] bytes = invalidContent.getBytes(UTF_8); + final byte[] bytes = invalidContent.getBytes(UTF_8); subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); fail("Extracting from \"" + invalidContent + "\" didn't throw an exception"); } catch (final Exception e) { - boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException; + final boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException; if (!correctType) { e.printStackTrace(); } @@ -125,14 +130,13 @@ private static void assertSubscriptionItems(final List subscri } @Test - public void fromZipInputStream() throws Exception { + void fromZipInputStream() throws Exception { final List zipPaths = Arrays.asList( "youtube_takeout_import_test_1.zip", "youtube_takeout_import_test_2.zip" ); - for (final String path : zipPaths) - { + for (final String path : zipPaths) { final File file = resolveTestResource(path); final FileInputStream fileInputStream = new FileInputStream(file); final List subscriptionItems = subscriptionExtractor.fromZipInputStream(fileInputStream); @@ -141,14 +145,13 @@ public void fromZipInputStream() throws Exception { } @Test - public void fromCsvInputStream() throws Exception { + void fromCsvInputStream() throws Exception { final List csvPaths = Arrays.asList( "youtube_takeout_import_test_1.csv", "youtube_takeout_import_test_2.csv" ); - for (String path : csvPaths) - { + for (final String path : csvPaths) { final File file = resolveTestResource(path); final FileInputStream fileInputStream = new FileInputStream(file); final List subscriptionItems = subscriptionExtractor.fromCsvInputStream(fileInputStream); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java index cc2b3111e0..f762611736 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeStreamExtractor; import java.util.Random; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java index 11312fcf10..4e98191555 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java @@ -1,18 +1,19 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.fail; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mozilla.javascript.EvaluatorException; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeThrottlingDecrypter; import java.io.IOException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.fail; - public class YoutubeThrottlingDecrypterTest { @BeforeEach @@ -30,7 +31,7 @@ public void testExtractFunction__success() throws ParsingException { try { final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, videoId); assertNotEquals(encryptedUrl, decryptedUrl); - } catch (EvaluatorException e) { + } catch (final EvaluatorException e) { fail("Failed to extract n param decrypt function for video " + videoId + "\n" + e); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java index 77696a04c2..165d27021e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java @@ -31,6 +31,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.kiosk.KioskInfo; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; /** * Test for {@link KioskInfo} @@ -46,7 +47,7 @@ public static void setUp() throws Exception { YoutubeTestsUtils.ensureStateless(); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH)); - LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending"); + final LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending"); kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java index 55af5679f0..2c0b9a8ede 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java @@ -20,18 +20,18 @@ * along with NewPipe. If not, see . */ +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; /** * Test for {@link YoutubeTrendingLinkHandlerFactory} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index bb3a78e287..68eaca7036 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -6,13 +6,12 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.schabi.newpipe.downloader.DownloaderTestImpl; -import org.schabi.newpipe.downloader.MockOnly; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; import java.net.URLEncoder; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 29f88c498f..f4ce076f6d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -7,9 +7,9 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; import static java.util.Collections.singletonList; import org.junit.jupiter.api.BeforeAll; @@ -211,6 +211,7 @@ public static void setUp() throws Exception { // Test Overrides //////////////////////////////////////////////////////////////////////////*/ + @Override @Test public void testMoreRelatedItems() throws Exception { final ListExtractor.InfoItemsPage initialPage = extractor().getInitialPage(); @@ -301,7 +302,7 @@ public static void setUp() throws Exception { public void testAtLeastOneVerified() throws IOException, ExtractionException { final List items = extractor.getInitialPage().getItems(); boolean verified = false; - for (InfoItem item : items) { + for (final InfoItem item : items) { if (((ChannelInfoItem) item).isVerified()) { verified = true; break; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index f0aaa5cf11..20f82140e6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -5,10 +5,10 @@ import static java.util.Arrays.asList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; public class YoutubeSearchQHTest { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index 085fce5be8..7d0b7d3fa5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index f1506154b6..4163951a44 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -42,7 +42,7 @@ import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamSegment; From c006c9c534f6453a7898b0cd157586c5270d3a38 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Fri, 15 Apr 2022 15:59:29 +0200 Subject: [PATCH 03/21] Abstract (instance) service names into constant --- .../extractor/services/peertube/PeertubeInstance.java | 4 ++-- .../extractor/services/peertube/PeertubeService.java | 2 +- .../services/youtube/invidious/InvidiousInstance.java | 4 +++- .../services/youtube/invidious/InvidiousService.java | 6 +----- .../services/youtube/youtube/YoutubeDirectService.java | 2 +- .../extractor/services/youtube/youtube/YoutubeInstance.java | 4 +++- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index c9e9959d42..728d61622a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -12,13 +12,13 @@ public class PeertubeInstance extends AbstractInstance { - public static final String INSTANCE_SERVICE_NAME = "Peertube"; + public static final String SERVICE_NAME = "Peertube"; public static final PeertubeInstance DEFAULT_INSTANCE = new PeertubeInstance("https://framatube.org", "FramaTube"); public PeertubeInstance(final String url) { - super(url, INSTANCE_SERVICE_NAME); + super(url, SERVICE_NAME); } public PeertubeInstance(final String url, final String name) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 7ffb4c709a..dd13438b7d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -48,7 +48,7 @@ public PeertubeService(final int id) { } public PeertubeService(final int id, final PeertubeInstance instance) { - super(id, "PeerTube", asList(VIDEO, COMMENTS)); + super(id, PeertubeInstance.SERVICE_NAME, asList(VIDEO, COMMENTS)); this.instance = instance; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java index 713a502ba2..fb16b08c50 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java @@ -12,6 +12,8 @@ public class InvidiousInstance extends YoutubeLikeInstance { + public static final String SERVICE_NAME = "Invidious"; + public InvidiousInstance(final String url) { super(url, tryExtractDomainFromUrl(url, "Invidious instance" + url.hashCode())); } @@ -74,7 +76,7 @@ public InvidiousService getNewStreamingService(final int id) { @Override public String getServiceName() { - return "Invidious"; + return SERVICE_NAME; } @FunctionalInterface diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java index 19952b2be4..5524ff3987 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java @@ -51,12 +51,8 @@ public class InvidiousService extends YoutubeLikeStreamingService { protected InvidiousInstance instance; - public InvidiousService(final int id) { - super(id, "Invidious", asList(AUDIO, VIDEO, LIVE, COMMENTS)); - } - public InvidiousService(final int id, final InvidiousInstance instance) { - super(id, "Invidious", asList(AUDIO, VIDEO, LIVE, COMMENTS)); + super(id, InvidiousInstance.SERVICE_NAME, asList(AUDIO, VIDEO, LIVE, COMMENTS)); this.instance = instance; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeDirectService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeDirectService.java index 9a6d4abeac..e0101b64f8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeDirectService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeDirectService.java @@ -71,7 +71,7 @@ public class YoutubeDirectService extends YoutubeLikeStreamingService { public YoutubeDirectService(final int id) { - super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS)); + super(id, YoutubeInstance.SERVICE_NAME, asList(AUDIO, VIDEO, LIVE, COMMENTS)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeInstance.java index ed98987f1c..f72d385f76 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeInstance.java @@ -4,8 +4,10 @@ public class YoutubeInstance extends YoutubeLikeInstance { + public static final String SERVICE_NAME = "YouTube"; + public static final YoutubeInstance YOUTUBE = - new YoutubeInstance("https://youtube.com", "YouTube"); + new YoutubeInstance("https://youtube.com", SERVICE_NAME); protected YoutubeInstance(final String url, final String name) { super(url, name); From a001a06b1bb0037c488b97586e92ed46c97a7f15 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Fri, 15 Apr 2022 21:42:06 +0200 Subject: [PATCH 04/21] Cleanup * Removed unused code * Fixed checkstyle problems * Added some docs --- .../schabi/newpipe/extractor/ServiceList.java | 2 +- .../extractor/instance/AbstractInstance.java | 2 -- .../newpipe/extractor/instance/Instance.java | 5 ----- .../InstanceMetaDataFetchException.java | 3 +++ .../services/bandcamp/BandcampService.java | 1 - .../services/media_ccc/MediaCCCService.java | 1 - .../services/peertube/PeertubeInstance.java | 2 +- .../services/youtube/YoutubeLikeInstance.java | 5 +++++ .../youtube/YoutubeLikeStreamingService.java | 8 ++++---- .../youtube/invidious/InvidiousInstance.java | 4 ++-- .../invidious/InvidiousParsingHelper.java | 4 ++-- .../extractors/InvidiousChannelExtractor.java | 8 ++++++-- .../extractors/InvidiousFeedExtractor.java | 8 ++++++-- .../extractors/InvidiousPlaylistExtractor.java | 11 +++++++---- .../extractors/InvidiousSearchExtractor.java | 5 ++++- .../extractors/InvidiousStreamExtractor.java | 4 ++-- .../extractors/InvidiousTrendingExtractor.java | 4 +++- .../InvidiousPlaylistLinkHandlerFactory.java | 2 +- .../InvidiousTrendingLinkHandlerFactory.java | 2 +- .../YoutubeTakeoutSubscriptionExtractor.java | 5 +++-- .../YoutubeLikePlaylistLinkHandlerFactory.java | 5 +++-- .../youtube/youtube/YoutubeParsingHelper.java | 18 +++++------------- .../youtube/YoutubeThrottlingDecrypter.java | 2 ++ .../extractors/YoutubeCommentsExtractor.java | 1 + .../YoutubeMixPlaylistExtractor.java | 3 ++- .../YoutubeMusicSearchExtractor.java | 1 + 26 files changed, 65 insertions(+), 51 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 9ddaac4543..2892ea82a3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -31,7 +31,7 @@ /** * A list of supported services. */ -// keep unusual names and inner assignments +@SuppressWarnings("ConstantName") // keep unusual names public final class ServiceList { private ServiceList() { //no instance diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/AbstractInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/AbstractInstance.java index f3e3f2b709..75ea3e59c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/AbstractInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/AbstractInstance.java @@ -9,8 +9,6 @@ public abstract class AbstractInstance implements Instance { - protected Boolean valid = null; - protected final String url; protected String name; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/Instance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/Instance.java index 703d4bf793..3c09dcdf92 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/Instance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/Instance.java @@ -17,17 +17,12 @@ public interface Instance { * The base url of this instance. *
* Note that the url is returned without trailing slashes. - * Use {@link #getUrlWithTrailingSlash()} if you want a trailing slash. * * @return base url */ @Nonnull String getUrl(); - default String getUrlWithTrailingSlash() { - return getUrl() + "/"; - } - /** * Fetch instance metadata. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/InstanceMetaDataFetchException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/InstanceMetaDataFetchException.java index 7afaa5b4b2..afbe1def32 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/instance/InstanceMetaDataFetchException.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/instance/InstanceMetaDataFetchException.java @@ -1,5 +1,8 @@ package org.schabi.newpipe.extractor.instance; +/** + * Thrown when an {@link Instance} couldn't be validated. + */ public class InstanceMetaDataFetchException extends RuntimeException { public InstanceMetaDataFetchException(final String message) { super(message); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index a1e178f5b7..659d41da0c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -13,7 +13,6 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 25e49bbaef..8738342f13 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 728d61622a..fc31329c18 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -42,7 +42,7 @@ public void fetchMetadata() { this.setName(JsonUtils.getString(json, "instance.name")); } catch (final Exception e) { throw new InstanceMetaDataFetchException( - "Unable to fetch trending from " + urlToCheck, e); + "Unable to fetch config from " + urlToCheck, e); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeInstance.java index 6a3af903a3..785fd2a58a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeInstance.java @@ -2,6 +2,11 @@ import org.schabi.newpipe.extractor.instance.AbstractInstance; +/** + * A YouTube-like instance (or proxy) + * + * @param + */ public abstract class YoutubeLikeInstance extends AbstractInstance { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeStreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeStreamingService.java index 2eacfa95f9..009e65283d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeStreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeLikeStreamingService.java @@ -33,15 +33,15 @@ protected YoutubeLikeStreamingService( } @Override - public abstract StreamExtractor getStreamExtractor(final LinkHandler linkHandler); + public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler); @Override - public abstract ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler); + public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler); @Override - public abstract PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler); + public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler); @Nonnull @Override - public abstract FeedExtractor getFeedExtractor(final String url) throws ExtractionException; + public abstract FeedExtractor getFeedExtractor(String url) throws ExtractionException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java index fb16b08c50..ba216559fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java @@ -65,7 +65,7 @@ private void checkEndpoint( responseCheck.check(response); } catch (final Exception e) { throw new InstanceMetaDataFetchException( - "Unable to fetch trending from " + urlToCheck, e); + "Unable to fetch from " + urlToCheck, e); } } @@ -81,6 +81,6 @@ public String getServiceName() { @FunctionalInterface public interface ResponseCheck { - void check(final Response r) throws Exception; + void check(Response r) throws Exception; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java index daca29af4b..71fb258de5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java @@ -142,13 +142,13 @@ public static String getThumbnailUrl(final JsonArray thumbnails) { return fixThumbnailUrl(url); } - public static String fixThumbnailUrl(String thumbnailUrl) { + public static String fixThumbnailUrl(final String thumbnailUrl) { if (thumbnailUrl.startsWith(HTTP) || thumbnailUrl.startsWith(HTTPS)) { return thumbnailUrl; } if (thumbnailUrl.startsWith("//")) { - thumbnailUrl = thumbnailUrl.substring(2); + return HTTPS + thumbnailUrl.substring(2); } return HTTPS + thumbnailUrl; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java index 1165aac5db..02add6f2b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java @@ -49,7 +49,9 @@ protected Page getPage(final int page) { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage( + final Page page + ) throws IOException, ExtractionException { final Response rp = NewPipe.getDownloader().get(page.getUrl()); final JsonArray array = InvidiousParsingHelper.getValidJsonArrayFromResponse(rp, page.getUrl()); @@ -71,7 +73,9 @@ public InfoItemsPage getPage(final Page page) throws IOException } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage( + @Nonnull final Downloader downloader + ) throws IOException, ExtractionException { final String apiUrl = baseUrl + "/api/v1/channels/" + getUid(getId()) + "?fields=author,description,subCount,authorThumbnails,authorBanners,authorId" + "®ion=" + getExtractorContentCountry().getCountryCode(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java index ee17e7d33f..8cd7ef6c96 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java @@ -51,12 +51,16 @@ public InfoItemsPage getInitialPage() { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage( + final Page page + ) throws IOException, ExtractionException { return null; } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage( + @Nonnull final Downloader downloader + ) throws IOException, ExtractionException { final String feedUrl = baseUrl + "/feed/channel/" + getLinkHandler().getId(); final Response response = downloader.get(feedUrl); document = Jsoup.parse(getValidResponseBody(response, feedUrl)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java index 311d6db0aa..f89337cfca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java @@ -67,7 +67,9 @@ public InfoItemsPage getInitialPage() throws IOException, Extrac } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage( + final Page page + ) throws IOException, ExtractionException { if (Integer.parseInt(page.getId()) != 1) { final Response rp = NewPipe.getDownloader().get(page.getUrl()); json = InvidiousParsingHelper.getValidJsonObjectFromResponse(rp, page.getUrl()); @@ -92,12 +94,13 @@ public InfoItemsPage getPage(final Page page) throws IOException } public Page getPage(final int page) throws ParsingException { - return InvidiousParsingHelper.getPage(baseUrl + "/api/v1/playlists/" + - getId(), page); + return InvidiousParsingHelper.getPage(baseUrl + "/api/v1/playlists/" + getId(), page); } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage( + @Nonnull final Downloader downloader + ) throws IOException, ExtractionException { final String apiUrl = baseUrl + "/api/v1/playlists/" + getId(); final Response response = downloader.get(apiUrl); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java index c316b8df48..30b902bdf6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java @@ -73,7 +73,10 @@ public void onFetchPage( results = InvidiousParsingHelper.getValidJsonArrayFromResponse(response, getUrl()); } - private void collectStreamsFrom(final MultiInfoItemsCollector collector, final JsonObject json) { + private void collectStreamsFrom( + final MultiInfoItemsCollector collector, + final JsonObject json + ) { final String type = json.getString("type"); switch (type) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java index 17949e82e6..37ccc02b48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java @@ -53,8 +53,8 @@ public InvidiousStreamExtractor(final InvidiousService service, final LinkHandle public void onFetchPage( @Nonnull final Downloader downloader ) throws IOException, ExtractionException { - final String apiUrl = baseUrl + "/api/v1/videos/" + getId() + - "?region=" + getExtractorContentCountry().getCountryCode(); + final String apiUrl = baseUrl + "/api/v1/videos/" + getId() + + "?region=" + getExtractorContentCountry().getCountryCode(); final Response response = downloader.get(apiUrl); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java index b8bb28eef0..886c533d04 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java @@ -64,7 +64,9 @@ public InfoItemsPage getInitialPage() throws IOException, Extrac } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage( + final Page page + ) throws IOException, ExtractionException { return InfoItemsPage.emptyPage(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java index 7bf1f5a629..62a65e3da1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java @@ -16,7 +16,7 @@ public InvidiousPlaylistLinkHandlerFactory(final InvidiousService service) { @Override public String getUrl(final String id, final List contentFilters, final String sortFilter) { - return baseUrl +"/playlist?list=" + id; + return baseUrl + "/playlist?list=" + id; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousTrendingLinkHandlerFactory.java index c8fef271a0..01e64cc4f6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousTrendingLinkHandlerFactory.java @@ -41,7 +41,7 @@ public String getUrl(final String id, public String getUrl(final String id, final List contentFilters, final String sortFilter, - final String baseUrl) { + @SuppressWarnings("HiddenField") final String baseUrl) { return String.format(KIOSK_MAP.get(id), baseUrl); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/extractors/YoutubeTakeoutSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/extractors/YoutubeTakeoutSubscriptionExtractor.java index 1731c9ac3d..f0a6ee2eaf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/extractors/YoutubeTakeoutSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/extractors/YoutubeTakeoutSubscriptionExtractor.java @@ -102,7 +102,7 @@ public List fromJsonInputStream(@Nonnull final InputStream con public List fromZipInputStream(@Nonnull final InputStream contentInputStream) throws ExtractionException { - try (final ZipInputStream zipInputStream = new ZipInputStream(contentInputStream)) { + try (ZipInputStream zipInputStream = new ZipInputStream(contentInputStream)) { ZipEntry zipEntry; while ((zipEntry = zipInputStream.getNextEntry()) != null) { if (zipEntry.getName().toLowerCase().endsWith(".csv")) { @@ -143,7 +143,8 @@ public List fromCsvInputStream(@Nonnull final InputStream cont int currentLine = 0; String line = ""; - try (final BufferedReader br = new BufferedReader(new InputStreamReader(contentInputStream))) { + try (BufferedReader br = + new BufferedReader(new InputStreamReader(contentInputStream))) { final List subscriptionItems = new ArrayList<>(); // ignore header and skip first line diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java index 152947ea3c..303ad68904 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java @@ -69,11 +69,12 @@ public boolean onAcceptUrl(final String url) { * Returns the url for the mix-playlist. *
* Example for Youtube: {@code https://youtube.com/watch?v=videoId&list=playlistId} + * * @param videoID id of the video - * @param listID id of the list + * @param listID id of the list * @return url for the mix-playlist */ - protected abstract String getMixUrl(final String videoID, final String listID); + protected abstract String getMixUrl(String videoID, String listID); /** * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeParsingHelper.java index ff8b66ca8b..4a29090411 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeParsingHelper.java @@ -1211,8 +1211,9 @@ private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelC final String metaInfoLinkUrl = YoutubeParsingHelper.getUrlFromNavigationEndpoint( infoPanelContentRenderer.getObject("sourceEndpoint")); try { - metaInfo.addUrl(new URL(Objects.requireNonNull(YoutubeUrlHelper.extractCachedUrlIfNeeded( - metaInfoLinkUrl)))); + metaInfo.addUrl(new URL( + Objects.requireNonNull(YoutubeUrlHelper.extractCachedUrlIfNeeded( + metaInfoLinkUrl)))); } catch (final NullPointerException | MalformedURLException e) { throw new ParsingException("Could not get metadata info URL", e); } @@ -1249,7 +1250,8 @@ private static MetaInfo getClarificationRendererContent( try { final String url = YoutubeParsingHelper.getUrlFromNavigationEndpoint(actionButton .getObject("command")); - metaInfo.addUrl(new URL(Objects.requireNonNull(YoutubeUrlHelper.extractCachedUrlIfNeeded(url)))); + metaInfo.addUrl(new URL( + Objects.requireNonNull(YoutubeUrlHelper.extractCachedUrlIfNeeded(url)))); } catch (final NullPointerException | MalformedURLException e) { throw new ParsingException("Could not get metadata info URL", e); } @@ -1299,16 +1301,6 @@ public static boolean isVerified(final JsonArray badges) { return false; } - @Nonnull - public static String unescapeDocument(@Nonnull final String doc) { - return doc - .replaceAll("\\\\x22", "\"") - .replaceAll("\\\\x7b", "{") - .replaceAll("\\\\x7d", "}") - .replaceAll("\\\\x5b", "[") - .replaceAll("\\\\x5d", "]"); - } - /** * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in * playback requests (and also for some clients, in the player request body). diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeThrottlingDecrypter.java index 8b8f13e1e8..75bfb54ae2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeThrottlingDecrypter.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/YoutubeThrottlingDecrypter.java @@ -41,7 +41,9 @@ public class YoutubeThrottlingDecrypter { "\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]+)(?:\\[(\\d+)])?\\([a-zA-Z0-9]\\)"); private static final Map N_PARAMS_CACHE = new HashMap<>(); + @SuppressWarnings("StaticVariableName") private static String FUNCTION; + @SuppressWarnings("StaticVariableName") private static String FUNCTION_NAME; private final String functionName; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsExtractor.java index fd4c0619e3..eeb97d2df3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeCommentsExtractor.java @@ -42,6 +42,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { * If the method or another one that is depending on disabled comments * is now called again, the method execution can avoid unnecessary calls */ + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private Optional optCommentsDisabled = Optional.empty(); public YoutubeCommentsExtractor( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixPlaylistExtractor.java index f730082707..d1a0e03502 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -245,7 +245,8 @@ public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException { @Nonnull public static String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) throws ParsingException { - return getThumbnailUrlFromVideoId(YoutubePlaylistHelper.extractVideoIdFromMixId(playlistId)); + return getThumbnailUrlFromVideoId( + YoutubePlaylistHelper.extractVideoIdFromMixId(playlistId)); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMusicSearchExtractor.java index 98f5faa1f2..cc694200d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -280,6 +280,7 @@ public InfoItemsPage getPage(final Page page) return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } + @SuppressWarnings("MethodLength") private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector, @Nonnull final JsonArray videos) { final TimeAgoParser timeAgoParser = getTimeAgoParser(); From cbbb8c36a4e0bc71c6d7df0c9fe2e9bf9a30db12 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Fri, 15 Apr 2022 22:44:29 +0200 Subject: [PATCH 05/21] Fix tests and url resolution --- .../youtube/shared/YoutubeUrlHelper.java | 4 +- .../YoutubeLikeStreamLinkHandlerFactory.java | 18 ++++-- .../schabi/newpipe/extractor/utils/Utils.java | 8 +-- .../youtube/YoutubeChannelExtractorTest.java | 23 ++++---- .../YoutubeChannelLinkHandlerFactoryTest.java | 20 +++---- ...YoutubeCommentsLinkHandlerFactoryTest.java | 22 +++---- ...YoutubePlaylistLinkHandlerFactoryTest.java | 58 +++++++++---------- .../YoutubeStreamLinkHandlerFactoryTest.java | 56 +++++++++--------- ...YoutubeTrendingLinkHandlerFactoryTest.java | 6 +- 9 files changed, 111 insertions(+), 104 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java index 46ce44e380..0128caf7b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java @@ -62,12 +62,12 @@ public static boolean isInvidioURL(@Nonnull final URL url) { "invidio.xamh.de", "y.com.cm" )); - final String host = Utils.removeMAndWWWFromUrl(url.getHost()).toLowerCase(); + final String host = Utils.removeMAndWWWFromHost(url.getHost()).toLowerCase(); return invInstances.contains(host); } public static boolean isY2ubeURL(@Nonnull final URL url) { - return Utils.removeMAndWWWFromUrl(url.getHost()).equalsIgnoreCase("y2u.be"); + return Utils.removeMAndWWWFromHost(url.getHost()).equalsIgnoreCase("y2u.be"); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java index de1e946929..3a0eb1f101 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java @@ -118,11 +118,19 @@ public String getId(final String theUrlString) final String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); return assertIsId(viewQueryValue); - } else if ((isInvidioURL(url) || isHooktubeURL(url)) - && path.equalsIgnoreCase("watch")) { - final String viewQueryValue = Utils.getQueryValue(url, "v"); - if (viewQueryValue != null) { - return assertIsId(viewQueryValue); + } else if (isInvidioURL(url) || isHooktubeURL(url)) { + // /watch?v= + if ("watch".equalsIgnoreCase(path)) { + final String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsId(viewQueryValue); + } + } + + // / + final String maybeId = extractId(path); + if (maybeId != null) { + return maybeId; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 14b5691e01..ac13283f4f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -201,11 +201,11 @@ public static boolean isHTTP(@Nonnull final URL url) { } public static String removeMAndWWWFromHost(final String host) { - if (host.startsWith("m.")) { - return host.replace("m.", ""); + if (host.toLowerCase().startsWith("m.")) { + return host.length() > 2 ? host.substring(2) : ""; } - if (host.startsWith("www.")) { - return host.replace("www.", ""); + if (host.toLowerCase().startsWith("www.")) { + return host.length() > 4 ? host.substring(4) : ""; } return host; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 4fa72b20e7..c71fb572f5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -24,11 +24,9 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; -import org.schabi.newpipe.extractor.services.youtube.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.youtube.extractors.YoutubeChannelExtractor; import java.io.IOException; -import java.util.Random; /** * Test for {@link ChannelExtractor} @@ -45,7 +43,7 @@ public static void setUp() throws IOException { } @Test - public void deletedFetch() throws Exception { + void deletedFetch() throws Exception { final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/UCAUc4iz6edWerIjlnL8OSSw"); @@ -53,7 +51,7 @@ public void deletedFetch() throws Exception { } @Test - public void nonExistentFetch() throws Exception { + void nonExistentFetch() throws Exception { final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST"); @@ -61,7 +59,7 @@ public void nonExistentFetch() throws Exception { } @Test - public void accountTerminatedTOSFetch() throws Exception { + void accountTerminatedTOSFetch() throws Exception { // "This account has been terminated for a violation of YouTube's Terms of Service." final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ"); @@ -72,7 +70,7 @@ public void accountTerminatedTOSFetch() throws Exception { } @Test - public void accountTerminatedCommunityFetch() throws Exception { + void accountTerminatedCommunityFetch() throws Exception { // "This account has been terminated for violating YouTube's Community Guidelines." final ChannelExtractor extractor = YouTube.getChannelExtractor("https://www.youtube.com/channel/UC0AuOxCr9TZ0TtEgL1zpIgA"); @@ -83,7 +81,7 @@ public void accountTerminatedCommunityFetch() throws Exception { } @Test - public void accountTerminatedHateFetch() throws Exception { + void accountTerminatedHateFetch() throws Exception { // "This account has been terminated due to multiple or severe violations // of YouTube's policy prohibiting hate speech." final ChannelExtractor extractor = @@ -95,7 +93,7 @@ public void accountTerminatedHateFetch() throws Exception { } @Test - public void accountTerminatedBullyFetch() throws Exception { + void accountTerminatedBullyFetch() throws Exception { // "This account has been terminated due to multiple or severe violations // of YouTube's policy prohibiting content designed to harass, bully or threaten." final ChannelExtractor extractor = @@ -107,7 +105,7 @@ public void accountTerminatedBullyFetch() throws Exception { } @Test - public void accountTerminatedSpamFetch() throws Exception { + void accountTerminatedSpamFetch() throws Exception { // "This account has been terminated due to multiple or severe violations // of YouTube's policy against spam, deceptive practices and misleading content // or other Terms of Service violations." @@ -120,7 +118,7 @@ public void accountTerminatedSpamFetch() throws Exception { } @Test - public void accountTerminatedCopyrightFetch() throws Exception { + void accountTerminatedCopyrightFetch() throws Exception { // "This account has been terminated because we received multiple third-party claims // of copyright infringement regarding material that the user posted." final ChannelExtractor extractor = @@ -142,7 +140,8 @@ public static void setUp() throws IOException { @Test void noVideoTab() throws Exception { - final ChannelExtractor extractor = YouTube.getChannelExtractor("https://invidio.us/channel/UC-9-kyTW8ZkZNDHQJ6FgpwQ"); + final ChannelExtractor extractor = + YouTube.getChannelExtractor("https://invidious.fdn.fr/channel/UC-9-kyTW8ZkZNDHQJ6FgpwQ"); extractor.fetchPage(); assertThrows(ContentNotSupportedException.class, extractor::getInitialPage); @@ -489,7 +488,7 @@ public static void setUp() throws Exception { } @Test - public void testGetPageInNewExtractor() throws Exception { + void testGetPageInNewExtractor() throws Exception { final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index f8f57012ff..df5175f9ca 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -26,7 +26,7 @@ public static void setUp() { } @Test - public void acceptUrlTest() throws ParsingException { + void acceptUrlTest() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://www.youtube.com/user/Gronkh")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/user/Netzkino/videos")); @@ -43,11 +43,11 @@ public void acceptUrlTest() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); - assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Gronkh")); - assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Netzkino/videos")); + assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/user/Gronkh")); + assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/user/Netzkino/videos")); - assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA")); - assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/channel/UClq42foiSgl7sSpLupnugGA")); + assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watchismo")); @@ -63,7 +63,7 @@ public void acceptUrlTest() throws ParsingException { } @Test - public void getIdFromUrl() throws ParsingException { + void getIdFromUrl() throws ParsingException { assertEquals("user/Gronkh", linkHandler.fromUrl("https://www.youtube.com/user/Gronkh").getId()); assertEquals("user/Netzkino", linkHandler.fromUrl("https://www.youtube.com/user/Netzkino/videos").getId()); @@ -77,11 +77,11 @@ public void getIdFromUrl() throws ParsingException { assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); - assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidio.us/user/Gronkh").getId()); - assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidio.us/user/Netzkino/videos").getId()); + assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidious.fdn.fr/user/Gronkh").getId()); + assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidious.fdn.fr/user/Netzkino/videos").getId()); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId()); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidious.fdn.fr/channel/UClq42foiSgl7sSpLupnugGA").getId()); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidious.fdn.fr/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); assertEquals("c/creatoracademy", linkHandler.fromUrl("https://www.youtube.com/c/creatoracademy").getId()); assertEquals("c/YouTubeCreators", linkHandler.fromUrl("https://www.youtube.com/c/YouTubeCreators").getId()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java index 365254547a..7ee95ac10b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java @@ -23,12 +23,12 @@ public static void setUp() { } @Test - public void getIdWithNullAsUrl() { + void getIdWithNullAsUrl() { assertThrows(IllegalArgumentException.class, () -> linkHandler.fromId(null)); } @Test - public void getIdFromYt() throws ParsingException { + void getIdFromYt() throws ParsingException { assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.youtube.com/watch?v=VM_6n762j6M").getId()); assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://m.youtube.com/watch?v=VM_6n762j6M").getId()); assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtube.com/watch?v=VM_6n762j6M").getId()); @@ -38,7 +38,7 @@ public void getIdFromYt() throws ParsingException { } @Test - public void testAcceptUrl() throws ParsingException { + void testAcceptUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=VM_6n762j6M&t=20")); assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/watch?v=VM_6n762j6M&t=20")); assertTrue(linkHandler.acceptUrl("https://youtube.com/watch?v=VM_6n762j6M&t=20")); @@ -46,7 +46,7 @@ public void testAcceptUrl() throws ParsingException { } @Test - public void testDeniesUrl() throws ParsingException { + void testDeniesUrl() throws ParsingException { assertFalse(linkHandler.acceptUrl("https://www.you com/watch?v=VM_6n762j6M")); assertFalse(linkHandler.acceptUrl("https://com/watch?v=VM_6n762j6M")); assertFalse(linkHandler.acceptUrl("htt ://com/watch?v=VM_6n762j6M")); @@ -54,16 +54,16 @@ public void testDeniesUrl() throws ParsingException { } @Test - public void getIdFromInvidious() throws ParsingException { - assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.invidio.us/watch?v=VM_6n762j6M").getId()); - assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/watch?v=VM_6n762j6M").getId()); - assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://INVIDIO.US/watch?v=VM_6n762j6M").getId()); - assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/VM_6n762j6M").getId()); - assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/VM_6n762j6M&t=20").getId()); + void getIdFromInvidious() throws ParsingException { + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.invidious.fdn.fr/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidious.fdn.fr/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://INVIDIOUS.FDN.FR/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidious.fdn.fr/VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidious.fdn.fr/VM_6n762j6M&t=20").getId()); } @Test - public void getIdFromY2ube() throws ParsingException { + void getIdFromY2ube() throws ParsingException { assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://y2u.be/VM_6n762j6M").getId()); assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://Y2U.Be/VM_6n762j6M").getId()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 23883b74f3..a6dd4e7f14 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -25,12 +25,12 @@ public static void setUp() { } @Test - public void getIdWithNullAsUrl() { + void getIdWithNullAsUrl() { assertThrows(IllegalArgumentException.class, () -> linkHandler.fromId(null)); } @Test - public void getIdfromYt() throws Exception { + void getIdfromYt() throws Exception { assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); @@ -46,7 +46,7 @@ public void getIdfromYt() throws Exception { } @Test - public void testAcceptYtUrl() throws ParsingException { + void testAcceptYtUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI")); @@ -63,7 +63,7 @@ public void testAcceptYtUrl() throws ParsingException { } @Test - public void testDeniesInvalidYtUrl() throws ParsingException { + void testDeniesInvalidYtUrl() throws ParsingException { assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertFalse(linkHandler.acceptUrl("ftp://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); @@ -73,20 +73,20 @@ public void testDeniesInvalidYtUrl() throws ParsingException { } @Test - public void testAcceptInvidioUrl() throws ParsingException { - assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); - assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); - assertTrue(linkHandler.acceptUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI")); - assertTrue(linkHandler.acceptUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); - assertTrue(linkHandler.acceptUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); - assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); - assertTrue(linkHandler.acceptUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); - assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); - assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + void testAcceptInvidioUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://www.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://www.invidious.fdn.fr/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertTrue(linkHandler.acceptUrl("https://WWW.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI")); + assertTrue(linkHandler.acceptUrl("HTTPS://www.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://www.invidious.fdn.fr/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("http://www.invidious.fdn.fr/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("www.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); + assertTrue(linkHandler.acceptUrl("www.invidious.fdn.fr/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); } @Test - public void testDeniesInvalidInvidioUrl() throws ParsingException { + void testDeniesInvalidInvidioUrl() throws ParsingException { assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertFalse(linkHandler.acceptUrl("ftp:/invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); @@ -96,21 +96,21 @@ public void testDeniesInvalidInvidioUrl() throws ParsingException { } @Test - public void testGetInvidioIdfromUrl() throws ParsingException { - assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); - assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); - assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); - assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); - assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); - assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); - assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); - assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); - assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); - assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); + void testGetInvidioIdfromUrl() throws ParsingException { + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.invidious.fdn.fr/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidious.fdn.fr/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.invidious.fdn.fr/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidious.fdn.fr/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); + assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidious.fdn.fr/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); } @Test - public void fromUrlIsMixVideo() throws Exception { + void fromUrlIsMixVideo() throws Exception { final String videoId = "_AzeUSL9lZc"; String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; assertEquals(url, linkHandler.fromUrl(url).getUrl()); @@ -121,10 +121,10 @@ public void fromUrlIsMixVideo() throws Exception { } @Test - public void fromUrlIsMixPlaylist() throws Exception { + void fromUrlIsMixPlaylist() throws Exception { final String videoId = "_AzeUSL9lZc"; final String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; assertEquals(url, - linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl()); + linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl()); } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 00bba32639..0ebc7e989e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -30,23 +30,23 @@ public static void setUp() { } @Test - public void getIdWithNullAsUrl() { + void getIdWithNullAsUrl() { assertThrows(IllegalArgumentException.class, () -> linkHandler.fromId(null)); } @Test - public void getIdForAd() throws ParsingException { + void getIdForAd() throws ParsingException { assertThrows(FoundAdException.class, () -> linkHandler.fromUrl(AD_URL)); } @Test - public void getIdForInvalidUrls() { + void getIdForInvalidUrls() { final List invalidUrls = new ArrayList<>(50); invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d"); invalidUrls.add("https://www.youtube.com/watchjZViOEv90d"); invalidUrls.add("https://www.youtube.com/"); invalidUrls.add("https://www.youtube.com/channel/UCBR8-60-B28hp2BmDPdntcQ"); - invalidUrls.add("https://invidio.us/channel/UCBR8-60-B28hp2BmDPdntcQ"); + invalidUrls.add("https://invidious.fdn.fr/channel/UCBR8-60-B28hp2BmDPdntcQ"); for (final String invalidUrl : invalidUrls) { Throwable exception = null; try { @@ -61,7 +61,7 @@ public void getIdForInvalidUrls() { } @Test - public void getIdfromYt() throws Exception { + void getIdfromYt() throws Exception { assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube.com/watch?v=jZViOEv90dI").getId()); assertEquals("W-fFHeTX70Q", linkHandler.fromUrl("https://www.youtube.com/watch?v=W-fFHeTX70Q").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100").getId()); @@ -101,7 +101,7 @@ public void getIdfromYt() throws Exception { } @Test - public void testAcceptYtUrl() throws ParsingException { + void testAcceptYtUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100")); assertTrue(linkHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI&t=100")); @@ -125,7 +125,7 @@ public void testAcceptYtUrl() throws ParsingException { } @Test - public void testAcceptHookUrl() throws ParsingException { + void testAcceptHookUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU")); assertTrue(linkHandler.acceptUrl("http://hooktube.com/watch?v=TglNG-yjabU")); assertTrue(linkHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE")); @@ -136,7 +136,7 @@ public void testAcceptHookUrl() throws ParsingException { } @Test - public void testGetHookIdfromUrl() throws ParsingException { + void testGetHookIdfromUrl() throws ParsingException { assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId()); assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://hooktube.com/watch?v=TglNG-yjabU").getId()); assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch?v=3msbfr6pBNE").getId()); @@ -148,28 +148,28 @@ public void testGetHookIdfromUrl() throws ParsingException { } @Test - public void testAcceptInvidiousUrl() throws ParsingException { - assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU")); - assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=TglNG-yjabU")); - assertTrue(linkHandler.acceptUrl("http://invidio.us/watch?v=TglNG-yjabU")); - assertTrue(linkHandler.acceptUrl("invidio.us/watch?v=3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); - assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("invidio.us/watch/3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("invidio.us/v/3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("invidio.us/w/3msbfr6pBNE")); + void testAcceptInvidiousUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/watch?v=TglNG-yjabU")); + assertTrue(linkHandler.acceptUrl("http://www.invidious.fdn.fr/watch?v=TglNG-yjabU")); + assertTrue(linkHandler.acceptUrl("http://invidious.fdn.fr/watch?v=TglNG-yjabU")); + assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/watch?v=3msbfr6pBNE")); + assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); + assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/embed/3msbfr6pBNE")); + assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/watch/3msbfr6pBNE")); + assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/v/3msbfr6pBNE")); + assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/w/3msbfr6pBNE")); } @Test - public void testGetInvidiousIdfromUrl() throws ParsingException { - assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId()); - assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://www.invidio.us/watch?v=TglNG-yjabU").getId()); - assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://invidio.us/watch?v=TglNG-yjabU").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/watch?v=3msbfr6pBNE").getId()); - assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/embed/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/v/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/w/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/watch/3msbfr6pBNE").getId()); + void testGetInvidiousIdfromUrl() throws ParsingException { + assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidious.fdn.fr/watch?v=TglNG-yjabU").getId()); + assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://www.invidious.fdn.fr/watch?v=TglNG-yjabU").getId()); + assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://invidious.fdn.fr/watch?v=TglNG-yjabU").getId()); + assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/watch?v=3msbfr6pBNE").getId()); + assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://invidious.fdn.fr/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId()); + assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/embed/3msbfr6pBNE").getId()); + assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/v/3msbfr6pBNE").getId()); + assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/w/3msbfr6pBNE").getId()); + assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/watch/3msbfr6pBNE").getId()); } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java index 2c0b9a8ede..7cec68017d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java @@ -68,9 +68,9 @@ public void acceptUrl() throws ParsingException { assertTrue(LinkHandlerFactory.acceptUrl("https://youtube.com/feed/trending")); assertTrue(LinkHandlerFactory.acceptUrl("m.youtube.com/feed/trending")); - assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidio.us/feed/trending")); - assertTrue(LinkHandlerFactory.acceptUrl("https://invidio.us/feed/trending")); - assertTrue(LinkHandlerFactory.acceptUrl("invidio.us/feed/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidious.fdn.fr/feed/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("https://invidious.fdn.fr/feed/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("invidious.fdn.fr/feed/trending")); assertFalse(LinkHandlerFactory.acceptUrl("https://youtu.be/feed/trending")); assertFalse(LinkHandlerFactory.acceptUrl("kdskjfiiejfia")); From 668649c9dd27cfb1632b7de1e7aaaee75ab72be9 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Tue, 19 Apr 2022 22:09:38 +0200 Subject: [PATCH 06/21] Fix comment extraction --- .../extractors/InvidiousCommentsExtractor.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java index d1aad79d92..8a0e7706be 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java @@ -47,6 +47,10 @@ public InfoItemsPage getInitialPage() throws ExtractionExcepti public InfoItemsPage getPage( final Page page ) throws IOException, ExtractionException { + if (page == null) { + return InfoItemsPage.emptyPage(); + } + final Downloader dl = NewPipe.getDownloader(); final Response response = dl.get(page.getUrl()); @@ -59,8 +63,14 @@ public InfoItemsPage getPage( public Page getNextPage() throws ParsingException { + final String continuation = json.getString("continuation"); + + if (continuation == null || continuation.isEmpty()) { + return null; + } + return new Page(baseUrl + "/api/v1/comments/" + getId() - + "?continuation=" + json.getString("continuation")); + + "?continuation=" + continuation); } @Override From e39960b4da2675e9362f981b7f0ac062cb2ab395 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Tue, 19 Apr 2022 22:09:49 +0200 Subject: [PATCH 07/21] Fix search extraction --- .../invidious/extractors/InvidiousSearchExtractor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java index 30b902bdf6..dd4b58cc30 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java @@ -46,6 +46,10 @@ public InfoItemsPage getInitialPage() throws IOException, ExtractionEx public InfoItemsPage getPage( final Page page ) throws IOException, ExtractionException { + if (results == null) { + return InfoItemsPage.emptyPage(); + } + int pageNumber = 1; try { if (page.getId() != null) { From 76722e26fcaab74de44bd561485fca68dacd53ab Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Tue, 19 Apr 2022 22:10:24 +0200 Subject: [PATCH 08/21] Cleanup --- .../invidious/extractors/InvidiousStreamExtractor.java | 3 +-- .../linkHandler/YoutubeSearchQueryHandlerFactory.java | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java index 37ccc02b48..828ba9dea8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java @@ -95,8 +95,7 @@ public Description getDescription() { @Override public int getAgeLimit() { - final boolean isFamilyFriendly = json.getBoolean("isFamilyFriendly"); - return isFamilyFriendly ? NO_AGE_LIMIT : 18; + return json.getBoolean("isFamilyFriendly") ? NO_AGE_LIMIT : 18; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 4b08e6da40..82322aa938 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -41,9 +41,6 @@ public String getUrl(final String searchString, if (!contentFilters.isEmpty()) { final String contentFilter = contentFilters.get(0); switch (contentFilter) { - case ALL: - default: - break; case VIDEOS: return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D"; @@ -59,6 +56,9 @@ public String getUrl(final String searchString, case MUSIC_PLAYLISTS: case MUSIC_ARTISTS: return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8); + case ALL: + default: + break; } } From 60869852cea15697f6f464f431ea3445d34031e3 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Tue, 19 Apr 2022 22:11:11 +0200 Subject: [PATCH 09/21] Fix date format --- .../services/youtube/invidious/InvidiousParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java index 71fb258de5..dbcbc2e80e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java @@ -98,7 +98,7 @@ public static JsonArray getValidJsonArrayFromResponse( public static DateWrapper getUploadDateFromEpochTime(final long epochTime) { return new DateWrapper( - OffsetDateTime.ofInstant(Instant.ofEpochMilli(epochTime), ZoneOffset.UTC)); + OffsetDateTime.ofInstant(Instant.ofEpochSecond(epochTime), ZoneOffset.UTC)); } public static String getUid(@Nonnull final String id) { From ede2bbe132fe3a58565723179c5532aabe8c6af8 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Tue, 19 Apr 2022 22:11:50 +0200 Subject: [PATCH 10/21] Fixed link handling --- .../InvidiousChannelLinkHandlerFactory.java | 8 +++++++- .../InvidiousLinkHandlerFactory.java | 19 +++++++++++++++++++ .../InvidiousPlaylistLinkHandlerFactory.java | 8 +++++++- .../InvidiousStreamLinkHandlerFactory.java | 8 +++++++- .../YoutubeLikeChannelLinkHandlerFactory.java | 8 +++----- .../YoutubeLikeLinkHandlerFactory.java | 18 ++++++++++++++++++ ...YoutubeLikePlaylistLinkHandlerFactory.java | 7 +++---- .../YoutubeLikeStreamLinkHandlerFactory.java | 15 +++++---------- .../YoutubeChannelLinkHandlerFactory.java | 3 ++- .../YoutubeLinkHandlerFactory.java | 14 ++++++++++++++ .../YoutubePlaylistLinkHandlerFactory.java | 3 ++- .../YoutubeStreamLinkHandlerFactory.java | 3 ++- .../schabi/newpipe/extractor/utils/Utils.java | 8 ++++++++ 13 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeLinkHandlerFactory.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousChannelLinkHandlerFactory.java index ade00ddda3..c07e5ecf68 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousChannelLinkHandlerFactory.java @@ -7,7 +7,8 @@ import java.util.List; -public class InvidiousChannelLinkHandlerFactory extends YoutubeLikeChannelLinkHandlerFactory { +public class InvidiousChannelLinkHandlerFactory extends YoutubeLikeChannelLinkHandlerFactory + implements InvidiousLinkHandlerFactory { protected final String baseUrl; @@ -16,6 +17,11 @@ public InvidiousChannelLinkHandlerFactory(final InvidiousService service) { baseUrl = service.getInstance().getUrl(); } + @Override + public String getInvidiousBaseUrl() { + return baseUrl; + } + /** * Returns URL to channel from an ID * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java new file mode 100644 index 0000000000..ed37a08697 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java @@ -0,0 +1,19 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.URL; + +public interface InvidiousLinkHandlerFactory extends YoutubeLikeLinkHandlerFactory { + + String getInvidiousBaseUrl(); + + @Override + default boolean isInvidiousUrl(final URL url) { + return Utils.removeMAndWWWFromHost(url.getHost()) + .equalsIgnoreCase(Utils.getHostOrNull(getInvidiousBaseUrl())) && + YoutubeUrlHelper.isInvidioURL(url); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java index 62a65e3da1..6191bae56d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousPlaylistLinkHandlerFactory.java @@ -5,7 +5,8 @@ import java.util.List; -public class InvidiousPlaylistLinkHandlerFactory extends YoutubeLikePlaylistLinkHandlerFactory { +public class InvidiousPlaylistLinkHandlerFactory extends YoutubeLikePlaylistLinkHandlerFactory + implements InvidiousLinkHandlerFactory { protected final String baseUrl; @@ -13,6 +14,11 @@ public InvidiousPlaylistLinkHandlerFactory(final InvidiousService service) { this.baseUrl = service.getInstance().getUrl(); } + @Override + public String getInvidiousBaseUrl() { + return baseUrl; + } + @Override public String getUrl(final String id, final List contentFilters, final String sortFilter) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java index fc50f4be0f..9fa3d0c0af 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java @@ -4,7 +4,8 @@ import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeStreamLinkHandlerFactory; -public class InvidiousStreamLinkHandlerFactory extends YoutubeLikeStreamLinkHandlerFactory { +public class InvidiousStreamLinkHandlerFactory extends YoutubeLikeStreamLinkHandlerFactory + implements InvidiousLinkHandlerFactory { protected final String baseUrl; @@ -12,6 +13,11 @@ public InvidiousStreamLinkHandlerFactory(final InvidiousService service) { this.baseUrl = service.getInstance().getUrl(); } + @Override + public String getInvidiousBaseUrl() { + return baseUrl; + } + @Override public String getUrl(final String id) throws ParsingException { return baseUrl + "/watch?v=" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeChannelLinkHandlerFactory.java index dadda5799e..b7f55840ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeChannelLinkHandlerFactory.java @@ -2,13 +2,13 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.net.URL; import java.util.regex.Pattern; -public abstract class YoutubeLikeChannelLinkHandlerFactory extends ListLinkHandlerFactory { +public abstract class YoutubeLikeChannelLinkHandlerFactory extends ListLinkHandlerFactory + implements YoutubeLikeLinkHandlerFactory { protected static final Pattern EXCLUDED_SEGMENTS = Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); @@ -30,9 +30,7 @@ public String getId(final String url) throws ParsingException { final URL urlObj = Utils.stringToURL(url); String path = urlObj.getPath(); - if (!Utils.isHTTP(urlObj) || !(YoutubeUrlHelper.isYoutubeURL(urlObj) - || YoutubeUrlHelper.isInvidioURL(urlObj) - || YoutubeUrlHelper.isHooktubeURL(urlObj))) { + if (!Utils.isHTTP(urlObj) || !isSupportedYouTubeLikeHost(urlObj)) { throw new ParsingException("the URL given is not a Youtube-URL"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeLinkHandlerFactory.java new file mode 100644 index 0000000000..4ec94e5e2a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeLinkHandlerFactory.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.extractor.services.youtube.shared.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; + +import java.net.URL; + +public interface YoutubeLikeLinkHandlerFactory { + + default boolean isSupportedYouTubeLikeHost(final URL url) { + return YoutubeUrlHelper.isYoutubeURL(url) + || YoutubeUrlHelper.isYoutubeServiceURL(url) + || isInvidiousUrl(url) + || YoutubeUrlHelper.isHooktubeURL(url) + || YoutubeUrlHelper.isY2ubeURL(url); + } + + boolean isInvidiousUrl(final URL url); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java index 303ad68904..2a85081c50 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikePlaylistLinkHandlerFactory.java @@ -6,21 +6,20 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.shared.YoutubePlaylistHelper; -import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.net.MalformedURLException; import java.net.URL; -public abstract class YoutubeLikePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { +public abstract class YoutubeLikePlaylistLinkHandlerFactory extends ListLinkHandlerFactory + implements YoutubeLikeLinkHandlerFactory { @Override public String getId(final String url) throws ParsingException { try { final URL urlObj = Utils.stringToURL(url); - if (!Utils.isHTTP(urlObj) || !(YoutubeUrlHelper.isYoutubeURL(urlObj) - || YoutubeUrlHelper.isInvidioURL(urlObj))) { + if (!Utils.isHTTP(urlObj) || !isSupportedYouTubeLikeHost(urlObj)) { throw new ParsingException("The url given is not a YouTube-URL"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java index 3a0eb1f101..aeed51a9b3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.shared.linkHandler; import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isHooktubeURL; -import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isInvidioURL; import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isY2ubeURL; import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isYoutubeServiceURL; import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isYoutubeURL; @@ -23,7 +22,8 @@ import javax.annotation.Nullable; -public abstract class YoutubeLikeStreamLinkHandlerFactory extends LinkHandlerFactory { +public abstract class YoutubeLikeStreamLinkHandlerFactory extends LinkHandlerFactory + implements YoutubeLikeLinkHandlerFactory { protected static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("^([a-zA-Z0-9_-]{11})"); @@ -85,12 +85,7 @@ public String getId(final String theUrlString) path = path.substring(1); } - if (!Utils.isHTTP(url) - || !(isYoutubeURL(url) - || isYoutubeServiceURL(url) - || isHooktubeURL(url) - || isInvidioURL(url) - || isY2ubeURL(url)) + if (!Utils.isHTTP(url) || !(isSupportedYouTubeLikeHost(url)) ) { if ("googleads.g.doubleclick.net".equalsIgnoreCase(host)) { throw new FoundAdException("Error found ad: " + urlString); @@ -103,7 +98,7 @@ public String getId(final String theUrlString) throw new ParsingException("Can't find handler for url: " + urlString); } - if (isYoutubeURL(url) || isInvidioURL(url) || isHooktubeURL(url)) { + if (isYoutubeURL(url) || isInvidiousUrl(url) || isHooktubeURL(url)) { if (isYoutubeURL(url) && "attribution_link".equalsIgnoreCase(path)) { final String uQueryValue = Utils.getQueryValue(url, "u"); @@ -118,7 +113,7 @@ public String getId(final String theUrlString) final String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); return assertIsId(viewQueryValue); - } else if (isInvidioURL(url) || isHooktubeURL(url)) { + } else if (isInvidiousUrl(url) || isHooktubeURL(url)) { // /watch?v= if ("watch".equalsIgnoreCase(path)) { final String viewQueryValue = Utils.getQueryValue(url, "v"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index ce296bd0e4..e88ec30bcd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -24,7 +24,8 @@ * along with NewPipe. If not, see . */ -public final class YoutubeChannelLinkHandlerFactory extends YoutubeLikeChannelLinkHandlerFactory { +public final class YoutubeChannelLinkHandlerFactory extends YoutubeLikeChannelLinkHandlerFactory + implements YoutubeLinkHandlerFactory { public static final YoutubeChannelLinkHandlerFactory INSTANCE = new YoutubeChannelLinkHandlerFactory(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeLinkHandlerFactory.java new file mode 100644 index 0000000000..cf3fff917d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeLinkHandlerFactory.java @@ -0,0 +1,14 @@ +package org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler; + +import org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper; +import org.schabi.newpipe.extractor.services.youtube.shared.linkHandler.YoutubeLikeLinkHandlerFactory; + +import java.net.URL; + +public interface YoutubeLinkHandlerFactory extends YoutubeLikeLinkHandlerFactory { + + @Override + default boolean isInvidiousUrl(final URL url) { + return YoutubeUrlHelper.isInvidioURL(url); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index f40f935606..6eff26c76d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -4,7 +4,8 @@ import java.util.List; -public final class YoutubePlaylistLinkHandlerFactory extends YoutubeLikePlaylistLinkHandlerFactory { +public final class YoutubePlaylistLinkHandlerFactory extends YoutubeLikePlaylistLinkHandlerFactory + implements YoutubeLinkHandlerFactory { private static final YoutubePlaylistLinkHandlerFactory INSTANCE = new YoutubePlaylistLinkHandlerFactory(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index fb593bb935..30702067cd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -22,7 +22,8 @@ * along with NewPipe. If not, see . */ -public final class YoutubeStreamLinkHandlerFactory extends YoutubeLikeStreamLinkHandlerFactory { +public final class YoutubeStreamLinkHandlerFactory extends YoutubeLikeStreamLinkHandlerFactory + implements YoutubeLinkHandlerFactory { private static final YoutubeStreamLinkHandlerFactory INSTANCE = new YoutubeStreamLinkHandlerFactory(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index ac13283f4f..2d51420a27 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -200,6 +200,14 @@ public static boolean isHTTP(@Nonnull final URL url) { return setsNoPort || usesDefaultPort; } + public static String getHostOrNull(@Nonnull final String url) { + try { + return stringToURL(url).getHost(); + } catch (final MalformedURLException ex) { + return null; + } + } + public static String removeMAndWWWFromHost(final String host) { if (host.toLowerCase().startsWith("m.")) { return host.length() > 2 ? host.substring(2) : ""; From 9a57502d8a736b53b5132da198d070766ce16a63 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Wed, 20 Apr 2022 20:21:53 +0200 Subject: [PATCH 11/21] Fix channels not working when opened through comment --- .../extractors/InvidiousCommentsExtractor.java | 2 +- .../InvidiousCommentsInfoItemExtractor.java | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java index 8a0e7706be..a17132bc55 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java @@ -92,7 +92,7 @@ private void collectStreamsFrom( entries.stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) - .map(commentObj -> new InvidiousCommentsInfoItemExtractor(commentObj, url)) + .map(commentObj -> new InvidiousCommentsInfoItemExtractor(commentObj, url, baseUrl)) .forEach(collector::commit); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java index 1569b042d9..e730c44a0e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java @@ -11,12 +11,18 @@ public class InvidiousCommentsInfoItemExtractor implements CommentsInfoItemExtractor { - private final String url; private final JsonObject json; + private final String url; + private final String baseUrl; - public InvidiousCommentsInfoItemExtractor(final JsonObject json, final String url) { + public InvidiousCommentsInfoItemExtractor( + final JsonObject json, + final String url, + final String baseUrl + ) { this.json = json; this.url = url; + this.baseUrl = baseUrl; } @Override @@ -48,7 +54,7 @@ public String getCommentId() { @Override public String getUploaderUrl() { - return json.getString("authorUrl"); + return baseUrl + json.getString("authorUrl"); } @Override From f0810a0ebcf14c346ec306b78f7531027cf56c95 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Wed, 20 Apr 2022 21:10:47 +0200 Subject: [PATCH 12/21] Invidious: Fix related videos not shown --- .../youtube/invidious/extractors/InvidiousStreamExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java index 828ba9dea8..e43f42c4b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java @@ -266,7 +266,8 @@ public StreamInfoItemsCollector getRelatedItems() throws IOException, Extraction json.getArray("recommendedVideos").stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) - .forEach(o -> new InvidiousStreamInfoItemExtractor(o, baseUrl)); + .map(o -> new InvidiousStreamInfoItemExtractor(o, baseUrl)) + .forEach(collector::commit); return collector; } catch (final Exception e) { From b8cd300ed75773f8d41c2393d9bee35453fdd95e Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:10:45 +0200 Subject: [PATCH 13/21] Adjusted checkstyle Removed suppression file and use inline comments: Modifying the suppression file is no longer required when moving code in affected classes --- .../services/bandcamp/extractors/BandcampExtractorHelper.java | 2 ++ .../newpipe/extractor/services/youtube/shared/ItagItem.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 1ce6ac0d0c..777d1a7870 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -52,12 +52,14 @@ public static String getStreamUrlFromIds(final long bandId, } + // CHECKSTYLE:OFF - url has to be there in one piece /** * Fetch artist details from mobile endpoint. * * More technical info. */ + // CHECKSTYLE:ON public static JsonObject getArtistDetails(final String id) throws ParsingException { try { return diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/ItagItem.java index 645d10565a..d34cf15388 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/ItagItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/ItagItem.java @@ -16,10 +16,12 @@ import javax.annotation.Nonnull; public class ItagItem { + // CHECKSTYLE:OFF - url has to be there in one piece /** * List can be found here * https://github.com/ytdl-org/youtube-dl/blob/9fc5eaf/youtube_dl/extractor/youtube.py#L1071 */ + // CHECKSTYLE:ON private static final ItagItem[] ITAG_LIST = { ///////////////////////////////////////////////////// // VIDEO ID Type Format Resolution FPS /// From 0016a8ef79aa71a7b99ed837bdc3b8745b23959f Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:28:03 +0200 Subject: [PATCH 14/21] Fixed channel and playlist pagination extraction :partying_face: --- .../invidious/InvidiousParsingHelper.java | 5 -- .../extractors/InvidiousChannelExtractor.java | 28 +++++++---- .../InvidiousPlaylistExtractor.java | 46 +++++++++++++++---- .../InvidiousLinkHandlerFactory.java | 4 +- .../YoutubeLikeLinkHandlerFactory.java | 2 +- .../stream/StreamInfoItemsCollector.java | 2 +- 6 files changed, 59 insertions(+), 28 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java index dbcbc2e80e..63830ee242 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java @@ -8,7 +8,6 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -111,10 +110,6 @@ public static String getUid(@Nonnull final String id) { return id; } - public static Page getPage(final String url, final int page) { - return new Page(url + "?page=" + page, String.valueOf(page)); - } - /** * Get thumbnail URL at a reasonable quality * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java index 02add6f2b0..ab6bc8b558 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; @@ -21,7 +22,6 @@ import javax.annotation.Nonnull; - public class InvidiousChannelExtractor extends ChannelExtractor { private final String baseUrl; @@ -41,21 +41,29 @@ public InfoItemsPage getInitialPage() throws IOException, Extrac return getPage(getPage(1)); } - protected Page getPage(final int page) { - return InvidiousParsingHelper.getPage( - baseUrl + "/api/v1/channels/videos/" + json.getString("authorId"), - page - ); + protected Page getPage(final int page) throws ParsingException { + return new Page( + baseUrl + "/api/v1/channels/videos/" + getUid(getId()) + + "?page=" + page, + String.valueOf(page)); } @Override public InfoItemsPage getPage( final Page page ) throws IOException, ExtractionException { + if (page == null) { + return InfoItemsPage.emptyPage(); + } + final Response rp = NewPipe.getDownloader().get(page.getUrl()); final JsonArray array = InvidiousParsingHelper.getValidJsonArrayFromResponse(rp, page.getUrl()); + if (array.isEmpty()) { + return InfoItemsPage.emptyPage(); + } + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); array.stream() .filter(JsonObject.class::isInstance) @@ -63,9 +71,11 @@ public InfoItemsPage getPage( .map(o -> new InvidiousStreamInfoItemExtractor(o, baseUrl)) .forEach(collector::commit); - final Page nextPage = array.size() < 59 - // max number of items per page - // with Second it is 29 but next Page logic is not implemented + // If there are less than 60 results we reached the end + // CHECKSTYLE:OFF - url has to be there in one piece + // https://github.com/iv-org/invidious/blob/4900ce24fac163d801a56af1fcf0f4c207448adf/src/invidious/routes/api/v1/channels.cr#L178 + // CHECKSTYLE:ON + final Page nextPage = array.size() < 60 ? null : getPage(Integer.parseInt(page.getId()) + 1); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java index f89337cfca..b08b0b0440 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java @@ -35,8 +35,7 @@ public InvidiousPlaylistExtractor( @Nonnull @Override public String getThumbnailUrl() { - final JsonArray thumbnails = json.getArray("authorThumbnails"); - return InvidiousParsingHelper.getThumbnailUrl(thumbnails); + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); } @Override @@ -63,14 +62,18 @@ public long getStreamCount() { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - return getPage(getPage(1)); + return getPage(getPageByIndex(0)); } @Override public InfoItemsPage getPage( final Page page ) throws IOException, ExtractionException { - if (Integer.parseInt(page.getId()) != 1) { + if (page == null) { + return InfoItemsPage.emptyPage(); + } + // Initial fetched page (onFetchPage) already contains these info so don't fetch again + if (Integer.parseInt(page.getId()) != 0) { final Response rp = NewPipe.getDownloader().get(page.getUrl()); json = InvidiousParsingHelper.getValidJsonObjectFromResponse(rp, page.getUrl()); } @@ -84,17 +87,40 @@ public InfoItemsPage getPage( .map(commentObj -> new InvidiousStreamInfoItemExtractor(commentObj, baseUrl)) .forEach(collector::commit); - final Page nextPage = videos.size() < 99 - // max number of items per page - ? null - : getPage(Integer.parseInt(page.getId()) + 1); + final int lastIndex = videos.isEmpty() + ? -1 + : videos.getObject(videos.size() - 1).getInt("index", -1); + final Page nextPage = lastIndex == -1 || lastIndex >= getStreamCount() - 1 + ? null + : getPageByIndex(lastIndex); return new InfoItemsPage<>(collector, nextPage); } - public Page getPage(final int page) throws ParsingException { - return InvidiousParsingHelper.getPage(baseUrl + "/api/v1/playlists/" + getId(), page); + /* + * Note: Querying is done by index and not pagination, because it's a lot easier + * // CHECKSTYLE:OFF - url has to be there in one piece + * https://github.com/iv-org/invidious/blob/4900ce24fac163d801a56af1fcf0f4c207448adf/src/invidious/routes/api/v1/misc.cr#L20-L22 + * // CHECKSTYLE:ON + * + * e.g. Paging contains multiple duplicate items: + * Playlist-Size=505 + * Page StartIndex EndIndex Video-Count + * 1 0 199 200 + * 2 50 249 200 + * 3 150 349 200 + * 4 250 449 200 + * 5 350 504 154 + * 6 450 504 54 + * 7 - + * + * Also note that the index is also used as offset + */ + public Page getPageByIndex(final int index) throws ParsingException { + return new Page( + baseUrl + "/api/v1/playlists/" + getId() + "?index=" + index, + String.valueOf(index)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java index ed37a08697..b9ad9936f9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java @@ -13,7 +13,7 @@ public interface InvidiousLinkHandlerFactory extends YoutubeLikeLinkHandlerFacto @Override default boolean isInvidiousUrl(final URL url) { return Utils.removeMAndWWWFromHost(url.getHost()) - .equalsIgnoreCase(Utils.getHostOrNull(getInvidiousBaseUrl())) && - YoutubeUrlHelper.isInvidioURL(url); + .equalsIgnoreCase(Utils.getHostOrNull(getInvidiousBaseUrl())) + && YoutubeUrlHelper.isInvidioURL(url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeLinkHandlerFactory.java index 4ec94e5e2a..48c2a7c3ed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeLinkHandlerFactory.java @@ -14,5 +14,5 @@ default boolean isSupportedYouTubeLikeHost(final URL url) { || YoutubeUrlHelper.isY2ubeURL(url); } - boolean isInvidiousUrl(final URL url); + boolean isInvidiousUrl(URL url); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index 231a929e58..13953c3570 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -107,7 +107,7 @@ public void commit(final StreamInfoItemExtractor extractor) { try { addItem(extract(extractor)); } catch (final FoundAdException ae) { - //System.out.println("AD_WARNING: " + ae.getMessage()); + // Do nothing } catch (final Exception e) { addError(e); } From 886d1ff08d2188004eceb5d5a5c6fdd80325f3e2 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:34:31 +0200 Subject: [PATCH 15/21] Fixed feed extractor (fast mode) --- .../youtube/invidious/extractors/InvidiousFeedExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java index 8cd7ef6c96..e5fa39b0d1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; +import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getUid; import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getValidResponseBody; import org.jsoup.Jsoup; @@ -61,7 +62,7 @@ public InfoItemsPage getPage( public void onFetchPage( @Nonnull final Downloader downloader ) throws IOException, ExtractionException { - final String feedUrl = baseUrl + "/feed/channel/" + getLinkHandler().getId(); + final String feedUrl = baseUrl + "/feed/channel/" + getUid(getLinkHandler().getId()); final Response response = downloader.get(feedUrl); document = Jsoup.parse(getValidResponseBody(response, feedUrl)); } From f179b3a8ff1ee71395d20c5b751b473ebdfa7c9a Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Fri, 22 Apr 2022 18:05:35 +0200 Subject: [PATCH 16/21] Avoid using``json.json.getNumber`` and remove dislike extraction * Dislikes are always 0 --- .../extractors/InvidiousChannelExtractor.java | 2 +- .../InvidiousCommentsInfoItemExtractor.java | 5 ++--- .../extractors/InvidiousPlaylistExtractor.java | 3 +-- .../extractors/InvidiousStreamExtractor.java | 17 +++++------------ .../InvidiousStreamInfoItemExtractor.java | 13 +++++++------ 5 files changed, 16 insertions(+), 24 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java index ab6bc8b558..675da9e363 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java @@ -112,7 +112,7 @@ public String getFeedUrl() { @Override public long getSubscriberCount() { - return json.getNumber("subCount").longValue(); + return json.getLong("subCount"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java index e730c44a0e..c45c502bbf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java @@ -27,7 +27,7 @@ public InvidiousCommentsInfoItemExtractor( @Override public int getLikeCount() { - return json.getNumber("likeCount").intValue(); + return json.getInt("likeCount"); } @Override @@ -43,8 +43,7 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() { - return InvidiousParsingHelper.getUploadDateFromEpochTime( - json.getNumber("published").longValue()); + return InvidiousParsingHelper.getUploadDateFromEpochTime(json.getLong("published")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java index b08b0b0440..636007d766 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java @@ -55,8 +55,7 @@ public String getUploaderAvatarUrl() { @Override public long getStreamCount() { - final Number number = json.getNumber("videoCount"); - return number == null ? -1 : number.longValue(); + return json.getLong("videoCount", -1); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java index e43f42c4b2..b034b93b6b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java @@ -71,15 +71,13 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() { - return InvidiousParsingHelper.getUploadDateFromEpochTime( - json.getNumber("published").longValue()); + return InvidiousParsingHelper.getUploadDateFromEpochTime(json.getLong("published")); } @Nonnull @Override public String getThumbnailUrl() { - final JsonArray thumbnail = json.getArray("authorThumbnails"); - return InvidiousParsingHelper.getThumbnailUrl(thumbnail); + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("videoThumbnails")); } @Nonnull @@ -100,7 +98,7 @@ public int getAgeLimit() { @Override public long getLength() { - return json.getNumber("lengthSeconds").longValue(); + return json.getLong("lengthSeconds"); } @Override @@ -111,17 +109,12 @@ public long getTimeStamp() throws ParsingException { @Override public long getViewCount() { - return json.getNumber("viewCount").longValue(); + return json.getLong("viewCount"); } @Override public long getLikeCount() { - return json.getNumber("likeCount").longValue(); - } - - @Override - public long getDislikeCount() { - return json.getNumber("dislikeCount").longValue(); + return json.getLong("likeCount"); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java index 5f968c129c..b47f024cdc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java @@ -42,8 +42,7 @@ public boolean isAd() throws ParsingException { @Override public long getDuration() { - final Number number = json.getNumber("lengthSeconds"); - return number == null ? -1 : number.longValue(); + return json.getLong("lengthSeconds", -1); } @Override @@ -74,12 +73,14 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() { - final Number epochTime = json.getNumber("published"); - if (epochTime != null) { - return getUploadDateFromEpochTime(epochTime.longValue()); + final long epochTime = json.getLong("published", -1); + + // Time is not always provided e.g. on related videos of a video it's missing + if (epochTime == -1) { + return null; } - return null; + return getUploadDateFromEpochTime(epochTime); } @Override From d5d8ae775ecc5835f7aad2d59196c8cee2fce0c8 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Sat, 23 Apr 2022 00:07:40 +0200 Subject: [PATCH 17/21] Fix Kiosk-List ordering --- .../newpipe/extractor/kiosk/KioskList.java | 9 ++++++--- .../services/media_ccc/MediaCCCService.java | 17 ++++++++--------- .../youtube/invidious/InvidiousService.java | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 3dc3fc4f53..af2b4bc1c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.kiosk; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -8,13 +10,13 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; -import javax.annotation.Nullable; import java.io.IOException; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nullable; public class KioskList { @@ -26,7 +28,8 @@ KioskExtractor createNewKiosk(StreamingService streamingService, } private final StreamingService service; - private final HashMap kioskList = new HashMap<>(); + // Use a linked hash map to keep track of the order + private final HashMap kioskList = new LinkedHashMap<>(); private String defaultKiosk = null; @Nullable diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 8738342f13..5cafaed5b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -91,25 +91,24 @@ public SuggestionExtractor getSuggestionExtractor() { public KioskList getKioskList() { final KioskList list = new KioskList(this); - // add kiosks here e.g.: list.addKioskEntry( - (streamingService, url, kioskId) -> new MediaCCCConferenceKiosk( + (streamingService, url, kioskId) -> new MediaCCCRecentKiosk( MediaCCCService.this, - new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), + new MediaCCCRecentListLinkHandlerFactory().fromUrl(url), kioskId ), - new MediaCCCConferencesListLinkHandlerFactory(), - "conferences" + new MediaCCCRecentListLinkHandlerFactory(), + "recent" ); list.addKioskEntry( - (streamingService, url, kioskId) -> new MediaCCCRecentKiosk( + (streamingService, url, kioskId) -> new MediaCCCConferenceKiosk( MediaCCCService.this, - new MediaCCCRecentListLinkHandlerFactory().fromUrl(url), + new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId ), - new MediaCCCRecentListLinkHandlerFactory(), - "recent" + new MediaCCCConferencesListLinkHandlerFactory(), + "conferences" ); list.addKioskEntry( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java index 5524ff3987..5a47545c1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java @@ -117,8 +117,8 @@ public KioskList getKioskList() { final InvidiousTrendingLinkHandlerFactory lhf = new InvidiousTrendingLinkHandlerFactory(InvidiousService.this); - list.addKioskEntry(kioskFactory, lhf, InvidiousTrendingLinkHandlerFactory.KIOSK_POPULAR); list.addKioskEntry(kioskFactory, lhf, InvidiousTrendingLinkHandlerFactory.KIOSK_TRENDING); + list.addKioskEntry(kioskFactory, lhf, InvidiousTrendingLinkHandlerFactory.KIOSK_POPULAR); list.setDefaultKiosk(InvidiousTrendingLinkHandlerFactory.KIOSK_TRENDING); return list; From 93ec901a34587710338616800a6176e162d09beb Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Sun, 24 Apr 2022 15:50:16 +0200 Subject: [PATCH 18/21] Refactored YT-like stream link handlers and fixed tests Also uses parameterized tests now --- .../InvidiousStreamLinkHandlerFactory.java | 10 +- .../YoutubeLikeStreamLinkHandlerFactory.java | 19 +- .../YoutubeStreamLinkHandlerFactory.java | 9 +- .../YoutubeStreamLinkHandlerFactoryTest.java | 284 ++++++++++-------- 4 files changed, 183 insertions(+), 139 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java index 9fa3d0c0af..241242ca78 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousStreamLinkHandlerFactory.java @@ -8,9 +8,12 @@ public class InvidiousStreamLinkHandlerFactory extends YoutubeLikeStreamLinkHand implements InvidiousLinkHandlerFactory { protected final String baseUrl; + protected final InvidiousPlaylistLinkHandlerFactory playlistLinkHandlerFactory; public InvidiousStreamLinkHandlerFactory(final InvidiousService service) { - this.baseUrl = service.getInstance().getUrl(); + baseUrl = service.getInstance().getUrl(); + playlistLinkHandlerFactory = + (InvidiousPlaylistLinkHandlerFactory) service.getPlaylistLHFactory(); } @Override @@ -18,6 +21,11 @@ public String getInvidiousBaseUrl() { return baseUrl; } + @Override + protected boolean isPlaylistUrl(final String url) { + return playlistLinkHandlerFactory.onAcceptUrl(url); + } + @Override public String getUrl(final String id) throws ParsingException { return baseUrl + "/watch?v=" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java index aeed51a9b3..3b33c9b8b1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/linkHandler/YoutubeLikeStreamLinkHandlerFactory.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; import java.net.MalformedURLException; @@ -32,11 +31,12 @@ public abstract class YoutubeLikeStreamLinkHandlerFactory extends LinkHandlerFac @Nullable private static String extractId(@Nullable final String id) { - if (id != null) { - final Matcher m = YOUTUBE_VIDEO_ID_REGEX_PATTERN.matcher(id); - return m.find() ? m.group(1) : null; + if (id == null) { + return null; } - return null; + + final Matcher m = YOUTUBE_VIDEO_ID_REGEX_PATTERN.matcher(id); + return m.find() ? m.group(1) : null; } private static String assertIsId(@Nullable final String id) throws ParsingException { @@ -85,8 +85,7 @@ public String getId(final String theUrlString) path = path.substring(1); } - if (!Utils.isHTTP(url) || !(isSupportedYouTubeLikeHost(url)) - ) { + if (!Utils.isHTTP(url) || !isSupportedYouTubeLikeHost(url)) { if ("googleads.g.doubleclick.net".equalsIgnoreCase(host)) { throw new FoundAdException("Error found ad: " + urlString); } @@ -94,8 +93,8 @@ public String getId(final String theUrlString) throw new ParsingException("The url is not a Youtube-URL"); } - if (YoutubePlaylistLinkHandlerFactory.getInstance().acceptUrl(urlString)) { - throw new ParsingException("Can't find handler for url: " + urlString); + if (isPlaylistUrl(urlString)) { + throw new ParsingException("This url is a playlist url: " + urlString); } if (isYoutubeURL(url) || isInvidiousUrl(url) || isHooktubeURL(url)) { @@ -156,6 +155,8 @@ public String getId(final String theUrlString) throw new ParsingException("Error no suitable url: " + urlString); } + protected abstract boolean isPlaylistUrl(String url); + @Override public boolean onAcceptUrl(final String url) throws FoundAdException { try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 30702067cd..15dfd21b9a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -31,12 +31,17 @@ public final class YoutubeStreamLinkHandlerFactory extends YoutubeLikeStreamLink private YoutubeStreamLinkHandlerFactory() { } - public static YoutubeStreamLinkHandlerFactory getInstance() { - return INSTANCE; + @Override + protected boolean isPlaylistUrl(final String url) { + return YoutubePlaylistLinkHandlerFactory.getInstance().onAcceptUrl(url); } @Override public String getUrl(final String id) { return "https://www.youtube.com/watch?v=" + id; } + + public static YoutubeStreamLinkHandlerFactory getInstance() { + return INSTANCE; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 0ebc7e989e..3cee4e699c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -3,24 +3,22 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; -import java.util.ArrayList; -import java.util.List; - /** * Test for {@link YoutubeStreamLinkHandlerFactory} */ public class YoutubeStreamLinkHandlerFactoryTest { - private static final String AD_URL = "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew"; private static YoutubeStreamLinkHandlerFactory linkHandler; @BeforeAll @@ -35,141 +33,173 @@ void getIdWithNullAsUrl() { } @Test - void getIdForAd() throws ParsingException { - assertThrows(FoundAdException.class, () -> linkHandler.fromUrl(AD_URL)); + void getIdForAd() { + assertThrows(FoundAdException.class, () -> linkHandler.fromUrl( + "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew")); } - @Test - void getIdForInvalidUrls() { - final List invalidUrls = new ArrayList<>(50); - invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d"); - invalidUrls.add("https://www.youtube.com/watchjZViOEv90d"); - invalidUrls.add("https://www.youtube.com/"); - invalidUrls.add("https://www.youtube.com/channel/UCBR8-60-B28hp2BmDPdntcQ"); - invalidUrls.add("https://invidious.fdn.fr/channel/UCBR8-60-B28hp2BmDPdntcQ"); - for (final String invalidUrl : invalidUrls) { - Throwable exception = null; - try { - linkHandler.fromUrl(invalidUrl).getId(); - } catch (final ParsingException e) { - exception = e; - } - if (exception == null) { - fail("Expected ParsingException for url: " + invalidUrl); - } - } + @ParameterizedTest + @ValueSource(strings = { + "https://www.youtube.com/watch?v=jZViOEv90d", + "https://www.youtube.com/watchjZViOEv90d", + "https://www.youtube.com/", + "https://www.youtube.com/channel/UCBR8-60-B28hp2BmDPdntcQ", + "https://invidious.fdn.fr/channel/UCBR8-60-B28hp2BmDPdntcQ" + }) + void getIdForInvalidUrls(final String invalidUrl) { + assertThrows(ParsingException.class, () -> linkHandler.fromUrl(invalidUrl).getId()); } - @Test - void getIdfromYt() throws Exception { - assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube.com/watch?v=jZViOEv90dI").getId()); - assertEquals("W-fFHeTX70Q", linkHandler.fromUrl("https://www.youtube.com/watch?v=W-fFHeTX70Q").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI&t=100").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI&t=100").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://youtu.be/jZViOEv90dI?t=9s").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("HTTPS://Youtu.be/jZViOEv90dI?t=9s").getId()); - assertEquals("uEJuoEs1UxY", linkHandler.fromUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY").getId()); - assertEquals("uEJuoEs1UxY", linkHandler.fromUrl("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube.com/embed/jZViOEv90dI").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube.com/watch?v=jZViOEv90dI").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://youtube.com/watch?v=jZViOEv90dI").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://youtu.be/jZViOEv90dI?t=9s").getId()); - assertEquals("7_WWz2DSnT8", linkHandler.fromUrl("https://youtu.be/7_WWz2DSnT8").getId()); - assertEquals("oy6NvWeVruY", linkHandler.fromUrl("https://m.youtube.com/watch?v=oy6NvWeVruY").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube.com/embed/jZViOEv90dI").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); - assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); - assertEquals("n8X9_MgEdCg", linkHandler.fromUrl("vnd.youtube://n8X9_MgEdCg").getId()); - assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); - assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://m.youtube.com/watch?v=-cdveCh1kQk)").getId()); - assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://www.youtube.com/watch?v=-cdveCh1kQk-").getId()); - assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://WWW.YouTube.com/watch?v=-cdveCh1kQkwhatever").getId()); - assertEquals("O0EDx9WAelc", linkHandler.fromUrl("HTTPS://www.youtube.com/watch?v=O0EDx9WAelc]").getId()); - assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://youtu.be/-cdveCh1kQk)hello").getId()); - assertEquals("OGS7c0-CmRs", linkHandler.fromUrl("https://YouTu.be/OGS7c0-CmRswhatever)").getId()); - assertEquals("-cdveCh1kQk", linkHandler.fromUrl("HTTPS://youtu.be/-cdveCh1kQk)").getId()); - assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("https://www.youtube.com/shorts/IOS2fqxwYbAhi").getId()); - assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/shorts/IOS2fqxwYbA").getId()); - assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/v/IOS2fqxwYbA").getId()); - assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/w/IOS2fqxwYbA").getId()); - assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/watch/IOS2fqxwYbA").getId()); + @ParameterizedTest + @ValueSource(strings = { + "https://www.youtube.com/watch?v=jZViOEv90dI", + "https://www.youtube.com/watch?v=jZViOEv90dI&t=100", + "https://WWW.YouTube.com/watch?v=jZViOEv90dI&t=100", + "HTTPS://www.youtube.com/watch?v=jZViOEv90dI&t=100", + "https://youtu.be/jZViOEv90dI?t=9s", + "HTTPS://Youtu.be/jZViOEv90dI?t=9s", + "https://www.youtube.com/embed/jZViOEv90dI", + "https://www.youtube-nocookie.com/embed/jZViOEv90dI", + "http://www.youtube.com/watch?v=jZViOEv90dI", + "http://youtube.com/watch?v=jZViOEv90dI", + "http://youtu.be/jZViOEv90dI?t=9s", + "http://www.youtube.com/embed/jZViOEv90dI", + "http://www.Youtube.com/embed/jZViOEv90dI", + "http://www.youtube-nocookie.com/embed/jZViOEv90dI", + "vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI", + "vnd.youtube:jZViOEv90dI" + }) + void getId_jZViOEv90dI_fromYt(final String url) throws Exception { + assertEquals("jZViOEv90dI", linkHandler.fromUrl(url).getId()); } - @Test - void testAcceptYtUrl() throws ParsingException { - assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100")); - assertTrue(linkHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI&t=100")); - assertTrue(linkHandler.acceptUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI&t=100")); - assertTrue(linkHandler.acceptUrl("https://youtu.be/jZViOEv90dI?t=9s")); - assertTrue(linkHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("http://www.youtube.com/watch?v=jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("http://youtu.be/jZViOEv90dI?t=9s")); - assertTrue(linkHandler.acceptUrl("http://www.youtube.com/embed/jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); - assertTrue(linkHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc")); - assertTrue(linkHandler.acceptUrl("https://www.youtube.com/shorts/IOS2fqxwYbA")); - assertTrue(linkHandler.acceptUrl("https://www.youtube.com/v/IOS2fqxwYbA")); - assertTrue(linkHandler.acceptUrl("https://www.youtube.com/w/IOS2fqxwYbA")); - assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch/IOS2fqxwYbA")); + @ParameterizedTest + @ValueSource(strings = { + "https://www.youtube.com/shorts/IOS2fqxwYbA", + "http://www.youtube.com/shorts/IOS2fqxwYbA", + "http://www.youtube.com/v/IOS2fqxwYbA", + "https://www.youtube.com/w/IOS2fqxwYbA", + "https://www.youtube.com/watch/IOS2fqxwYbA" + }) + void getId_IOS2fqxwYbA_fromYt(final String url) throws Exception { + assertEquals("IOS2fqxwYbA", linkHandler.fromUrl(url).getId()); } - @Test - void testAcceptHookUrl() throws ParsingException { - assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU")); - assertTrue(linkHandler.acceptUrl("http://hooktube.com/watch?v=TglNG-yjabU")); - assertTrue(linkHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); - assertTrue(linkHandler.acceptUrl("hooktube.com/watch/3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("hooktube.com/v/3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("hooktube.com/embed/3msbfr6pBNE")); + @ParameterizedTest + @ValueSource(strings = { + "https://m.youtube.com/watch?v=-cdveCh1kQk)", + "https://www.youtube.com/watch?v=-cdveCh1kQk-", + "https://WWW.YouTube.com/watch?v=-cdveCh1kQkwhatever", + "https://youtu.be/-cdveCh1kQk)hello", + "HTTPS://youtu.be/-cdveCh1kQk)" + }) + void getId_cdveCh1kQk_fromYt(final String url) throws Exception { + assertEquals("-cdveCh1kQk", linkHandler.fromUrl(url).getId()); } - @Test - void testGetHookIdfromUrl() throws ParsingException { - assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId()); - assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://hooktube.com/watch?v=TglNG-yjabU").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch?v=3msbfr6pBNE").getId()); - assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/w/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId()); + @ParameterizedTest + @CsvSource({ + "W-fFHeTX70Q,https://www.youtube.com/watch?v=W-fFHeTX70Q", + "uEJuoEs1UxY,http://www.youtube.com/watch_popup?v=uEJuoEs1UxY", + "uEJuoEs1UxY,http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY", + "7_WWz2DSnT8,https://youtu.be/7_WWz2DSnT8", + "oy6NvWeVruY,https://m.youtube.com/watch?v=oy6NvWeVruY", + "EhxJLojIE_o,http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare", + "n8X9_MgEdCg,vnd.youtube://n8X9_MgEdCg", + "O0EDx9WAelc,https://music.youtube.com/watch?v=O0EDx9WAelc", + "O0EDx9WAelc,HTTPS://www.youtube.com/watch?v=O0EDx9WAelc]", + "OGS7c0-CmRs,https://YouTu.be/OGS7c0-CmRswhatever)" + }) + void getId_diverse_fromYt(final String expectedId, final String url) throws Exception { + assertEquals(expectedId, linkHandler.fromUrl(url).getId()); } - @Test - void testAcceptInvidiousUrl() throws ParsingException { - assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/watch?v=TglNG-yjabU")); - assertTrue(linkHandler.acceptUrl("http://www.invidious.fdn.fr/watch?v=TglNG-yjabU")); - assertTrue(linkHandler.acceptUrl("http://invidious.fdn.fr/watch?v=TglNG-yjabU")); - assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/watch?v=3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("https://invidious.fdn.fr/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); - assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/embed/3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/watch/3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/v/3msbfr6pBNE")); - assertTrue(linkHandler.acceptUrl("invidious.fdn.fr/w/3msbfr6pBNE")); + @ParameterizedTest + @ValueSource(strings = { + "https://www.youtube.com/watch?v=jZViOEv90dI", + "https://www.youtube.com/watch?v=jZViOEv90dI&t=100", + "https://WWW.YouTube.com/watch?v=jZViOEv90dI&t=100", + "HTTPS://www.youtube.com/watch?v=jZViOEv90dI&t=100", + "https://youtu.be/jZViOEv90dI?t=9s", + "https://www.youtube.com/embed/jZViOEv90dI", + "https://www.youtube-nocookie.com/embed/jZViOEv90dI", + "http://www.youtube.com/watch?v=jZViOEv90dI", + "http://youtu.be/jZViOEv90dI?t=9s", + "http://www.youtube.com/embed/jZViOEv90dI", + "http://www.youtube-nocookie.com/embed/jZViOEv90dI", + "http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare", + "vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI", + "vnd.youtube:jZViOEv90dI", + "vnd.youtube.launch:jZViOEv90dI", + "https://music.youtube.com/watch?v=O0EDx9WAelc", + "https://www.youtube.com/shorts/IOS2fqxwYbA", + "http://www.youtube.com/shorts/IOS2fqxwYbA", + "http://www.youtube.com/v/IOS2fqxwYbA", + "https://www.youtube.com/w/IOS2fqxwYbA", + "https://www.youtube.com/watch/IOS2fqxwYbA" + }) + void acceptYtUrl(final String url) throws ParsingException { + assertTrue(linkHandler.acceptUrl(url)); } - @Test - void testGetInvidiousIdfromUrl() throws ParsingException { - assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidious.fdn.fr/watch?v=TglNG-yjabU").getId()); - assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://www.invidious.fdn.fr/watch?v=TglNG-yjabU").getId()); - assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://invidious.fdn.fr/watch?v=TglNG-yjabU").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/watch?v=3msbfr6pBNE").getId()); - assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://invidious.fdn.fr/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/embed/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/v/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/w/3msbfr6pBNE").getId()); - assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidious.fdn.fr/watch/3msbfr6pBNE").getId()); + @ParameterizedTest + @ValueSource(strings = { + "https://hooktube.com/watch?v=TglNG-yjabU", + "http://hooktube.com/watch?v=TglNG-yjabU", + "https://hooktube.com/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2", + "hooktube.com/watch?v=3msbfr6pBNE", + "hooktube.com/watch/3msbfr6pBNE", + "hooktube.com/v/3msbfr6pBNE", + "hooktube.com/embed/3msbfr6pBNE" + }) + void acceptHookUrl(final String url) throws ParsingException { + assertTrue(linkHandler.acceptUrl(url)); + } + + @ParameterizedTest + @CsvSource({ + "TglNG-yjabU,https://hooktube.com/watch?v=TglNG-yjabU", + "TglNG-yjabU,http://hooktube.com/watch?v=TglNG-yjabU", + "ocH3oSnZG3c,https://hooktube.com/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2", + "3msbfr6pBNE,hooktube.com/watch?v=3msbfr6pBNE", + "3msbfr6pBNE,hooktube.com/watch/3msbfr6pBNE", + "3msbfr6pBNE,hooktube.com/v/3msbfr6pBNE", + "3msbfr6pBNE,hooktube.com/w/3msbfr6pBNE", + "3msbfr6pBNE,hooktube.com/embed/3msbfr6pBNE" + }) + void getHookIdfromUrl(final String expectedId, final String url) throws Exception { + assertEquals(expectedId, linkHandler.fromUrl(url).getId()); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://invidious.fdn.fr/watch?v=TglNG-yjabU", + "http://www.invidious.fdn.fr/watch?v=TglNG-yjabU", + "http://invidious.fdn.fr/watch?v=TglNG-yjabU", + "invidious.fdn.fr/watch?v=3msbfr6pBNE", + "https://invidious.fdn.fr/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2", + "invidious.fdn.fr/embed/3msbfr6pBNE", + "invidious.fdn.fr/watch/3msbfr6pBNE", + "invidious.fdn.fr/v/3msbfr6pBNE", + "invidious.fdn.fr/w/3msbfr6pBNE" + }) + void acceptInvidiousUrl(final String url) throws ParsingException { + assertTrue(linkHandler.acceptUrl(url)); + } + + @ParameterizedTest + @CsvSource({ + "TglNG-yjabU,https://invidious.fdn.fr/watch?v=TglNG-yjabU", + "TglNG-yjabU,http://www.invidious.fdn.fr/watch?v=TglNG-yjabU", + "TglNG-yjabU,http://invidious.fdn.fr/watch?v=TglNG-yjabU", + "ocH3oSnZG3c,https://invidious.fdn.fr/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2", + "3msbfr6pBNE,invidious.fdn.fr/watch?v=3msbfr6pBNE", + "3msbfr6pBNE,invidious.fdn.fr/embed/3msbfr6pBNE", + "3msbfr6pBNE,invidious.fdn.fr/v/3msbfr6pBNE", + "3msbfr6pBNE,invidious.fdn.fr/w/3msbfr6pBNE", + "3msbfr6pBNE,invidious.fdn.fr/watch/3msbfr6pBNE" + }) + void getInvidiousIdfromUrl(final String expectedId, final String url) throws Exception { + assertEquals(expectedId, linkHandler.fromUrl(url).getId()); } } \ No newline at end of file From ec5598f845787104eeb452880014ed948760bf70 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Tue, 26 Apr 2022 19:27:14 +0200 Subject: [PATCH 19/21] Fixed incorrect ``isInvidiousUrl`` check --- .../invidious/linkHandler/InvidiousLinkHandlerFactory.java | 2 +- .../extractor/services/youtube/shared/YoutubeUrlHelper.java | 2 +- .../youtube/linkHandler/YoutubeLinkHandlerFactory.java | 2 +- .../linkHandler/YoutubeTrendingLinkHandlerFactory.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java index b9ad9936f9..2fa34b2a2c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkHandler/InvidiousLinkHandlerFactory.java @@ -14,6 +14,6 @@ public interface InvidiousLinkHandlerFactory extends YoutubeLikeLinkHandlerFacto default boolean isInvidiousUrl(final URL url) { return Utils.removeMAndWWWFromHost(url.getHost()) .equalsIgnoreCase(Utils.getHostOrNull(getInvidiousBaseUrl())) - && YoutubeUrlHelper.isInvidioURL(url); + || YoutubeUrlHelper.isInvidiousURL(url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java index 0128caf7b0..b75a696947 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/shared/YoutubeUrlHelper.java @@ -43,7 +43,7 @@ public static boolean isHooktubeURL(@Nonnull final URL url) { return host.equalsIgnoreCase("hooktube.com"); } - public static boolean isInvidioURL(@Nonnull final URL url) { + public static boolean isInvidiousURL(@Nonnull final URL url) { // Valid, working instances as of 2022-03 final Set invInstances = new HashSet<>(Arrays.asList( "invidious.snopyta.org", diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeLinkHandlerFactory.java index cf3fff917d..86c7c32278 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeLinkHandlerFactory.java @@ -9,6 +9,6 @@ public interface YoutubeLinkHandlerFactory extends YoutubeLikeLinkHandlerFactory @Override default boolean isInvidiousUrl(final URL url) { - return YoutubeUrlHelper.isInvidioURL(url); + return YoutubeUrlHelper.isInvidiousURL(url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index 1949db5ba6..7fbe842143 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -20,7 +20,7 @@ * along with NewPipe. If not, see . */ -import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isInvidioURL; +import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isInvidiousURL; import static org.schabi.newpipe.extractor.services.youtube.shared.YoutubeUrlHelper.isYoutubeURL; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; @@ -54,7 +54,7 @@ public boolean onAcceptUrl(final String url) { } final String urlPath = urlObj.getPath(); - return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidioURL(urlObj)) + return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidiousURL(urlObj)) && urlPath.equals("/feed/trending"); } } From dce23a27e57fb716ee56f475659259322779a655 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Tue, 26 Apr 2022 20:06:31 +0200 Subject: [PATCH 20/21] De-deduplicate and improve exception message --- .../invidious/InvidiousParsingHelper.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java index 63830ee242..b616b0f45a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java @@ -61,12 +61,19 @@ public static String getValidResponseBody( final Response response, final String apiUrl ) throws ExtractionException { - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Could not get page " + apiUrl - + " (HTTP " + response.responseCode() + " : " + response.responseMessage()); - } else if (response.responseCode() >= 400) { - throw new ExtractionException("Could not get page " + apiUrl - + " (HTTP " + response.responseCode() + " : " + response.responseMessage()); + if (response.responseCode() >= 400) { + final String trimmedBody = response.responseBody().length() > 100 + ? response.responseBody().substring(0, 98) + "..." + : response.responseBody(); + final String message = "Could not get page " + apiUrl + + " [code=" + response.responseCode() + + ",message=" + response.responseMessage() + + ",body=" + trimmedBody + + "]"; + if (response.responseCode() == 404) { + throw new ContentNotAvailableException(message); + } + throw new ExtractionException(message); } return response.responseBody(); From 1446e7a893e51994a0da1c702a4a9768eb59f13d Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Sat, 7 May 2022 20:10:32 +0200 Subject: [PATCH 21/21] Use correct name for PeerTube --- .../newpipe/extractor/services/peertube/PeertubeInstance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index fc31329c18..ecc27ec790 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -12,7 +12,7 @@ public class PeertubeInstance extends AbstractInstance { - public static final String SERVICE_NAME = "Peertube"; + public static final String SERVICE_NAME = "PeerTube"; public static final PeertubeInstance DEFAULT_INSTANCE = new PeertubeInstance("https://framatube.org", "FramaTube");