Merge branch 'dev'

TeamNewPipe · Jul 25, 2024 · 176da72 · 176da72
2 parents d39fc43 + 530c157
commit 176da72
Show file tree

Hide file tree

Showing 434 changed files with 31,368 additions and 29,291 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -32,7 +32,7 @@ jobs:
         run: ./gradlew aggregatedJavadocs
 
       - name: Deploy JavaDocs
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./build/docs
diff --git a/README.md b/README.md
@@ -21,7 +21,7 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the
 -dontwarn org.mozilla.javascript.tools.**
 ```
 
-**Note:** To use NewPipe Extractor in Android projects with a `minSdk` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. If the `minSdk` is below 19, the `desugar_jdk_libs_nio` artifact is required, which requires Android Gradle Plugin (AGP) version 7.4.0.
+**Note:** To use NewPipe Extractor in Android projects with a `minSdk` below 33, [core library desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) with the `desugar_jdk_libs_nio` artifact is required.
 
 ### Testing changes
 

diff --git a/build.gradle b/build.gradle
@@ -8,7 +8,7 @@ allprojects {
     sourceCompatibility = JavaVersion.VERSION_11
     targetCompatibility = JavaVersion.VERSION_11
 
-    version 'v0.24.0'
+    version 'v0.24.2'
     group 'com.github.TeamNewPipe'
 
     repositories {
@@ -28,8 +28,8 @@ allprojects {
 
     ext {
         nanojsonVersion = "1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751"
-        spotbugsVersion = "4.8.3"
-        junitVersion = "5.10.2"
+        spotbugsVersion = "4.8.6"
+        junitVersion = "5.10.3"
         checkstyleVersion = "10.4"
     }
 }

diff --git a/extractor/build.gradle b/extractor/build.gradle
@@ -31,7 +31,7 @@ dependencies {
 
     // do not upgrade to 1.7.14, since in 1.7.14 Rhino uses the `SourceVersion` class, which is not
     // available on Android (even when using desugaring), and `NoClassDefFoundError` is thrown
-    implementation 'org.mozilla:rhino:1.7.13'
+    implementation 'org.mozilla:rhino:1.7.15'
 
     checkstyle "com.puppycrawl.tools:checkstyle:$checkstyleVersion"
 
@@ -41,5 +41,5 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-params'
 
     testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
-    testImplementation 'com.google.code.gson:gson:2.10.1'
+    testImplementation 'com.google.code.gson:gson:2.11.0'
 }
diff --git a/...va/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/...va/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java
@@ -16,6 +16,7 @@
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.utils.ImageSuffix;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -155,25 +156,34 @@ public static String getImageUrl(final long id, final boolean isAlbum) {
 
     /**
      * @return <code>true</code> if the given URL looks like it comes from a bandcamp custom domain
-     * or if it comes from <code>bandcamp.com</code> itself
+     * or a <code>*.bandcamp.com</code> subdomain
      */
-    public static boolean isSupportedDomain(final String url) throws ParsingException {
+    public static boolean isArtistDomain(final String url) throws ParsingException {
 
         // Accept all bandcamp.com URLs
         if (url.toLowerCase().matches("https?://.+\\.bandcamp\\.com(/.*)?")) {
             return true;
         }
 
+        // Reject non-artist bandcamp.com URLs
+        if (url.toLowerCase().matches("https?://bandcamp\\.com(/.*)?")) {
+            return false;
+        }
+
         try {
             // Test other URLs for whether they contain a footer that links to bandcamp
-            return Jsoup.parse(NewPipe.getDownloader().get(url).responseBody())
-                    .getElementById("pgFt")
-                    .getElementById("pgFt-inner")
-                    .getElementById("footer-logo-wrapper")
-                    .getElementById("footer-logo")
-                    .getElementsByClass("hiddenAccess")
-                    .text().equals("Bandcamp");
-        } catch (final NullPointerException e) {
+            return Jsoup.parse(
+                    NewPipe.getDownloader()
+                            .get(Utils.replaceHttpWithHttps(url))
+                            .responseBody()
+                    )
+                    .getElementsByClass("cart-wrapper")
+                    .get(0)
+                    .getElementsByTag("a")
+                    .get(0)
+                    .attr("href")
+                    .equals("https://bandcamp.com/cart");
+        } catch (final NullPointerException | IndexOutOfBoundsException e) {
             return false;
         } catch (final IOException | ReCaptchaException e) {
             throw new ParsingException("Could not determine whether URL is custom domain "

diff --git a/...ava/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java b/...ava/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java
@@ -24,7 +24,7 @@
 public class BandcampRadioExtractor extends KioskExtractor<StreamInfoItem> {
 
     public static final String KIOSK_RADIO = "Radio";
-    public static final String RADIO_API_URL = BASE_API_URL + "/bcweekly/1/list";
+    public static final String RADIO_API_URL = BASE_API_URL + "/bcweekly/3/list";
 
     private JsonObject json = null;
 

diff --git a/...schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/...schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java
@@ -35,6 +35,12 @@ public long getDuration() {
         return 0;
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return show.getString("desc");
+    }
+
     @Nullable
     @Override
     public String getTextualUploadDate() {
@@ -75,8 +81,8 @@ public long getViewCount() {
 
     @Override
     public String getUploaderName() {
-        // JSON does not contain uploader name
-        return "";
+        // The "title" field contains the title of the series, e.g. "Bandcamp Weekly".
+        return show.getString("title");
     }
 
     @Override

diff --git a/.../services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/.../services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java
@@ -43,7 +43,12 @@ public String getName() {
 
     @Override
     public String getUrl() {
-        return getUploaderUrl() + track.getString("title_link");
+        final String relativeUrl = track.getString("title_link");
+        if (relativeUrl != null) {
+            return getUploaderUrl() + relativeUrl;
+        } else {
+            return null;
+        }
     }
 
     @Override
@@ -66,7 +71,7 @@ public String getUploaderName() {
     @Nonnull
     @Override
     public List<Image> getThumbnails() throws ParsingException {
-        if (substituteCovers.isEmpty()) {
+        if (substituteCovers.isEmpty() && getUrl() != null) {
             try {
                 final StreamExtractor extractor = service.getStreamExtractor(getUrl());
                 extractor.fetchPage();

diff --git a/...bi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/...bi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java
@@ -33,7 +33,8 @@ public static BandcampChannelLinkHandlerFactory getInstance() {
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
         try {
-            final String response = NewPipe.getDownloader().get(url).responseBody();
+            final String response = NewPipe.getDownloader().get(Utils.replaceHttpWithHttps(url))
+                    .responseBody();
 
             // Use band data embedded in website to extract ID
             final JsonObject bandData = JsonUtils.getJsonData(response, "data-band");
@@ -90,7 +91,7 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
             }
 
             // Test whether domain is supported
-            return BandcampExtractorHelper.isSupportedDomain(lowercaseUrl);
+            return BandcampExtractorHelper.isArtistDomain(lowercaseUrl);
         }
     }
 }
diff --git a/...i/newpipe/extractor/services/bandcamp/linkHandler/BandcampCommentsLinkHandlerFactory.java b/...i/newpipe/extractor/services/bandcamp/linkHandler/BandcampCommentsLinkHandlerFactory.java
@@ -3,6 +3,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.util.List;
 
@@ -24,7 +25,7 @@ public static BandcampCommentsLinkHandlerFactory getInstance() {
 
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
-        return url;
+        return Utils.replaceHttpWithHttps(url);
     }
 
     @Override
@@ -39,14 +40,14 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 
     @Override
     public String getUrl(final String id,
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        return id;
+        return Utils.replaceHttpWithHttps(id);
     }
 }
diff --git a/...i/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java b/...i/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java
@@ -5,6 +5,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.util.List;
 
@@ -33,7 +34,7 @@ public String getUrl(final String url,
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        return url;
+        return Utils.replaceHttpWithHttps(url);
     }
 
     /**
@@ -48,6 +49,6 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 }
diff --git a/...bi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/...bi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java
@@ -8,7 +8,6 @@
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 public final class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -28,10 +27,6 @@ public String getUrl(final String query,
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("query \"" + query + "\" could not be encoded", e);
-        }
+        return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
     }
 }
diff --git a/...abi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/...abi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java
@@ -5,6 +5,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
 
@@ -49,7 +50,7 @@ public String getUrl(final String input)
         if (input.matches("\\d+")) {
             return BASE_URL + "/?show=" + input;
         } else {
-            return input;
+            return Utils.replaceHttpWithHttps(input);
         }
     }
 
@@ -71,6 +72,6 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 }
diff --git a/...i/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/...i/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
@@ -4,7 +4,6 @@
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 public final class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -42,10 +41,6 @@ public String getUrl(final String query,
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("Could not create search string with query: " + query, e);
-        }
+        return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
     }
 }
diff --git a/...va/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/...va/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
@@ -38,7 +38,6 @@
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -379,8 +378,7 @@ public List<Frameset> getFrames() throws ExtractionException {
     }
 
     @Nonnull
-    private String getRelatedItemsUrl(@Nonnull final List<String> tags)
-            throws UnsupportedEncodingException {
+    private String getRelatedItemsUrl(@Nonnull final List<String> tags) {
         final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT_VIDEOS;
         final StringBuilder params = new StringBuilder();
         params.append("start=0&count=8&sort=-createdAt");

diff --git a/...bi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/...bi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
@@ -5,7 +5,6 @@
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -49,21 +48,17 @@ public String getUrl(final String searchString,
                          final String sortFilter,
                          final String baseUrl)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            final String endpoint;
-            if (contentFilters.isEmpty()
-                    || contentFilters.get(0).equals(VIDEOS)
-                    || contentFilters.get(0).equals(SEPIA_VIDEOS)) {
-                endpoint = SEARCH_ENDPOINT_VIDEOS;
-            } else if (contentFilters.get(0).equals(CHANNELS)) {
-                endpoint = SEARCH_ENDPOINT_CHANNELS;
-            } else {
-                endpoint = SEARCH_ENDPOINT_PLAYLISTS;
-            }
-            return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString);
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("Could not encode query", e);
+        final String endpoint;
+        if (contentFilters.isEmpty()
+                || contentFilters.get(0).equals(VIDEOS)
+                || contentFilters.get(0).equals(SEPIA_VIDEOS)) {
+            endpoint = SEARCH_ENDPOINT_VIDEOS;
+        } else if (contentFilters.get(0).equals(CHANNELS)) {
+            endpoint = SEARCH_ENDPOINT_CHANNELS;
+        } else {
+            endpoint = SEARCH_ENDPOINT_PLAYLISTS;
         }
+        return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString);
     }
 
     @Override