From bff7518cfe132259dbc426f476804f74ae6f4bd0 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Fri, 24 May 2024 19:24:41 +0200 Subject: [PATCH] Throw upon trying to fetch unencrypted URL --- .../extractor/downloader/Downloader.java | 70 +++++++++++-------- .../downloader/DownloaderTestImpl.java | 2 +- .../newpipe/downloader/MockDownloader.java | 2 +- .../downloader/RecordingDownloader.java | 2 +- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java index aa7987156d..aa7f9a6030 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java @@ -3,6 +3,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -25,10 +26,10 @@ public abstract class Downloader { * localization. It should only be used when the resource that will be fetched won't be affected * by the localization. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @return the result of the GET request */ - public Response get(final String url) throws IOException, ReCaptchaException { + public final Response get(final String url) throws IOException, ReCaptchaException { return get(url, null, NewPipe.getPreferredLocalization()); } @@ -37,11 +38,11 @@ public Response get(final String url) throws IOException, ReCaptchaException { *
* It will set the {@code Accept-Language} header to the language of the localization parameter. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param localization the source of the value of the {@code Accept-Language} header * @return the result of the GET request */ - public Response get(final String url, final Localization localization) + public final Response get(final String url, final Localization localization) throws IOException, ReCaptchaException { return get(url, null, localization); } @@ -49,12 +50,12 @@ public Response get(final String url, final Localization localization) /** * Do a GET request with the specified headers. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @return the result of the GET request */ - public Response get(final String url, @Nullable final Map> headers) + public final Response get(final String url, @Nullable final Map> headers) throws IOException, ReCaptchaException { return get(url, headers, NewPipe.getPreferredLocalization()); } @@ -64,17 +65,17 @@ public Response get(final String url, @Nullable final Map> *
* It will set the {@code Accept-Language} header to the language of the localization parameter. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @param localization the source of the value of the {@code Accept-Language} header * @return the result of the GET request */ - public Response get(final String url, + public final Response get(final String url, @Nullable final Map> headers, final Localization localization) throws IOException, ReCaptchaException { - return execute(Request.newBuilder() + return executeIfHttps(Request.newBuilder() .get(url) .headers(headers) .localization(localization) @@ -84,24 +85,24 @@ public Response get(final String url, /** * Do a HEAD request. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @return the result of the HEAD request */ - public Response head(final String url) throws IOException, ReCaptchaException { + public final Response head(final String url) throws IOException, ReCaptchaException { return head(url, null); } /** * Do a HEAD request with the specified headers. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @return the result of the HEAD request */ - public Response head(final String url, @Nullable final Map> headers) + public final Response head(final String url, @Nullable final Map> headers) throws IOException, ReCaptchaException { - return execute(Request.newBuilder() + return executeIfHttps(Request.newBuilder() .head(url) .headers(headers) .build()); @@ -110,13 +111,13 @@ public Response head(final String url, @Nullable final Map> /** * Do a POST request with the specified headers, sending the data array. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @param dataToSend byte array that will be sent when doing the request. * @return the result of the POST request */ - public Response post(final String url, + public final Response post(final String url, @Nullable final Map> headers, @Nullable final byte[] dataToSend) throws IOException, ReCaptchaException { @@ -128,19 +129,19 @@ public Response post(final String url, *
* It will set the {@code Accept-Language} header to the language of the localization parameter. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @param dataToSend byte array that will be sent when doing the request. * @param localization the source of the value of the {@code Accept-Language} header * @return the result of the POST request */ - public Response post(final String url, + public final Response post(final String url, @Nullable final Map> headers, @Nullable final byte[] dataToSend, final Localization localization) throws IOException, ReCaptchaException { - return execute(Request.newBuilder() + return executeIfHttps(Request.newBuilder() .post(url, dataToSend) .headers(headers) .localization(localization) @@ -151,7 +152,7 @@ public Response post(final String url, * Convenient method to send a POST request using the specified value of the * {@code Content-Type} header with a given {@link Localization}. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @param dataToSend byte array that will be sent when doing the request. @@ -161,7 +162,7 @@ public Response post(final String url, * @return the result of the POST request * @see #post(String, Map, byte[], Localization) */ - public Response postWithContentType(final String url, + public final Response postWithContentType(final String url, @Nullable final Map> headers, @Nullable final byte[] dataToSend, final Localization localization, @@ -179,7 +180,7 @@ public Response postWithContentType(final String url, * Convenient method to send a POST request using the specified value of the * {@code Content-Type} header. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @param dataToSend byte array that will be sent when doing the request. @@ -188,7 +189,7 @@ public Response postWithContentType(final String url, * @return the result of the POST request * @see #post(String, Map, byte[], Localization) */ - public Response postWithContentType(final String url, + public final Response postWithContentType(final String url, @Nullable final Map> headers, @Nullable final byte[] dataToSend, final String contentType) @@ -201,7 +202,7 @@ public Response postWithContentType(final String url, * Convenient method to send a POST request the JSON mime type as the value of the * {@code Content-Type} header with a given {@link Localization}. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @param dataToSend byte array that will be sent when doing the request. @@ -209,7 +210,7 @@ public Response postWithContentType(final String url, * @return the result of the POST request * @see #post(String, Map, byte[], Localization) */ - public Response postWithContentTypeJson(final String url, + public final Response postWithContentTypeJson(final String url, @Nullable final Map> headers, @Nullable final byte[] dataToSend, final Localization localization) @@ -221,14 +222,14 @@ public Response postWithContentTypeJson(final String url, * Convenient method to send a POST request the JSON mime type as the value of the * {@code Content-Type} header. * - * @param url the URL that is pointing to the wanted resource + * @param url the URL that is pointing to the wanted resource (must start with HTTPS) * @param headers a list of headers that will be used in the request. * Any default headers should be overridden by these. * @param dataToSend byte array that will be sent when doing the request. * @return the result of the POST request * @see #post(String, Map, byte[], Localization) */ - public Response postWithContentTypeJson(final String url, + public final Response postWithContentTypeJson(final String url, @Nullable final Map> headers, @Nullable final byte[] dataToSend) throws IOException, ReCaptchaException { @@ -236,11 +237,24 @@ public Response postWithContentTypeJson(final String url, NewPipe.getPreferredLocalization()); } + public final Response executeIfHttps(final @Nonnull Request request) + throws IOException, ReCaptchaException { + + if (!request.url().equals(Utils.replaceHttpWithHttps(request.url()))) { + throw new IOException( + "All queries must be made using HTTPS. Extractors must guarantee " + + "that HTTPS links are provided." + ); + } else { + return execute(request); + } + } + /** * Do a request using the specified {@link Request} object. * * @return the result of the request */ - public abstract Response execute(@Nonnull Request request) + protected abstract Response execute(@Nonnull Request request) throws IOException, ReCaptchaException; } diff --git a/extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java b/extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java index 3461269914..0697c30ce2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java +++ b/extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java @@ -47,7 +47,7 @@ public static DownloaderTestImpl getInstance() { } @Override - public Response execute(@Nonnull final Request request) + protected Response execute(@Nonnull final Request request) throws IOException, ReCaptchaException { final String httpMethod = request.httpMethod(); final String url = request.url(); diff --git a/extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java b/extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java index bcf72b213e..c77a9ca69e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java +++ b/extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java @@ -46,7 +46,7 @@ public MockDownloader(@Nonnull final String path) throws IOException { } @Override - public Response execute(@Nonnull final Request request) { + protected Response execute(@Nonnull final Request request) { final Response result = mocks.get(request); if (result == null) { throw new NullPointerException("No mock response for request with url '" + request diff --git a/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java b/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java index 7d649732bb..9fcb4142b4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java +++ b/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java @@ -70,7 +70,7 @@ public RecordingDownloader(final String stringPath) throws IOException { public Response execute(@Nonnull final Request request) throws IOException, ReCaptchaException { final Downloader downloader = DownloaderTestImpl.getInstance(); - Response response = downloader.execute(request); + Response response = downloader.executeIfHttps(request); String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1"); response = new Response( response.responseCode(),