Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Throw upon cleartext HTTP queries #1181

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
+45 −31
Open

Throw upon cleartext HTTP queries #1181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 42 additions & 28 deletions extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
Expand All @@ -25,10 +26,10 @@ public abstract class Downloader {
* localization. It should only be used when the resource that will be fetched won't be affected
* by the localization.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the GET request
*/
public Response get(final String url) throws IOException, ReCaptchaException {
public final Response get(final String url) throws IOException, ReCaptchaException {
return get(url, null, NewPipe.getPreferredLocalization());
}

Expand All @@ -37,24 +38,24 @@ public Response get(final String url) throws IOException, ReCaptchaException {
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(final String url, final Localization localization)
public final Response get(final String url, final Localization localization)
throws IOException, ReCaptchaException {
return get(url, null, localization);
}

/**
* Do a GET request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the GET request
*/
public Response get(final String url, @Nullable final Map<String, List<String>> headers)
public final Response get(final String url, @Nullable final Map<String, List<String>> headers)
throws IOException, ReCaptchaException {
return get(url, headers, NewPipe.getPreferredLocalization());
}
Expand All @@ -64,17 +65,17 @@ public Response get(final String url, @Nullable final Map<String, List<String>>
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(final String url,
public final Response get(final String url,
@Nullable final Map<String, List<String>> headers,
final Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.get(url)
.headers(headers)
.localization(localization)
Expand All @@ -84,24 +85,24 @@ public Response get(final String url,
/**
* Do a HEAD request.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the HEAD request
*/
public Response head(final String url) throws IOException, ReCaptchaException {
public final Response head(final String url) throws IOException, ReCaptchaException {
return head(url, null);
}

/**
* Do a HEAD request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the HEAD request
*/
public Response head(final String url, @Nullable final Map<String, List<String>> headers)
public final Response head(final String url, @Nullable final Map<String, List<String>> headers)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.head(url)
.headers(headers)
.build());
Expand All @@ -110,13 +111,13 @@ public Response head(final String url, @Nullable final Map<String, List<String>>
/**
* Do a POST request with the specified headers, sending the data array.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
*/
public Response post(final String url,
public final Response post(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
Expand All @@ -128,19 +129,19 @@ public Response post(final String url,
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the POST request
*/
public Response post(final String url,
public final Response post(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.post(url, dataToSend)
.headers(headers)
.localization(localization)
Expand All @@ -151,7 +152,7 @@ public Response post(final String url,
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header with a given {@link Localization}.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
Expand All @@ -161,7 +162,7 @@ public Response post(final String url,
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentType(final String url,
public final Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization,
Expand All @@ -179,7 +180,7 @@ public Response postWithContentType(final String url,
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
Expand All @@ -188,7 +189,7 @@ public Response postWithContentType(final String url,
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentType(final String url,
public final Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final String contentType)
Expand All @@ -201,15 +202,15 @@ public Response postWithContentType(final String url,
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header with a given {@link Localization}.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentTypeJson(final String url,
public final Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
Expand All @@ -221,26 +222,39 @@ public Response postWithContentTypeJson(final String url,
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentTypeJson(final String url,
public final Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
return postWithContentTypeJson(url, headers, dataToSend,
NewPipe.getPreferredLocalization());
}

public final Response executeIfHttps(final @Nonnull Request request)
throws IOException, ReCaptchaException {

if (!request.url().equals(Utils.replaceHttpWithHttps(request.url()))) {
throw new IOException(
"All queries must be made using HTTPS. Extractors must guarantee "
+ "that HTTPS links are provided."
);
} else {
return execute(request);
}
}

/**
* Do a request using the specified {@link Request} object.
*
* @return the result of the request
*/
public abstract Response execute(@Nonnull Request request)
protected abstract Response execute(@Nonnull Request request)
throws IOException, ReCaptchaException;
}
2 changes: 1 addition & 1 deletion extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public static DownloaderTestImpl getInstance() {
}

@Override
public Response execute(@Nonnull final Request request)
protected Response execute(@Nonnull final Request request)
throws IOException, ReCaptchaException {
final String httpMethod = request.httpMethod();
final String url = request.url();
Expand Down
2 changes: 1 addition & 1 deletion extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public MockDownloader(@Nonnull final String path) throws IOException {
}

@Override
public Response execute(@Nonnull final Request request) {
protected Response execute(@Nonnull final Request request) {
final Response result = mocks.get(request);
if (result == null) {
throw new NullPointerException("No mock response for request with url '" + request
Expand Down
2 changes: 1 addition & 1 deletion extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public RecordingDownloader(final String stringPath) throws IOException {
public Response execute(@Nonnull final Request request) throws IOException,
ReCaptchaException {
final Downloader downloader = DownloaderTestImpl.getInstance();
Response response = downloader.execute(request);
Response response = downloader.executeIfHttps(request);
String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");
response = new Response(
response.responseCode(),
Expand Down
Loading