diff --git a/poetry.lock b/poetry.lock index 6bec3f2ed1..a3ad412c46 100644 --- a/poetry.lock +++ b/poetry.lock @@ -17,6 +17,26 @@ PyJWT = ">=1.0.0,<3" python-dateutil = ">=2.1.0,<3" requests = ">=2.0.0,<3" +[[package]] +name = "advocate" +version = "1.0.0" +description = "A wrapper around the requests library for safely making HTTP requests on behalf of a third party" +optional = false +python-versions = "*" +files = [ + {file = "advocate-1.0.0-py2.py3-none-any.whl", hash = "sha256:e8b340e49fadc0e416fbc9e81ef52d74858ccad16357dabde6cf9d99a7407d70"}, + {file = "advocate-1.0.0.tar.gz", hash = "sha256:1bf1170e41334279996580329c594e017540ab0eaf7a152323e743f0a85a353d"}, +] + +[package.dependencies] +ndg-httpsclient = "*" +netifaces = ">=0.10.5" +pyasn1 = "*" +pyopenssl = "*" +requests = ">=2.18.0,<3.0" +six = "*" +urllib3 = ">=1.22,<2.0" + [[package]] name = "alembic" version = "1.13.1" @@ -2388,6 +2408,61 @@ files = [ {file = "mysqlclient-2.1.1.tar.gz", hash = "sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782"}, ] +[[package]] +name = "ndg-httpsclient" +version = "0.5.1" +description = "Provides enhanced HTTPS support for httplib and urllib2 using PyOpenSSL" +optional = false +python-versions = ">=2.7,<3.0.dev0 || >=3.4.dev0" +files = [ + {file = "ndg_httpsclient-0.5.1-py2-none-any.whl", hash = "sha256:d2c7225f6a1c6cf698af4ebc962da70178a99bcde24ee6d1961c4f3338130d57"}, + {file = "ndg_httpsclient-0.5.1-py3-none-any.whl", hash = "sha256:dd174c11d971b6244a891f7be2b32ca9853d3797a72edb34fa5d7b07d8fff7d4"}, + {file = "ndg_httpsclient-0.5.1.tar.gz", hash = "sha256:d72faed0376ab039736c2ba12e30695e2788c4aa569c9c3e3d72131de2592210"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.1" +PyOpenSSL = "*" + +[[package]] +name = "netifaces" +version = "0.11.0" +description = "Portable network interface information." +optional = false +python-versions = "*" +files = [ + {file = "netifaces-0.11.0-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eb4813b77d5df99903af4757ce980a98c4d702bbcb81f32a0b305a1537bdf0b1"}, + {file = "netifaces-0.11.0-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:5f9ca13babe4d845e400921973f6165a4c2f9f3379c7abfc7478160e25d196a4"}, + {file = "netifaces-0.11.0-cp27-cp27m-win32.whl", hash = "sha256:7dbb71ea26d304e78ccccf6faccef71bb27ea35e259fb883cfd7fd7b4f17ecb1"}, + {file = "netifaces-0.11.0-cp27-cp27m-win_amd64.whl", hash = "sha256:0f6133ac02521270d9f7c490f0c8c60638ff4aec8338efeff10a1b51506abe85"}, + {file = "netifaces-0.11.0-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:08e3f102a59f9eaef70948340aeb6c89bd09734e0dca0f3b82720305729f63ea"}, + {file = "netifaces-0.11.0-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c03fb2d4ef4e393f2e6ffc6376410a22a3544f164b336b3a355226653e5efd89"}, + {file = "netifaces-0.11.0-cp34-cp34m-win32.whl", hash = "sha256:73ff21559675150d31deea8f1f8d7e9a9a7e4688732a94d71327082f517fc6b4"}, + {file = "netifaces-0.11.0-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815eafdf8b8f2e61370afc6add6194bd5a7252ae44c667e96c4c1ecf418811e4"}, + {file = "netifaces-0.11.0-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:50721858c935a76b83dd0dd1ab472cad0a3ef540a1408057624604002fcfb45b"}, + {file = "netifaces-0.11.0-cp35-cp35m-win32.whl", hash = "sha256:c9a3a47cd3aaeb71e93e681d9816c56406ed755b9442e981b07e3618fb71d2ac"}, + {file = "netifaces-0.11.0-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:aab1dbfdc55086c789f0eb37affccf47b895b98d490738b81f3b2360100426be"}, + {file = "netifaces-0.11.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c37a1ca83825bc6f54dddf5277e9c65dec2f1b4d0ba44b8fd42bc30c91aa6ea1"}, + {file = "netifaces-0.11.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:28f4bf3a1361ab3ed93c5ef360c8b7d4a4ae060176a3529e72e5e4ffc4afd8b0"}, + {file = "netifaces-0.11.0-cp36-cp36m-win32.whl", hash = "sha256:2650beee182fed66617e18474b943e72e52f10a24dc8cac1db36c41ee9c041b7"}, + {file = "netifaces-0.11.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cb925e1ca024d6f9b4f9b01d83215fd00fe69d095d0255ff3f64bffda74025c8"}, + {file = "netifaces-0.11.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:84e4d2e6973eccc52778735befc01638498781ce0e39aa2044ccfd2385c03246"}, + {file = "netifaces-0.11.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18917fbbdcb2d4f897153c5ddbb56b31fa6dd7c3fa9608b7e3c3a663df8206b5"}, + {file = "netifaces-0.11.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:48324183af7f1bc44f5f197f3dad54a809ad1ef0c78baee2c88f16a5de02c4c9"}, + {file = "netifaces-0.11.0-cp37-cp37m-win32.whl", hash = "sha256:8f7da24eab0d4184715d96208b38d373fd15c37b0dafb74756c638bd619ba150"}, + {file = "netifaces-0.11.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2479bb4bb50968089a7c045f24d120f37026d7e802ec134c4490eae994c729b5"}, + {file = "netifaces-0.11.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:3ecb3f37c31d5d51d2a4d935cfa81c9bc956687c6f5237021b36d6fdc2815b2c"}, + {file = "netifaces-0.11.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:96c0fe9696398253f93482c84814f0e7290eee0bfec11563bd07d80d701280c3"}, + {file = "netifaces-0.11.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c92ff9ac7c2282009fe0dcb67ee3cd17978cffbe0c8f4b471c00fe4325c9b4d4"}, + {file = "netifaces-0.11.0-cp38-cp38-win32.whl", hash = "sha256:d07b01c51b0b6ceb0f09fc48ec58debd99d2c8430b09e56651addeaf5de48048"}, + {file = "netifaces-0.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:469fc61034f3daf095e02f9f1bbac07927b826c76b745207287bc594884cfd05"}, + {file = "netifaces-0.11.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5be83986100ed1fdfa78f11ccff9e4757297735ac17391b95e17e74335c2047d"}, + {file = "netifaces-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54ff6624eb95b8a07e79aa8817288659af174e954cca24cdb0daeeddfc03c4ff"}, + {file = "netifaces-0.11.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:841aa21110a20dc1621e3dd9f922c64ca64dd1eb213c47267a2c324d823f6c8f"}, + {file = "netifaces-0.11.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e76c7f351e0444721e85f975ae92718e21c1f361bda946d60a214061de1f00a1"}, + {file = "netifaces-0.11.0.tar.gz", hash = "sha256:043a79146eb2907edf439899f262b3dfe41717d34124298ed281139a8b93ca32"}, +] + [[package]] name = "nodeenv" version = "1.8.0" @@ -5225,4 +5300,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "10482b8930a9e3c91c95dc79105f8bc35d9f4440e112f41fba61b74ed7ae0871" +content-hash = "e7985ee5c3ca3a4389b4e85fda033a9b3b867dbbe4b4a7fca8ea5c35fc401148" diff --git a/pyproject.toml b/pyproject.toml index 8684dc3e81..e5cc871457 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ readme = "README.md" [tool.poetry.dependencies] python = ">=3.8,<3.11" +advocate = "1.0.0" aniso8601 = "8.0.0" authlib = "0.15.5" backoff = "2.2.1" diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py index 14a42cc3ca..10d6d6edf5 100644 --- a/redash/query_runner/__init__.py +++ b/redash/query_runner/__init__.py @@ -3,13 +3,17 @@ from contextlib import ExitStack from functools import wraps -import requests import sqlparse from dateutil import parser from rq.timeouts import JobTimeoutException from sshtunnel import open_tunnel from redash import settings, utils +from redash.utils.requests_session import ( + UnacceptableAddressException, + requests_or_advocate, + requests_session, +) logger = logging.getLogger(__name__) @@ -375,7 +379,7 @@ def get_response(self, url, auth=None, http_method="get", **kwargs): error = None response = None try: - response = requests.request(http_method, url, auth=auth, **kwargs) + response = requests_session.request(http_method, url, auth=auth, **kwargs) # Raise a requests HTTP exception with the appropriate reason # for 4xx and 5xx response status codes which is later caught # and passed back. @@ -385,11 +389,14 @@ def get_response(self, url, auth=None, http_method="get", **kwargs): if response.status_code != 200: error = "{} ({}).".format(self.response_error, response.status_code) - except requests.HTTPError as exc: + except requests_or_advocate.HTTPError as exc: logger.exception(exc) error = "Failed to execute query. " f"Return Code: {response.status_code} Reason: {response.text}" - except requests.RequestException as exc: + except UnacceptableAddressException as exc: + logger.exception(exc) + error = "Can't query private addresses." + except requests_or_advocate.RequestException as exc: # Catch all other requests exceptions and return the error. logger.exception(exc) error = str(exc) diff --git a/redash/query_runner/csv.py b/redash/query_runner/csv.py index 4c72f3c3fa..3d3cf61b9c 100644 --- a/redash/query_runner/csv.py +++ b/redash/query_runner/csv.py @@ -1,10 +1,13 @@ import io import logging -import requests import yaml from redash.query_runner import BaseQueryRunner, NotSupported, register +from redash.utils.requests_session import ( + UnacceptableAddressException, + requests_or_advocate, +) logger = logging.getLogger(__name__) @@ -56,7 +59,7 @@ def run_query(self, query, user): pass try: - response = requests.get(url=path, headers={"User-agent": ua}) + response = requests_or_advocate.get(url=path, headers={"User-agent": ua}) workbook = pd.read_csv(io.BytesIO(response.content), sep=",", **args) df = workbook.copy() @@ -96,6 +99,9 @@ def run_query(self, query, user): except KeyboardInterrupt: error = "Query cancelled by user." data = None + except UnacceptableAddressException: + error = "Can't query private addresses." + data = None except Exception as e: error = "Error reading {0}. {1}".format(path, str(e)) data = None diff --git a/redash/query_runner/excel.py b/redash/query_runner/excel.py index 5606149b31..488632e022 100644 --- a/redash/query_runner/excel.py +++ b/redash/query_runner/excel.py @@ -1,9 +1,12 @@ import logging -import requests import yaml from redash.query_runner import BaseQueryRunner, NotSupported, register +from redash.utils.requests_session import ( + UnacceptableAddressException, + requests_or_advocate, +) logger = logging.getLogger(__name__) @@ -54,7 +57,7 @@ def run_query(self, query, user): pass try: - response = requests.get(url=path, headers={"User-agent": ua}) + response = requests_or_advocate.get(url=path, headers={"User-agent": ua}) workbook = pd.read_excel(response.content, **args) df = workbook.copy() @@ -94,6 +97,9 @@ def run_query(self, query, user): except KeyboardInterrupt: error = "Query cancelled by user." data = None + except UnacceptableAddressException: + error = "Can't query private addresses." + data = None except Exception as e: error = "Error reading {0}. {1}".format(path, str(e)) data = None diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index 449f773a19..75c438cff2 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -71,6 +71,9 @@ # Whether file downloads are enforced or not. ENFORCE_FILE_SAVE = parse_boolean(os.environ.get("REDASH_ENFORCE_FILE_SAVE", "true")) +# Whether api calls using the json query runner will block private addresses +ENFORCE_PRIVATE_ADDRESS_BLOCK = parse_boolean(os.environ.get("REDASH_ENFORCE_PRIVATE_IP_BLOCK", "true")) + # Whether to use secure cookies by default. COOKIES_SECURE = parse_boolean(os.environ.get("REDASH_COOKIES_SECURE", str(ENFORCE_HTTPS))) # Whether the session cookie is set to secure. diff --git a/redash/utils/requests_session.py b/redash/utils/requests_session.py new file mode 100644 index 0000000000..c562894181 --- /dev/null +++ b/redash/utils/requests_session.py @@ -0,0 +1,18 @@ +from advocate.exceptions import UnacceptableAddressException # noqa: F401 + +from redash import settings + +if settings.ENFORCE_PRIVATE_ADDRESS_BLOCK: + import advocate as requests_or_advocate +else: + import requests as requests_or_advocate + + +class ConfiguredSession(requests_or_advocate.Session): + def request(self, *args, **kwargs): + if not settings.REQUESTS_ALLOW_REDIRECTS: + kwargs.update({"allow_redirects": False}) + return super().request(*args, **kwargs) + + +requests_session = ConfiguredSession() diff --git a/tests/query_runner/test_http.py b/tests/query_runner/test_http.py index d3e439ccb9..1fa361cb92 100644 --- a/tests/query_runner/test_http.py +++ b/tests/query_runner/test_http.py @@ -1,9 +1,12 @@ from unittest import TestCase import mock -import requests from redash.query_runner import BaseHTTPQueryRunner +from redash.utils.requests_session import ( + ConfiguredSession, + requests_or_advocate, +) class RequiresAuthQueryRunner(BaseHTTPQueryRunner): @@ -34,7 +37,7 @@ def test_get_auth_empty_requires_authentication(self): query_runner = RequiresAuthQueryRunner({}) self.assertRaisesRegex(ValueError, "Username and Password required", query_runner.get_auth) - @mock.patch("requests.request") + @mock.patch.object(ConfiguredSession, "request") def test_get_response_success(self, mock_get): mock_response = mock.Mock() mock_response.status_code = 200 @@ -48,7 +51,7 @@ def test_get_response_success(self, mock_get): self.assertEqual(response.status_code, 200) self.assertIsNone(error) - @mock.patch("requests.request") + @mock.patch.object(ConfiguredSession, "request") def test_get_response_success_custom_auth(self, mock_get): mock_response = mock.Mock() mock_response.status_code = 200 @@ -63,7 +66,7 @@ def test_get_response_success_custom_auth(self, mock_get): self.assertEqual(response.status_code, 200) self.assertIsNone(error) - @mock.patch("requests.request") + @mock.patch.object(ConfiguredSession, "request") def test_get_response_failure(self, mock_get): mock_response = mock.Mock() mock_response.status_code = 301 @@ -76,12 +79,12 @@ def test_get_response_failure(self, mock_get): mock_get.assert_called_once_with("get", url, auth=None) self.assertIn(query_runner.response_error, error) - @mock.patch("requests.request") + @mock.patch.object(ConfiguredSession, "request") def test_get_response_httperror_exception(self, mock_get): mock_response = mock.Mock() mock_response.status_code = 500 mock_response.text = "Server Error" - http_error = requests.HTTPError() + http_error = requests_or_advocate.HTTPError() mock_response.raise_for_status.side_effect = http_error mock_get.return_value = mock_response @@ -92,13 +95,13 @@ def test_get_response_httperror_exception(self, mock_get): self.assertIsNotNone(error) self.assertIn("Failed to execute query", error) - @mock.patch("requests.request") + @mock.patch.object(ConfiguredSession, "request") def test_get_response_requests_exception(self, mock_get): mock_response = mock.Mock() mock_response.status_code = 500 mock_response.text = "Server Error" exception_message = "Some requests exception" - requests_exception = requests.RequestException(exception_message) + requests_exception = requests_or_advocate.RequestException(exception_message) mock_response.raise_for_status.side_effect = requests_exception mock_get.return_value = mock_response @@ -109,7 +112,7 @@ def test_get_response_requests_exception(self, mock_get): self.assertIsNotNone(error) self.assertEqual(exception_message, error) - @mock.patch("requests.request") + @mock.patch.object(ConfiguredSession, "request") def test_get_response_generic_exception(self, mock_get): mock_response = mock.Mock() mock_response.status_code = 500