diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f35f4c6..bdc3798 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,27 +9,11 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-18.04 - python-version: 2.7 - env: - TOXENV: py27 - - os: ubuntu-18.04 - python-version: 3.4 - env: - TOXENV: py - - os: ubuntu-18.04 - python-version: 3.5 - env: - TOXENV: py35 - - python-version: 3.6 - env: - TOXENV: py - - python-version: 3.7 - env: - TOXENV: py - - python-version: 3.8 - env: - TOXENV: py + - python-version: '3.7' + - python-version: '3.8' + - python-version: '3.9' + - python-version: '3.10' + - python-version: '3.11' steps: - uses: actions/checkout@v2 @@ -44,7 +28,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run tests - env: ${{ matrix.env }} + env: + TOXENV: py run: | pip install -U tox SPLASH_URL=http://127.0.0.1:8050 tox diff --git a/CHANGES.rst b/CHANGES.rst index 3daeca7..86e0a45 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changes ======= +0.9.0 (to be released) +---------------------- + +* Removed official support for Python 2.7, 3.4, 3.5 and 3.6, and added official + support for Python 3.9, 3.10 and 3.11. + +* Deprecated ``SplashJsonResponse.body_as_unicode()``, to be replaced by + ``SplashJsonResponse.text``. + +* Removed calls to obsolete ``to_native_str``, removed in Scrapy 2.8. + 0.8.0 (2021-10-05) ------------------ diff --git a/README.rst b/README.rst index 94a8f82..a40f579 100644 --- a/README.rst +++ b/README.rst @@ -460,7 +460,7 @@ Run a simple `Splash Lua Script`_:: # ... def parse_result(self, response): - doc_title = response.body_as_unicode() + doc_title = response.text # ... diff --git a/scrapy_splash/request.py b/scrapy_splash/request.py index 0af10e9..878979c 100644 --- a/scrapy_splash/request.py +++ b/scrapy_splash/request.py @@ -90,13 +90,11 @@ def _original_url(self): def _original_method(self): return self._splash_args.get('http_method', 'GET') - def __str__(self): + def __repr__(self): if not self._processed: - return super(SplashRequest, self).__str__() + return super().__repr__() return "<%s %s via %s>" % (self._original_method, self._original_url, self.url) - __repr__ = __str__ - class SplashFormRequest(SplashRequest, FormRequest): """ diff --git a/scrapy_splash/response.py b/scrapy_splash/response.py index e5250c2..15f0345 100644 --- a/scrapy_splash/response.py +++ b/scrapy_splash/response.py @@ -4,6 +4,7 @@ import json import base64 import re +from warnings import warn from scrapy.http import Response, TextResponse from scrapy import Selector @@ -129,6 +130,14 @@ def text(self): return self._ubody def body_as_unicode(self): + warn( + ( + "The body_as_unicode() method is deprecated, use the text " + "property instead." + ), + DeprecationWarning, + stacklevel=2, + ) return self._ubody @property diff --git a/setup.py b/setup.py index e330d33..8aefcc6 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='scrapy-splash', - version='0.7.2', + version='0.8.0', url='https://github.com/scrapy-plugins/scrapy-splash', description='JavaScript support for Scrapy using Splash', long_description=open('README.rst').read() + "\n\n" + open("CHANGES.rst").read(), diff --git a/tests/test_fingerprints.py b/tests/test_fingerprints.py index 7051f31..309e48a 100644 --- a/tests/test_fingerprints.py +++ b/tests/test_fingerprints.py @@ -4,7 +4,7 @@ import pytest import scrapy -from scrapy.dupefilters import request_fingerprint +from scrapy.utils.request import request_fingerprint from scrapy_splash import SplashRequest from scrapy_splash.dupefilter import splash_request_fingerprint diff --git a/tests/test_integration.py b/tests/test_integration.py index 8ca28b3..f5c1072 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -340,7 +340,7 @@ def test_access_http_auth(settings): items, url, crawler = yield crawl_items(LuaSpider, HelloWorldProtected, settings, kwargs) response = assert_single_response(items) - assert 'hello' in response.body_as_unicode() + assert 'hello' in response.text assert response.status == 200 assert response.splash_response_status == 200 @@ -351,8 +351,8 @@ def test_protected_splash_no_auth(settings_auth): items, url, crawler = yield crawl_items(LuaSpider, HelloWorld, settings_auth) response = assert_single_response(items) - assert 'Unauthorized' in response.body_as_unicode() - assert 'hello' not in response.body_as_unicode() + assert 'Unauthorized' in response.text + assert 'hello' not in response.text assert response.status == 401 assert response.splash_response_status == 401 @@ -367,7 +367,7 @@ def test_protected_splash_manual_headers_auth(settings_auth): items, url, crawler = yield crawl_items(LuaSpider, HelloWorld, settings_auth, kwargs) response = assert_single_response(items) - assert 'hello' in response.body_as_unicode() + assert 'hello' in response.text assert response.status == 200 assert response.splash_response_status == 200 @@ -375,7 +375,7 @@ def test_protected_splash_manual_headers_auth(settings_auth): items, url, crawler = yield crawl_items(LuaSpider, HelloWorldProtected, settings_auth, kwargs) response = assert_single_response(items) - assert 'hello' not in response.body_as_unicode() + assert 'hello' not in response.text assert response.status == 401 assert response.splash_response_status == 200 @@ -390,8 +390,8 @@ def test_protected_splash_settings_auth(settings_auth): items, url, crawler = yield crawl_items(LuaSpider, HelloWorld, settings_auth) response = assert_single_response(items) - assert 'Unauthorized' not in response.body_as_unicode() - assert 'hello' in response.body_as_unicode() + assert 'Unauthorized' not in response.text + assert 'hello' in response.text assert response.status == 200 assert response.splash_response_status == 200 @@ -418,7 +418,7 @@ def test_protected_splash_settings_auth(settings_auth): response = assert_single_response(items) assert response.status == 200 assert response.splash_response_status == 200 - assert 'hello' in response.body_as_unicode() + assert 'hello' in response.text # enable remote auth, but not splash auth - request should fail del settings_auth['SPLASH_USER'] @@ -439,8 +439,8 @@ def test_protected_splash_httpauth_middleware(settings_auth): items, url, crawler = yield crawl_items(ScrapyAuthSpider, HelloWorld, settings_auth) response = assert_single_response(items) - assert 'Unauthorized' not in response.body_as_unicode() - assert 'hello' in response.body_as_unicode() + assert 'Unauthorized' not in response.text + assert 'hello' in response.text assert response.status == 200 assert response.splash_response_status == 200 @@ -449,7 +449,7 @@ def test_protected_splash_httpauth_middleware(settings_auth): HelloWorldProtected, settings_auth) response = assert_single_response(items) - assert 'hello' not in response.body_as_unicode() + assert 'hello' not in response.text assert response.status == 401 assert response.splash_response_status == 200 @@ -458,7 +458,7 @@ def test_protected_splash_httpauth_middleware(settings_auth): HelloWorldDisallowAuth, settings_auth) response = assert_single_response(items) - assert 'hello' in response.body_as_unicode() + assert 'hello' in response.text assert response.status == 200 assert response.splash_response_status == 200 @@ -467,7 +467,7 @@ def test_protected_splash_httpauth_middleware(settings_auth): HelloWorldProtected, settings_auth) response = assert_single_response(items) - assert 'hello' in response.body_as_unicode() + assert 'hello' in response.text assert response.status == 200 assert not hasattr(response, 'splash_response_status') diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 777aa54..193bcd0 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -188,7 +188,7 @@ def cb(): assert response2.url == req.meta['splash']['args']['url'] assert response2.data == res assert response2.body == res_body.encode('utf8') - assert response2.text == response2.body_as_unicode() == res_body + assert response2.text == response2.text == res_body assert response2.encoding == 'utf8' assert response2.headers == {b'Content-Type': [b'application/json']} assert response2.splash_response_headers == response2.headers diff --git a/tox.ini b/tox.ini index 5f00eda..8fbd3e2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27,py34,py35,py36,py37,py38 +envlist = py37,py38,py39,py310,py311 [common] deps = @@ -24,15 +24,3 @@ deps = commands = pip install -e . py.test --doctest-modules --cov=scrapy_splash {posargs:scrapy_splash tests} - -[testenv:py27] -deps = - {[common]deps} - queuelib < 1.6.0 - scrapy < 2 - -[testenv:py35] -deps = - {[common]deps} - # https://github.com/scrapy/scrapy/pull/4094#issuecomment-704092404 - scrapy < 2