Run linters on Python 3.11, upgrade pylint (#4243)

.github/workflows/py-ci.yml

@@ -13,14 +13,14 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools
-          python -m pip install 'pylint==2.10.2' 'pycodestyle>=2.6.1'
+          python -m pip install 'pylint==3.2.7' 'pycodestyle>=2.6.1'
       - name: Run pylint
         run: |
-          pylint $(find -name '*.py' -not -path './venv/*')
+          pylint .
       - name: Run pycodestyle
         run: |
           pycodestyle $(find -name '*.py' -not -path './venv/*')
@@ -31,7 +31,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools
@@ -70,7 +70,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: install poetry
         run: python -m pip install poetry
       - name: install deps

api/python/quilt3/data_transfer.py

@@ -136,7 +136,7 @@ def find_correct_client(self, api_type, bucket, param_dict):
                 S3Api.LIST_OBJECTS_V2: check_list_objects_v2_works_for_client,
                 S3Api.LIST_OBJECT_VERSIONS: check_list_object_versions_works_for_client
             }
-            assert api_type in check_fn_mapper.keys(), f"Only certain APIs are supported with unsigned_client. The " \
+            assert api_type in check_fn_mapper, f"Only certain APIs are supported with unsigned_client. The " \
                 f"API '{api_type}' is not current supported. You may want to use S3ClientProvider.standard_client " \
                 f"instead "
             check_fn = check_fn_mapper[api_type]

api/python/tests/integration/test_packages.py

@@ -1390,7 +1390,7 @@ def test_filter(self):
         pkg['b'].set_meta({'foo': 'bar'})
 
         p_copy = pkg.filter(lambda lk, entry: lk == 'a/', include_directories=True)
-        assert list(p_copy) == []
+        assert not list(p_copy)
 
         p_copy = pkg.filter(lambda lk, entry: lk in ('a/', 'a/df'), include_directories=True)
         assert list(p_copy) == ['a'] and list(p_copy['a']) == ['df']
@@ -1872,7 +1872,8 @@ def test_push_dest_fn_non_string(self):
             with self.subTest(value=val):
                 with pytest.raises(TypeError) as excinfo:
                     pkg.push('foo/bar', registry='s3://test-bucket',
-                             dest=(lambda v: lambda *args, **kwargs: v)(val), force=True)
+                             # pylint: disable=cell-var-from-loop
+                             dest=lambda *args, **kwargs: val, force=True)
                 assert 'str is expected' in str(excinfo.value)
 
     @patch('quilt3.workflows.validate', mock.MagicMock(return_value=None))
@@ -1882,7 +1883,8 @@ def test_push_dest_fn_non_supported_uri(self):
             with self.subTest(value=val):
                 with pytest.raises(quilt3.util.URLParseError):
                     pkg.push('foo/bar', registry='s3://test-bucket',
-                             dest=(lambda v: lambda *args, **kwargs: v)(val), force=True)
+                             # pylint: disable=cell-var-from-loop
+                             dest=lambda *args, **kwargs: val, force=True)
 
     @patch('quilt3.workflows.validate', mock.MagicMock(return_value=None))
     def test_push_dest_fn_s3_uri_with_version_id(self):

lambdas/indexer/test/test_index.py

@@ -1504,8 +1504,7 @@ def test_get_contents_large(self, get_memory_mock):
     @pytest.mark.extended
     @patch('document_queue.ELASTIC_LIMIT_BYTES', 64_000)
     def test_get_contents_extended(self):
-        directory = (BASE_DIR / 'extended')
-        files = directory.glob('**/*-c000')
+        files = (BASE_DIR / 'extended').glob('**/*-c000')
         for f in files:
             parquet = f.read_bytes()
             size = len(parquet)
@@ -1657,8 +1656,7 @@ def test_parquet_contents(self):
     # see PRE conditions in conftest.py
     @pytest.mark.extended
     def test_parquet_extended(self):
-        directory = (BASE_DIR / 'extended')
-        files = directory.glob('**/*.parquet')
+        files = (BASE_DIR / 'extended').glob('**/*.parquet')
         for f in files:
             print(f"Testing {f}")
             parquet = f.read_bytes()

lambdas/shared/tests/test_decorator.py

@@ -184,6 +184,7 @@ def handler(request):
         code, body, headers = handler(Request(self._make_get({'foo': 'bar'}, None), None))
         assert code == 200
         assert body == 'blah'
+        # pylint: disable=use-implicit-booleaness-not-comparison
         assert headers == {}
 
         code, _, headers = handler(Request(self._make_get(None, None), None))

pylintrc

@@ -1,9 +1,12 @@
 [MASTER]
-ignore-paths=.*/api/python/quilt3/admin/_graphql_client/.*
+ignore-paths=api/python/quilt3/admin/_graphql_client/.*,.*/venv/.*,catalog/.*
+recursive=yes
 
 [MESSAGES CONTROL]
 disable=attribute-defined-outside-init,
         broad-except,
+        broad-exception-raised,
+        consider-using-f-string,
         consider-using-sys-exit,
         cyclic-import,
         duplicate-code,
@@ -28,7 +31,6 @@ disable=attribute-defined-outside-init,
         no-member,
         no-name-in-module,
         no-self-argument,
-        no-self-use,
         not-callable,
         no-value-for-parameter,
         pointless-statement,
@@ -47,7 +49,9 @@ disable=attribute-defined-outside-init,
         too-many-public-methods,
         too-many-statements,
         unnecessary-pass,
+        unspecified-encoding,
         unused-argument,
         unused-variable,
+        use-dict-literal,
         wrong-import-order,
         wrong-import-position,