extract_declared_dependencies.parse_setup_py(): Handle legacy encodings

Use tokenizer.open() to automatically handle parsing any encoding
declarations[1] at the start of a setup.py file, and prepare a text
mode file object that automatically uses the appropriate decoder.

[1]: https://docs.python.org/3/reference/lexical_analysis.html#encoding-declarations

fawltydeps/extract_declared_dependencies.py

@@ -5,6 +5,7 @@
 import logging
 import re
 import sys
+import tokenize
 from dataclasses import replace
 from pathlib import Path
 from tempfile import NamedTemporaryFile
@@ -111,7 +112,8 @@ def _is_setup_function_call(node: ast.AST) -> bool:
             and node.value.func.id == "setup"
         )
 
-    setup_contents = ast.parse(path.read_text(), filename=str(source.path))
+    with tokenize.open(path) as setup_py:
+        setup_contents = ast.parse(setup_py.read(), filename=str(source.path))
     for node in ast.walk(setup_contents):
         tracked_vars.evaluate(node)
         if _is_setup_function_call(node):

tests/conftest.py

@@ -24,12 +24,15 @@ def local_pypi(request, monkeypatch):
 
 @pytest.fixture
 def write_tmp_files(tmp_path: Path):
-    def _inner(file_contents: Dict[str, str]) -> Path:
+    def _inner(file_contents: Dict[str, Union[str, bytes]]) -> Path:
         for filename, contents in file_contents.items():
             path = tmp_path / filename
             assert path.relative_to(tmp_path)
             path.parent.mkdir(parents=True, exist_ok=True)
-            path.write_text(dedent(contents))
+            if isinstance(contents, bytes):
+                path.write_bytes(contents)
+            else:
+                path.write_text(dedent(contents))
         return tmp_path
 
     return _inner

tests/test_extract_declared_dependencies_success.py

@@ -14,7 +14,12 @@
 from fawltydeps.traverse_project import find_sources
 from fawltydeps.types import DepsSource
 
-from .utils import assert_unordered_equivalence, collect_dep_names, deps_factory
+from .utils import (
+    assert_unordered_equivalence,
+    collect_dep_names,
+    dedent_bytes,
+    deps_factory,
+)
 
 
 @pytest.mark.parametrize(
@@ -281,6 +286,21 @@ def setup(**kwargs):
             ["pandas", "click", "annoy", "foobar"],
             id="nested_variable_reference__succeeds",
         ),
+        pytest.param(
+            dedent_bytes(
+                b"""\
+                # -*- coding: big5 -*-
+                from setuptools import setup
+
+                setup(
+                    name="\xa4@\xa8\xc7\xa4\xa4\xa4\xe5\xa6r\xb2\xc5",
+                    install_requires=["pandas", "click"]
+                )
+                """
+            ),
+            ["pandas", "click"],
+            id="legacy_encoding__succeeds",
+        ),
     ],
 )
 def test_parse_setup_py(write_tmp_files, file_content, expect_deps):