Avoid imports inside functions

.github/workflows/tests.yml

@@ -25,31 +25,46 @@ jobs:
       run: tox -e py
 
     - name: Upload coverage report
-      run: bash <(curl -s https://codecov.io/bash)
+      run: |
+        curl -Os https://uploader.codecov.io/latest/linux/codecov
+        chmod +x codecov
+        ./codecov
 
   tests-other:
-    name: "Test: py3, ${{ matrix.os }}"
+    name: "Test: py38-scrapy22, Ubuntu"
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install tox
+      run: pip install tox
+
+    - name: Run tests
+      run: tox -e py38-scrapy22
+
+    - name: Upload coverage report
+      run: |
+        curl -Os https://uploader.codecov.io/latest/linux/codecov
+        chmod +x codecov
+        ./codecov
+
+  tests-other-os:
+    name: "Test: py38, ${{ matrix.os }}"
     runs-on: "${{ matrix.os }}"
     strategy:
       matrix:
-        include:
-        - python-version: 3
-          os: ubuntu-latest
-          env:
-            TOXENV: py38-scrapy22
-        - python-version: 3
-          os: macos-latest
-          env:
-            TOXENV: py
-        - python-version: 3
-          os: windows-latest
-          env:
-            TOXENV: py
+        os: [macos-latest, windows-latest]
 
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.8
+    - name: Set up Python
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
@@ -58,5 +73,4 @@ jobs:
       run: pip install tox
 
     - name: Run tests
-      env: ${{ matrix.env }}
       run: tox -e py

itemadapter/_imports.py

@@ -0,0 +1,22 @@
+# attempt the following imports only once,
+# to be imported from itemadapter's submodules
+
+try:
+    import scrapy  # pylint: disable=W0611 (unused-import)
+except ImportError:
+    scrapy = None  # type: ignore [assignment]
+
+try:
+    import dataclasses  # pylint: disable=W0611 (unused-import)
+except ImportError:
+    dataclasses = None  # type: ignore [assignment]
+
+try:
+    import attr  # pylint: disable=W0611 (unused-import)
+except ImportError:
+    attr = None  # type: ignore [assignment]
+
+try:
+    import pydantic  # pylint: disable=W0611 (unused-import)
+except ImportError:
+    pydantic = None  # type: ignore [assignment]

itemadapter/adapter.py

@@ -12,6 +12,8 @@
     _is_pydantic_model,
 )
 
+from itemadapter._imports import attr, dataclasses
+
 
 __all__ = [
     "AdapterInterface",
@@ -100,8 +102,8 @@ def __len__(self) -> int:
 class AttrsAdapter(_MixinAttrsDataclassAdapter, AdapterInterface):
     def __init__(self, item: Any) -> None:
         super().__init__(item)
-        import attr
-
+        if attr is None:
+            raise RuntimeError("attr module is not available")
         # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals
         self._fields_dict = attr.fields_dict(self.item.__class__)
 
@@ -115,19 +117,19 @@ def is_item_class(cls, item_class: type) -> bool:
 
     @classmethod
     def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType:
-        from attr import fields_dict
-
+        if attr is None:
+            raise RuntimeError("attr module is not available")
         try:
-            return fields_dict(item_class)[field_name].metadata  # type: ignore
+            return attr.fields_dict(item_class)[field_name].metadata  # type: ignore
         except KeyError:
             raise KeyError(f"{item_class.__name__} does not support field: {field_name}")
 
 
 class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface):
     def __init__(self, item: Any) -> None:
         super().__init__(item)
-        import dataclasses
-
+        if dataclasses is None:
+            raise RuntimeError("dataclasses module is not available")
         # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals
         self._fields_dict = {field.name: field for field in dataclasses.fields(self.item)}
 
@@ -141,9 +143,9 @@ def is_item_class(cls, item_class: type) -> bool:
 
     @classmethod
     def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType:
-        from dataclasses import fields
-
-        for field in fields(item_class):
+        if dataclasses is None:
+            raise RuntimeError("dataclasses module is not available")
+        for field in dataclasses.fields(item_class):
             if field.name == field_name:
                 return field.metadata  # type: ignore
         raise KeyError(f"{item_class.__name__} does not support field: {field_name}")

itemadapter/utils.py

@@ -3,55 +3,47 @@
 from types import MappingProxyType
 from typing import Any
 
+from itemadapter._imports import attr, dataclasses, pydantic, scrapy
 
 __all__ = ["is_item", "get_field_meta_from_class"]
 
 
 def _get_scrapy_item_classes() -> tuple:
-    try:
-        import scrapy
-    except ImportError:
+    if scrapy is None:
         return ()
+    try:
+        # handle deprecated base classes
+        _base_item_cls = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem)
+    except AttributeError:
+        return (scrapy.item.Item,)
     else:
-        try:
-            # handle deprecated base classes
-            _base_item_cls = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem)
-        except AttributeError:
-            return (scrapy.item.Item,)
-        else:
-            return (scrapy.item.Item, _base_item_cls)
+        return (scrapy.item.Item, _base_item_cls)
 
 
 def _is_dataclass(obj: Any) -> bool:
     """In py36, this returns False if the "dataclasses" backport module is not installed."""
-    try:
-        import dataclasses
-    except ImportError:
+    if dataclasses is None:
         return False
     return dataclasses.is_dataclass(obj)
 
 
 def _is_attrs_class(obj: Any) -> bool:
-    try:
-        import attr
-    except ImportError:
+    if attr is None:
         return False
     return attr.has(obj)
 
 
 def _is_pydantic_model(obj: Any) -> bool:
-    try:
-        from pydantic import BaseModel
-    except ImportError:
+    if pydantic is None:
         return False
-    return issubclass(obj, BaseModel)
+    return issubclass(obj, pydantic.BaseModel)
 
 
 def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingProxyType:
     metadata = {}
     field = item_model.__fields__[field_name].field_info
 
-    for attr in [
+    for attribute in [
         "alias",
         "title",
         "description",
@@ -67,9 +59,9 @@ def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingPro
         "max_length",
         "regex",
     ]:
-        value = getattr(field, attr)
+        value = getattr(field, attribute)
         if value is not None:
-            metadata[attr] = value
+            metadata[attribute] = value
     if not field.allow_mutation:
         metadata["allow_mutation"] = field.allow_mutation
     metadata.update(field.extra)

tests/__init__.py

@@ -1,14 +1,31 @@
 import importlib
-from typing import Optional
+import sys
+from contextlib import contextmanager
+from typing import Callable, Optional
 
 from itemadapter import ItemAdapter
 
 
-def mocked_import(name, *args, **kwargs):
-    """Allow only internal itemadapter imports."""
-    if name.split(".")[0] == "itemadapter":
+def make_mock_import(block_name: str) -> Callable:
+    def mock_import(name: str, *args, **kwargs):
+        """Prevent importing a specific module, let everything else pass."""
+        if name.split(".")[0] == block_name:
+            raise ImportError(name)
         return importlib.__import__(name, *args, **kwargs)
-    raise ImportError(name)
+
+    return mock_import
+
+
+@contextmanager
+def clear_itemadapter_imports() -> None:
+    backup = {}
+    for key in sys.modules.copy().keys():
+        if key.startswith("itemadapter"):
+            backup[key] = sys.modules.pop(key)
+    try:
+        yield
+    finally:
+        sys.modules.update(backup)
 
 
 try:

tests/test_adapter_attrs.py

@@ -3,7 +3,6 @@
 from types import MappingProxyType
 from unittest import mock
 
-from itemadapter.adapter import AttrsAdapter
 from itemadapter.utils import get_field_meta_from_class
 
 from tests import (
@@ -12,12 +11,15 @@
     PydanticModel,
     ScrapyItem,
     ScrapySubclassedItem,
-    mocked_import,
+    make_mock_import,
+    clear_itemadapter_imports,
 )
 
 
 class AttrsTestCase(unittest.TestCase):
     def test_false(self):
+        from itemadapter.adapter import AttrsAdapter
+
         self.assertFalse(AttrsAdapter.is_item(int))
         self.assertFalse(AttrsAdapter.is_item(sum))
         self.assertFalse(AttrsAdapter.is_item(1234))
@@ -35,14 +37,32 @@ def test_false(self):
         self.assertFalse(AttrsAdapter.is_item(AttrsItem))
 
     @unittest.skipIf(not AttrsItem, "attrs module is not available")
-    @mock.patch("builtins.__import__", mocked_import)
+    @mock.patch("builtins.__import__", make_mock_import("attr"))
+    def test_module_import_error(self):
+        with clear_itemadapter_imports():
+            from itemadapter.adapter import AttrsAdapter
+
+            self.assertFalse(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234)))
+            with self.assertRaises(RuntimeError, msg="attr module is not available"):
+                AttrsAdapter(AttrsItem(name="asdf", value=1234))
+            with self.assertRaises(RuntimeError, msg="attr module is not available"):
+                AttrsAdapter.get_field_meta_from_class(AttrsItem, "name")
+            with self.assertRaises(TypeError, msg="AttrsItem is not a valid item class"):
+                get_field_meta_from_class(AttrsItem, "name")
+
+    @unittest.skipIf(not AttrsItem, "attrs module is not available")
+    @mock.patch("itemadapter.utils.attr", None)
     def test_module_not_available(self):
+        from itemadapter.adapter import AttrsAdapter
+
         self.assertFalse(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234)))
         with self.assertRaises(TypeError, msg="AttrsItem is not a valid item class"):
             get_field_meta_from_class(AttrsItem, "name")
 
     @unittest.skipIf(not AttrsItem, "attrs module is not available")
     def test_true(self):
+        from itemadapter.adapter import AttrsAdapter
+
         self.assertTrue(AttrsAdapter.is_item(AttrsItem()))
         self.assertTrue(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234)))
         # field metadata

tests/test_adapter_dataclasses.py

@@ -3,7 +3,6 @@
 from types import MappingProxyType
 from unittest import mock
 
-from itemadapter.adapter import DataclassAdapter
 from itemadapter.utils import get_field_meta_from_class
 
 from tests import (
@@ -12,12 +11,15 @@
     PydanticModel,
     ScrapyItem,
     ScrapySubclassedItem,
-    mocked_import,
+    make_mock_import,
+    clear_itemadapter_imports,
 )
 
 
 class DataclassTestCase(unittest.TestCase):
     def test_false(self):
+        from itemadapter.adapter import DataclassAdapter
+
         self.assertFalse(DataclassAdapter.is_item(int))
         self.assertFalse(DataclassAdapter.is_item(sum))
         self.assertFalse(DataclassAdapter.is_item(1234))
@@ -35,14 +37,32 @@ def test_false(self):
         self.assertFalse(DataclassAdapter.is_item(DataClassItem))
 
     @unittest.skipIf(not DataClassItem, "dataclasses module is not available")
-    @mock.patch("builtins.__import__", mocked_import)
+    @mock.patch("builtins.__import__", make_mock_import("dataclasses"))
+    def test_module_import_error(self):
+        with clear_itemadapter_imports():
+            from itemadapter.adapter import DataclassAdapter
+
+            self.assertFalse(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234)))
+            with self.assertRaises(RuntimeError, msg="attr module is not available"):
+                DataclassAdapter(DataClassItem(name="asdf", value=1234))
+            with self.assertRaises(RuntimeError, msg="attr module is not available"):
+                DataclassAdapter.get_field_meta_from_class(DataClassItem, "name")
+            with self.assertRaises(TypeError, msg="DataClassItem is not a valid item class"):
+                get_field_meta_from_class(DataClassItem, "name")
+
+    @unittest.skipIf(not DataClassItem, "dataclasses module is not available")
+    @mock.patch("itemadapter.utils.dataclasses", None)
     def test_module_not_available(self):
+        from itemadapter.adapter import DataclassAdapter
+
         self.assertFalse(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234)))
         with self.assertRaises(TypeError, msg="DataClassItem is not a valid item class"):
             get_field_meta_from_class(DataClassItem, "name")
 
     @unittest.skipIf(not DataClassItem, "dataclasses module is not available")
     def test_true(self):
+        from itemadapter.adapter import DataclassAdapter
+
         self.assertTrue(DataclassAdapter.is_item(DataClassItem()))
         self.assertTrue(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234)))
         # field metadata