Merge pull request #29 from zytedata/fix-from-crawler

fix `extract_from` spider parameter not working
zytedata · Jan 26, 2024 · 570edab · 570edab
2 parents 0cb9d92 + 1899409
commit 570edab
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 1 deletion.
diff --git a/tests/test_ecommerce.py b/tests/test_ecommerce.py
@@ -303,6 +303,37 @@ def test_arguments():
             "getdict",
             {"geolocation": "DE"},
         ),
+        (
+            "extract_from",
+            "browserHtml",
+            "ZYTE_API_PROVIDER_PARAMS",
+            None,
+            "getdict",
+            {
+                "productOptions": {"extractFrom": "browserHtml"},
+                "productNavigationOptions": {"extractFrom": "browserHtml"},
+            },
+        ),
+        (
+            "extract_from",
+            "httpResponseBody",
+            "ZYTE_API_PROVIDER_PARAMS",
+            {"geolocation": "US"},
+            "getdict",
+            {
+                "productOptions": {"extractFrom": "httpResponseBody"},
+                "productNavigationOptions": {"extractFrom": "httpResponseBody"},
+                "geolocation": "US",
+            },
+        ),
+        (
+            "extract_from",
+            None,
+            "ZYTE_API_PROVIDER_PARAMS",
+            {"geolocation": "US"},
+            "getdict",
+            {"geolocation": "US"},
+        ),
     ):
         kwargs = {param: arg}
         settings = {}

diff --git a/zyte_spider_templates/spiders/ecommerce.py b/zyte_spider_templates/spiders/ecommerce.py
@@ -10,7 +10,11 @@
 from zyte_common_items import ProbabilityRequest, Product, ProductNavigation
 
 from zyte_spider_templates.documentation import document_enum
-from zyte_spider_templates.spiders.base import BaseSpider, BaseSpiderParams
+from zyte_spider_templates.spiders.base import (
+    ARG_SETTING_PRIORITY,
+    BaseSpider,
+    BaseSpiderParams,
+)
 
 
 @document_enum
@@ -129,7 +133,9 @@ def from_crawler(cls, crawler: Crawler, *args, **kwargs) -> scrapy.Spider:
                     "productNavigationOptions": {
                         "extractFrom": spider.args.extract_from
                     },
+                    **spider.settings.get("ZYTE_API_PROVIDER_PARAMS", {}),
                 },
+                priority=ARG_SETTING_PRIORITY,
             )
 
         return spider