From 18994092194ee13c471c933f6778e5868bbeafd8 Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Fri, 26 Jan 2024 17:21:57 +0800 Subject: [PATCH] fix from_crawler() not working for EcommerceSpider --- tests/test_ecommerce.py | 31 ++++++++++++++++++++++ zyte_spider_templates/spiders/ecommerce.py | 8 +++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/test_ecommerce.py b/tests/test_ecommerce.py index 24c4382..5343727 100644 --- a/tests/test_ecommerce.py +++ b/tests/test_ecommerce.py @@ -303,6 +303,37 @@ def test_arguments(): "getdict", {"geolocation": "DE"}, ), + ( + "extract_from", + "browserHtml", + "ZYTE_API_PROVIDER_PARAMS", + None, + "getdict", + { + "productOptions": {"extractFrom": "browserHtml"}, + "productNavigationOptions": {"extractFrom": "browserHtml"}, + }, + ), + ( + "extract_from", + "httpResponseBody", + "ZYTE_API_PROVIDER_PARAMS", + {"geolocation": "US"}, + "getdict", + { + "productOptions": {"extractFrom": "httpResponseBody"}, + "productNavigationOptions": {"extractFrom": "httpResponseBody"}, + "geolocation": "US", + }, + ), + ( + "extract_from", + None, + "ZYTE_API_PROVIDER_PARAMS", + {"geolocation": "US"}, + "getdict", + {"geolocation": "US"}, + ), ): kwargs = {param: arg} settings = {} diff --git a/zyte_spider_templates/spiders/ecommerce.py b/zyte_spider_templates/spiders/ecommerce.py index 694f317..68de7f3 100644 --- a/zyte_spider_templates/spiders/ecommerce.py +++ b/zyte_spider_templates/spiders/ecommerce.py @@ -10,7 +10,11 @@ from zyte_common_items import ProbabilityRequest, Product, ProductNavigation from zyte_spider_templates.documentation import document_enum -from zyte_spider_templates.spiders.base import BaseSpider, BaseSpiderParams +from zyte_spider_templates.spiders.base import ( + ARG_SETTING_PRIORITY, + BaseSpider, + BaseSpiderParams, +) @document_enum @@ -129,7 +133,9 @@ def from_crawler(cls, crawler: Crawler, *args, **kwargs) -> scrapy.Spider: "productNavigationOptions": { "extractFrom": spider.args.extract_from }, + **spider.settings.get("ZYTE_API_PROVIDER_PARAMS", {}), }, + priority=ARG_SETTING_PRIORITY, ) return spider