remove 'www*.' prefix when parsing allowed_domains

tests/test_ecommerce.py

@@ -21,6 +21,7 @@
 )
 
 from . import get_crawler
+from .test_utils import URL_TO_DOMAIN
 
 
 def test_parameters():
@@ -618,3 +619,12 @@ def test_get_parse_navigation_request():
             "page_type": "productNavigation",
         },
     }
+
+
+@pytest.mark.parametrize("url,allowed_domain", URL_TO_DOMAIN)
+def test_set_allowed_domains(url, allowed_domain):
+    crawler = get_crawler()
+
+    kwargs = {"url": url}
+    spider = EcommerceSpider.from_crawler(crawler, **kwargs)
+    assert spider.allowed_domains == [allowed_domain]

tests/test_utils.py

@@ -0,0 +1,20 @@
+import pytest
+
+from zyte_spider_templates.utils import get_domain
+
+URL_TO_DOMAIN = (
+    ("https://example.com", "example.com"),
+    ("https://www.example.com", "example.com"),
+    ("https://www2.example.com", "example.com"),
+    ("https://prefixwww.example.com", "prefixwww.example.com"),
+    ("https://wwworld.example.com", "wwworld.example.com"),
+    ("https://my.wwworld-example.com", "my.wwworld-example.com"),
+    ("https://wwwow.com", "wwwow.com"),
+    ("https://wowww.com", "wowww.com"),
+    ("https://awww.com", "awww.com"),
+)
+
+
+@pytest.mark.parametrize("url,domain", URL_TO_DOMAIN)
+def test_get_domain(url, domain):
+    assert get_domain(url) == domain

zyte_spider_templates/spiders/base.py

@@ -4,7 +4,6 @@
 import scrapy
 from pydantic import BaseModel, Field
 from scrapy.crawler import Crawler
-from scrapy.utils.url import parse_url
 
 from zyte_spider_templates._geolocations import (
     GEOLOCATION_OPTIONS_WITH_CODE,
@@ -67,7 +66,6 @@ class BaseSpider(scrapy.Spider):
     @classmethod
     def from_crawler(cls, crawler: Crawler, *args, **kwargs) -> scrapy.Spider:
         spider = super().from_crawler(crawler, *args, **kwargs)
-        spider.allowed_domains = [parse_url(spider.args.url).netloc]
 
         if spider.args.geolocation:
             # We set the geolocation in ZYTE_API_PROVIDER_PARAMS for injected

zyte_spider_templates/spiders/ecommerce.py

@@ -15,6 +15,7 @@
     BaseSpider,
     BaseSpiderParams,
 )
+from zyte_spider_templates.utils import get_domain
 
 
 @document_enum
@@ -124,6 +125,7 @@ class EcommerceSpider(Args[EcommerceSpiderParams], BaseSpider):
     @classmethod
     def from_crawler(cls, crawler: Crawler, *args, **kwargs) -> scrapy.Spider:
         spider = super(EcommerceSpider, cls).from_crawler(crawler, *args, **kwargs)
+        spider.allowed_domains = [get_domain(spider.args.url)]
 
         if spider.args.extract_from is not None:
             spider.settings.set(

zyte_spider_templates/utils.py

@@ -0,0 +1,7 @@
+import re
+
+from scrapy.utils.url import parse_url
+
+
+def get_domain(url: str) -> str:
+    return re.sub(r"^www\d*\.", "", parse_url(url).netloc)