Add retries for both scrapers

Javex · Sep 27, 2023 · 0485ab9 · 0485ab9
1 parent a844cb9
commit 0485ab9
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 4 deletions.
diff --git a/hotprices_au/coles.py b/hotprices_au/coles.py
@@ -4,7 +4,7 @@
 from datetime import datetime
 from bs4 import BeautifulSoup
 
-from . import output
+from . import output, request
 
 
 class ColesScraper:
@@ -13,7 +13,7 @@ def __init__(self, store_id, quick=False):
         self.quick = quick
         self.store_id = store_id
 
-        self.session = requests.Session()
+        self.session = request.get_base_session()
         self.session.headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
             'Origin': 'https://www.coles.com.au',

diff --git a/hotprices_au/request.py b/hotprices_au/request.py
@@ -0,0 +1,16 @@
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+
+def get_base_session():
+    session = requests.Session()
+    retry = Retry(
+        total=10,
+        backoff_factor=1,
+        backoff_max=60,
+        status_forcelist=[403],
+        allowed_methods=['GET', 'POST'],
+    )
+    session.mount('https://', HTTPAdapter(max_retries=retry))
+    return session
diff --git a/hotprices_au/woolies.py b/hotprices_au/woolies.py
@@ -1,7 +1,7 @@
 import requests
 import json
 
-from . import output
+from . import output, request
 
 
 class WooliesAPI:
@@ -10,7 +10,7 @@ def __init__(self, quick=False):
         self.quick = quick
 
 
-        self.session = requests.Session()
+        self.session = request.get_base_session()
         self.session.headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
             'Origin': 'https://www.woolworths.com.au',