Use zimscraperlib session to fetch web content with automatic meaningful retries

scraper/src/mindtouch2zim/client.py

@@ -3,14 +3,15 @@
 from pathlib import Path
 from typing import Any
 
-import requests
 from bs4 import BeautifulSoup, NavigableString
 from pydantic import BaseModel
+from requests import Response
 
 from mindtouch2zim.constants import (
     HTTP_TIMEOUT_LONG_SECONDS,
     HTTP_TIMEOUT_NORMAL_SECONDS,
     logger,
+    web_session,
 )
 
 
@@ -121,7 +122,7 @@
         full_url = f"{self.library_url}{url_subpath_and_query}"
         logger.debug(f"Fetching {full_url}")
 
-        resp = requests.get(
+        resp = web_session.get(
             url=full_url,
             allow_redirects=True,
             timeout=HTTP_TIMEOUT_NORMAL_SECONDS,
@@ -131,12 +132,10 @@
         cache_file.write_text(resp.text)
         return resp.text
 
-    def _get_api_resp(
-        self, api_sub_path_and_query: str, timeout: float
-    ) -> requests.Response:
+    def _get_api_resp(self, api_sub_path_and_query: str, timeout: float) -> Response:
         api_url = f"{self.api_url}{api_sub_path_and_query}"
         logger.debug(f"Calling API at {api_url}")
-        resp = requests.get(
+        resp = web_session.get(
             url=api_url,
             headers={"x-deki-token": self.deki_token},
             timeout=timeout,

scraper/src/mindtouch2zim/constants.py

@@ -1,6 +1,7 @@
 import logging
 import pathlib
 
+from zimscraperlib.download import get_session
 from zimscraperlib.logging import (
     getLogger,
 )
@@ -18,3 +19,5 @@
 HTTP_TIMEOUT_LONG_SECONDS = 30
 
 logger = getLogger(NAME, level=logging.DEBUG)
+
+web_session = get_session()

scraper/src/mindtouch2zim/processor.py

@@ -35,7 +35,13 @@
     MindtouchClient,
     MindtouchHome,
 )
-from mindtouch2zim.constants import LANGUAGE_ISO_639_3, NAME, VERSION, logger
+from mindtouch2zim.constants import (
+    LANGUAGE_ISO_639_3,
+    NAME,
+    VERSION,
+    logger,
+    web_session,
+)
 from mindtouch2zim.ui import (
     ConfigModel,
     PageContentModel,
@@ -345,7 +351,9 @@
                 )
 
             welcome_image = BytesIO()
-            stream_file(home.welcome_image_url, byte_stream=welcome_image)
+            stream_file(
+                home.welcome_image_url, byte_stream=welcome_image, session=web_session
+            )
             add_item_for(creator, "content/logo.png", content=welcome_image.getvalue())
             del welcome_image
 
@@ -437,7 +445,11 @@
                 for asset_url in asset_urls:
                     try:
                         asset_content = BytesIO()
-                        stream_file(asset_url.value, byte_stream=asset_content)
+                        stream_file(
+                            asset_url.value,
+                            byte_stream=asset_content,
+                            session=web_session,
+                        )
                         logger.debug(
                             f"Adding {asset_url.value} to {asset_path.value} in the ZIM"
                         )
@@ -474,7 +486,7 @@
             raise ValueError(f"Cannot process empty css_location for {target_filename}")
         if not css_content:
             css_buffer = BytesIO()
-            stream_file(css_location, byte_stream=css_buffer)
+            stream_file(css_location, byte_stream=css_buffer, session=web_session)
             css_content = css_buffer.getvalue()
         url_rewriter = CssUrlsRewriter(
             article_url=HttpUrl(css_location),
@@ -548,7 +560,9 @@
             try:
                 logger.debug(f"Downloading {icon_url} illustration")
                 illustration_content = BytesIO()
-                stream_file(icon_url, byte_stream=illustration_content)
+                stream_file(
+                    icon_url, byte_stream=illustration_content, session=web_session
+                )
                 illustration_format = format_for(
                     illustration_content, from_suffix=False
                 )

scraper/src/mindtouch2zim/vimeo.py

@@ -1,8 +1,7 @@
-import requests
-
 from mindtouch2zim.constants import (
     HTTP_TIMEOUT_NORMAL_SECONDS,
     logger,
+    web_session,
 )
 
 
@@ -14,7 +13,7 @@
 
 def get_vimeo_thumbnail_url(video_url: str) -> str:
     """From a vimeo URL - player or normal - retrieve corresponding thumbnail URL"""
-    resp = requests.get(
+    resp = web_session.get(
         f"https://vimeo.com/api/oembed.json?url={video_url}",
         timeout=HTTP_TIMEOUT_NORMAL_SECONDS,
     )