Increase timeout

alexandrainst · Mar 5, 2024 · 580a107 · 580a107
1 parent 8f134b2
commit 580a107
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/config/scrape/scrape.yaml b/config/scrape/scrape.yaml
@@ -19,4 +19,5 @@ test_case_id: "1"
 
 # Constants
 sleep: 5
-max_consecutive_nonexistent_page_count: 100
+max_consecutive_nonexistent_page_count: 100
+timeout_pdf_download: 20
diff --git a/src/doms_databasen/scraper.py b/src/doms_databasen/scraper.py
@@ -197,7 +197,7 @@ def _already_scraped(self, case_dir) -> bool:
         """
         return case_dir.exists() and len(os.listdir(case_dir)) == N_FILES_RAW_CASE_DIR
 
-    def _wait_download(self, files_before: set, timeout: int = 10) -> str:
+    def _wait_download(self, files_before: set) -> str:
         """Waits for a file to be downloaded to the download directory.
 
         Args:
@@ -211,7 +211,7 @@ def _wait_download(self, files_before: set, timeout: int = 10) -> str:
                 Name of downloaded file (empty string if timeout)
         """
         time.sleep(1)
-        endtime = time.time() + timeout
+        endtime = time.time() + self.config.scrape.timeout_pdf_download
         while True:
             files_now = set(os.listdir(self.download_dir))
             new_files = files_now - files_before