From e6afa54cd10a391c1aca996c23c4d6b2c9da4b7e Mon Sep 17 00:00:00 2001
From: extreme4all <>
Date: Tue, 6 Feb 2024 22:22:27 +0100
Subject: [PATCH] fine tune batch insert

---
 src/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main.py b/src/main.py
index 4e73240..5db691b 100644
--- a/src/main.py
+++ b/src/main.py
@@ -137,6 +137,7 @@ async def process_data(receive_queue: Queue, error_queue: Queue):
             start_time=start_time,
             _queue=receive_queue,
             topic="scraper",
+            interval=15
         )
 
         # Check if queue is empty
@@ -161,7 +162,7 @@ async def process_data(receive_queue: Queue, error_queue: Queue):
 
         now = time.time()
 
-        if len(batch) > 100 or now-start_time > 5:
+        if len(batch) > 100 or now-start_time > 15:
             await insert_data(batch=batch, error_queue=error_queue)
             batch = []