diff --git a/src/main.py b/src/main.py index 3d54194..485e8eb 100644 --- a/src/main.py +++ b/src/main.py @@ -102,14 +102,27 @@ async def update_player(session: AsyncSession, data: dict): def log_speed( - counter: int, start_time: float, receive_queue: Queue + counter: int, start_time: float, _queue: Queue, topic: str, interval: int = 15 ) -> tuple[float, int]: - end_time = time.time() - delta_time = end_time - start_time + # Calculate the time elapsed since the function started + delta_time = time.time() - start_time + + # Check if the specified interval has not elapsed yet + if delta_time < interval: + # Return the original start time and the current counter value + return start_time, counter + + # Calculate the processing speed (messages per second) speed = counter / delta_time - logger.info( - f"qsize={receive_queue.qsize()}, processed {counter} in {delta_time:.2f} seconds, {speed:.2f} msg/sec" + + # Log the processing speed and relevant information + log_message = ( + f"{topic=}, qsize={_queue.qsize()}, " + f"processed {counter} in {delta_time:.2f} seconds, {speed:.2f} msg/sec" ) + logger.info(log_message) + + # Return the current time and reset the counter to zero return time.time(), 0 @@ -121,14 +134,15 @@ async def process_data(receive_queue: Queue, error_queue: Queue): # Run indefinitely while True: - # Check if both queues are empty + start_time, counter = log_speed( + counter=counter, + start_time=start_time, + _queue=receive_queue, + topic="scraper", + ) + + # Check if queue is empty if receive_queue.empty(): - # If there were previous iterations with data, log processing speed - if counter > 0: - start_time, counter = log_speed( - counter=counter, start_time=start_time, receive_queue=receive_queue - ) - # Sleep for 1 second and then continue to the next iteration await asyncio.sleep(1) continue @@ -178,11 +192,6 @@ async def process_data(receive_queue: Queue, error_queue: Queue): receive_queue.task_done() continue - # Log processing speed every 100 iterations - if counter % 100 == 0 and counter > 0: - start_time, counter = log_speed( - counter=counter, start_time=start_time, receive_queue=receive_queue - ) # Increment the counter counter += 1