set a sleep max

Summary: Currently, we have 100 retries, and using a 2x sleep time would result in an endless wait for a single error case. Instead, this diff sets a maximum sleep time of 10 minutes for retries because most delays are due to rate limits, which typically resolve within the next hour.

Reviewed By: mbhatt1

Differential Revision: D58239477

fbshipit-source-id: 5cfe6a06b5a0ab098a7777ee7aa0db071be59394

CybersecurityBenchmarks/benchmark/llm.py

@@ -97,7 +97,11 @@ def _query_with_retries(
                 return func(*args)
             except Exception as exception:
                 last_exception = exception
-                sleep_time = backoff_factor * (2**retry)
+                sleep_time = (
+                    backoff_factor * (2**retry)
+                    if backoff_factor * (2**retry) <= 600
+                    else 600
+                )
                 time.sleep(sleep_time)
                 LOG.debug(
                     f"LLM Query failed with error: {exception}. Sleeping for {sleep_time} seconds..."
@@ -119,7 +123,11 @@ def _chat_with_retries(
                 return func(*args)
             except Exception as exception:
                 last_exception = exception
-                sleep_time = backoff_factor * (2**retry)
+                sleep_time = (
+                    backoff_factor * (2**retry)
+                    if backoff_factor * (2**retry) <= 600
+                    else 600
+                )
                 time.sleep(sleep_time)
                 LOG.debug(
                     f"LLM Query failed with error: {exception}. Sleeping for {sleep_time} seconds..."