Merge pull request #1363 from Oneirocom/429

retry and backoff when getting a 429 from openai text/chat completions

package.json

@@ -109,6 +109,7 @@
     "@use-gesture/react": "10.2.27",
     "@welldone-software/why-did-you-render": "7.0.1",
     "axios": "1.4.0",
+    "axios-retry": "^3.8.0",
     "bullmq": "4.6.0",
     "class-variance-authority": "^0.7.0",
     "classnames": "2.3.2",

packages/config/src/config.ts

@@ -122,6 +122,10 @@ export const AGENT_RESPONSE_TIMEOUT_MSEC =
 
 export const CLOUD_AGENT_KEY = getVarForEnvironment('CLOUD_AGENT_KEY') || v4()
 
+export const BACKOFF_RETRY_LIMIT = Number(
+  getVarForEnvironment('BACKOFF_RETRY_LIMIT') || 0
+)
+
 export const AWS_ACCESS_KEY = getVarForEnvironment('AWS_ACCESS_KEY') || ''
 export const AWS_SECRET_KEY = getVarForEnvironment('AWS_SECRET_KEY') || ''
 export const AWS_REGION = getVarForEnvironment('AWS_REGION') || ''

packages/plugins/openai/server/src/functions/makeChatCompletion.ts

@@ -6,9 +6,14 @@ import {
 } from '@magickml/core'
 import axios from 'axios'
 import { OPENAI_ENDPOINT } from '../constants'
-import { DEFAULT_OPENAI_KEY, PRODUCTION } from '@magickml/config'
+import {
+  DEFAULT_OPENAI_KEY,
+  PRODUCTION,
+  BACKOFF_RETRY_LIMIT,
+} from '@magickml/config'
 import { GPT4_MODELS } from '@magickml/plugin-openai-shared'
 import { trackOpenAIUsage } from '@magickml/server-core'
+import axiosRetry from 'axios-retry'
 
 /**
  * Generate a completion text based on prior chat conversation input.
@@ -109,6 +114,16 @@ export async function makeChatCompletion(
   }
 
   try {
+    // Exponential back-off retry delay between requests
+    axiosRetry(axios, {
+      retries: BACKOFF_RETRY_LIMIT,
+      retryDelay: axiosRetry.exponentialDelay,
+      shouldResetTimeout: true,
+      retryCondition: error => {
+        return error?.response?.status === 429
+      },
+    })
+
     const start = Date.now()
     // Make the API call to OpenAI
     const completion = await axios.post(

packages/plugins/openai/server/src/functions/makeTextCompletion.ts

@@ -2,9 +2,14 @@
 import { CompletionHandlerInputData, saveRequest } from '@magickml/core'
 import axios from 'axios'
 import { OPENAI_ENDPOINT } from '../constants'
-import { DEFAULT_OPENAI_KEY, PRODUCTION } from '@magickml/config'
+import {
+  DEFAULT_OPENAI_KEY,
+  PRODUCTION,
+  BACKOFF_RETRY_LIMIT,
+} from '@magickml/config'
 import { GPT4_MODELS } from '@magickml/plugin-openai-shared'
 import { trackOpenAIUsage } from '@magickml/server-core'
+import axiosRetry from 'axios-retry'
 
 /**
  * Makes an API request to an AI text completion service.
@@ -72,6 +77,16 @@ export async function makeTextCompletion(
 
   // Make the API request and handle the response.
   try {
+    // Exponential back-off retry delay between requests
+    axiosRetry(axios, {
+      retries: BACKOFF_RETRY_LIMIT,
+      retryDelay: axiosRetry.exponentialDelay,
+      shouldResetTimeout: true,
+      retryCondition: error => {
+        return error?.response?.status === 429
+      },
+    })
+
     const start = Date.now()
     const resp = await axios.post(`${OPENAI_ENDPOINT}/completions`, settings, {
       headers: headers,