diff --git a/package-lock.json b/package-lock.json index fd80b3ebca..e4a5e92ae7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -78,6 +78,7 @@ "@use-gesture/react": "10.2.27", "@welldone-software/why-did-you-render": "7.0.1", "axios": "1.4.0", + "axios-retry": "^3.8.0", "bullmq": "4.6.0", "class-variance-authority": "^0.7.0", "classnames": "2.3.2", @@ -22472,6 +22473,15 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/axios-retry": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.8.0.tgz", + "integrity": "sha512-CfIsQyWNc5/AE7x/UEReRUadiBmQeoBpSEC+4QyGLJMswTsP1tz0GW2YYPnE7w9+ESMef5zOgLDFpHynNyEZ1w==", + "dependencies": { + "@babel/runtime": "^7.15.4", + "is-retry-allowed": "^2.2.0" + } + }, "node_modules/axobject-query": { "version": "3.2.1", "dev": true, @@ -32750,6 +32760,17 @@ "node": ">=0.10.0" } }, + "node_modules/is-retry-allowed": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", + "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-root": { "version": "2.1.0", "license": "MIT", diff --git a/package.json b/package.json index ae406bf7b7..4ca6e58e1a 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "@use-gesture/react": "10.2.27", "@welldone-software/why-did-you-render": "7.0.1", "axios": "1.4.0", + "axios-retry": "^3.8.0", "bullmq": "4.6.0", "class-variance-authority": "^0.7.0", "classnames": "2.3.2", diff --git a/packages/config/src/config.ts b/packages/config/src/config.ts index 8f78a2ddda..caeb909150 100644 --- a/packages/config/src/config.ts +++ b/packages/config/src/config.ts @@ -122,6 +122,10 @@ export const AGENT_RESPONSE_TIMEOUT_MSEC = export const CLOUD_AGENT_KEY = getVarForEnvironment('CLOUD_AGENT_KEY') || v4() +export const BACKOFF_RETRY_LIMIT = Number( + getVarForEnvironment('BACKOFF_RETRY_LIMIT') || 0 +) + export const AWS_ACCESS_KEY = getVarForEnvironment('AWS_ACCESS_KEY') || '' export const AWS_SECRET_KEY = getVarForEnvironment('AWS_SECRET_KEY') || '' export const AWS_REGION = getVarForEnvironment('AWS_REGION') || '' diff --git a/packages/plugins/openai/server/src/functions/makeChatCompletion.ts b/packages/plugins/openai/server/src/functions/makeChatCompletion.ts index 50084ec650..46dc8e0c6d 100644 --- a/packages/plugins/openai/server/src/functions/makeChatCompletion.ts +++ b/packages/plugins/openai/server/src/functions/makeChatCompletion.ts @@ -6,9 +6,14 @@ import { } from '@magickml/core' import axios from 'axios' import { OPENAI_ENDPOINT } from '../constants' -import { DEFAULT_OPENAI_KEY, PRODUCTION } from '@magickml/config' +import { + DEFAULT_OPENAI_KEY, + PRODUCTION, + BACKOFF_RETRY_LIMIT, +} from '@magickml/config' import { GPT4_MODELS } from '@magickml/plugin-openai-shared' import { trackOpenAIUsage } from '@magickml/server-core' +import axiosRetry from 'axios-retry' /** * Generate a completion text based on prior chat conversation input. @@ -109,6 +114,16 @@ export async function makeChatCompletion( } try { + // Exponential back-off retry delay between requests + axiosRetry(axios, { + retries: BACKOFF_RETRY_LIMIT, + retryDelay: axiosRetry.exponentialDelay, + shouldResetTimeout: true, + retryCondition: error => { + return error?.response?.status === 429 + }, + }) + const start = Date.now() // Make the API call to OpenAI const completion = await axios.post( diff --git a/packages/plugins/openai/server/src/functions/makeTextCompletion.ts b/packages/plugins/openai/server/src/functions/makeTextCompletion.ts index 922e33e7dd..83f9577a1e 100644 --- a/packages/plugins/openai/server/src/functions/makeTextCompletion.ts +++ b/packages/plugins/openai/server/src/functions/makeTextCompletion.ts @@ -2,9 +2,14 @@ import { CompletionHandlerInputData, saveRequest } from '@magickml/core' import axios from 'axios' import { OPENAI_ENDPOINT } from '../constants' -import { DEFAULT_OPENAI_KEY, PRODUCTION } from '@magickml/config' +import { + DEFAULT_OPENAI_KEY, + PRODUCTION, + BACKOFF_RETRY_LIMIT, +} from '@magickml/config' import { GPT4_MODELS } from '@magickml/plugin-openai-shared' import { trackOpenAIUsage } from '@magickml/server-core' +import axiosRetry from 'axios-retry' /** * Makes an API request to an AI text completion service. @@ -72,6 +77,16 @@ export async function makeTextCompletion( // Make the API request and handle the response. try { + // Exponential back-off retry delay between requests + axiosRetry(axios, { + retries: BACKOFF_RETRY_LIMIT, + retryDelay: axiosRetry.exponentialDelay, + shouldResetTimeout: true, + retryCondition: error => { + return error?.response?.status === 429 + }, + }) + const start = Date.now() const resp = await axios.post(`${OPENAI_ENDPOINT}/completions`, settings, { headers: headers,