add fallback middleware

Signed-off-by: oilbeater <[email protected]>

README.md

@@ -1,3 +1,5 @@
+# Malacca
+
 ![Malacca Logo](./docs/malacca.jpg)
 
 **Malacca** is an open-source AI gateway designed to become the central hub in the world of AI.

src/middlewares/fallback.ts

@@ -0,0 +1,34 @@
+import { Context, Next } from 'hono';
+import { AppContext } from '.';
+
+export const fallbackMiddleware = async (c: Context<AppContext>, next: Next) => {
+    try {
+        await next();
+
+        // Check if the response status is in the 5xx range
+        if (c.res && c.res.status >= 500 && c.res.status < 600) {
+            throw new Error(`Upstream returned ${c.res.status} status`);
+        }
+    } catch (error) {
+        try {
+            // Call CF Workers AI as a fallback
+            const fallbackResponse = await c.env.AI.run(
+                "@cf/meta/llama-3.1-8b-instruct",
+                await c.req.json()
+            );
+
+            let response: Response;
+            if (fallbackResponse instanceof ReadableStream) {
+                response = new Response(fallbackResponse);
+            } else {
+                response = new Response(fallbackResponse.response);
+            }
+
+            // Add a header to indicate fallback was used
+            response.headers.set('X-Fallback-Used', 'true');
+            return response;
+        } catch (fallbackError) {
+            return new Response('Both primary and fallback providers failed', { status: 500 });
+        }
+    }
+};

src/middlewares/index.ts

@@ -7,6 +7,7 @@ export { loggingMiddleware } from './logging';
 export { virtualKeyMiddleware } from './virtualKey';
 export { rateLimiterMiddleware } from './rateLimiter';
 export { guardMiddleware } from './guard';
+export { fallbackMiddleware } from './fallback';
 export interface AppContext {
     Bindings: Env,
     Variables: {

src/providers/azureOpenAI.ts

@@ -7,7 +7,8 @@ import {
     loggingMiddleware,
     virtualKeyMiddleware,
     rateLimiterMiddleware,
-    guardMiddleware
+    guardMiddleware,
+    fallbackMiddleware
 } from '../middlewares';
 
 const BasePath = '/azure-openai/:resource_name/deployments/:deployment_name';
@@ -22,7 +23,7 @@ const initMiddleware = async (c: Context, next: Next) => {
 };
 
 
-azureOpenAIRoute.use(initMiddleware, metricsMiddleware, loggingMiddleware, bufferMiddleware, virtualKeyMiddleware, rateLimiterMiddleware, guardMiddleware, cacheMiddleware);
+azureOpenAIRoute.use(initMiddleware, metricsMiddleware, loggingMiddleware, bufferMiddleware, virtualKeyMiddleware, rateLimiterMiddleware, guardMiddleware, cacheMiddleware, fallbackMiddleware);
 
 azureOpenAIRoute.post('/*', async (c: Context) => {
     return azureOpenAIProvider.handleRequest(c);