Add Vercel KV cache

docs/core_docs/docs/how_to/llm_caching.mdx

@@ -161,6 +161,22 @@ import AdvancedUpstashRedisCacheExample from "@examples/cache/upstash_redis_adva
 
 <CodeBlock language="typescript">{AdvancedUpstashRedisCacheExample}</CodeBlock>
 
+## Caching with Vercel KV
+
+LangChain provides an Vercel KV-based cache. Like the Redis-based cache, this cache is useful if you want to share the cache across multiple processes or servers. The Vercel KV client uses HTTP and supports edge environments. To use it, you'll need to install the `@vercel/kv` package:
+
+```bash npm2yarn
+npm install @vercel/kv
+```
+
+You'll also need an Vercel account and a [KV database](https://vercel.com/docs/storage/vercel-kv/kv-reference) to connect to. Once you've done that, retrieve your REST URL and REST token.
+
+Then, you can pass a `cache` option when you instantiate the LLM. For example:
+
+import VercelKVCacheExample from "@examples/cache/vercel_kv.ts";
+
+<CodeBlock language="typescript">{VercelKVCacheExample}</CodeBlock>
+
 ## Caching with Cloudflare KV
 
 :::info

examples/src/cache/vercel_kv.ts

@@ -0,0 +1,14 @@
+import { OpenAI } from "@langchain/openai";
+import { VercelKVCache } from "@langchain/community/caches/vercel_kv";
+import { createClient } from "@vercel/kv";
+
+// See https://vercel.com/docs/storage/vercel-kv/kv-reference#createclient-example for connection options
+const cache = new VercelKVCache({
+  config: createClient({
+    url: "VERCEL_KV_API_URL",
+    token: "VERCEL_KV_API_TOKEN",
+  }),
+  ttl: 3600,
+});
+
+const model = new OpenAI({ cache });

libs/langchain-community/langchain.config.js

@@ -221,6 +221,7 @@ export const config = {
     "caches/ioredis": "caches/ioredis",
     "caches/momento": "caches/momento",
     "caches/upstash_redis": "caches/upstash_redis",
+    "caches/vercel_kv": "caches/vercel_kv",
     // graphs
     "graphs/neo4j_graph": "graphs/neo4j_graph",
     "graphs/memgraph_graph": "graphs/memgraph_graph",
@@ -450,9 +451,12 @@ export const config = {
     "structured_query/supabase",
     "structured_query/vectara",
     "retrievers/zep_cloud",
+    // cache
     "cache/cloudflare_kv",
     "cache/momento",
     "cache/upstash_redis",
+    "cache/vercel_kv",
+    //graphs
     "graphs/neo4j_graph",
     "graphs/memgraph_graph",
     // document_compressors

libs/langchain-community/src/caches/tests/vercel_kv.int.test.ts

@@ -0,0 +1,34 @@
+/* eslint-disable no-process-env */
+import { ChatOpenAI } from "@langchain/openai";
+import { createClient } from "@vercel/kv";
+import { VercelKVCache } from "../vercel_kv.js";
+
+test.skip("VercelKVCache works with ChatOpenAI", async () => {
+  if (
+    !process.env.VERCEL_KV_API_URL ||
+    !process.env.VERCEL_KV_API_TOKEN ||
+    !process.env.OPENAI_API_KEY
+  ) {
+    throw new Error("Missing Vercel KV API URL, token, or OpenAI API key");
+  }
+
+  const vercelKVCache = new VercelKVCache({
+    client: createClient({
+      url: process.env.VERCEL_KV_API_URL,
+      token: process.env.VERCEL_KV_API_TOKEN,
+    }),
+    ttl: 60,
+  });
+
+  const chat = new ChatOpenAI({
+    temperature: 0,
+    cache: vercelKVCache,
+    maxTokens: 10,
+  });
+
+  const prompt = "What color is the sky?";
+  const result1 = await chat.invoke(prompt);
+  const result2 = await chat.invoke(prompt);
+
+  expect(result1).toEqual(result2);
+});

libs/langchain-community/src/caches/vercel_kv.ts

@@ -0,0 +1,87 @@
+import { kv, type VercelKV } from "@vercel/kv";
+
+import { Generation } from "@langchain/core/outputs";
+import {
+  BaseCache,
+  deserializeStoredGeneration,
+  getCacheKey,
+  serializeGeneration,
+} from "@langchain/core/caches";
+import { StoredGeneration } from "@langchain/core/messages";
+
+export type VercelKVCacheProps = {
+  /**
+   * An existing Vercel KV client
+   */
+  client?: VercelKV;
+  /**
+   * Time-to-live (TTL) for cached items in seconds
+   */
+  ttl?: number;
+};
+
+/**
+ * A cache that uses Vercel KV as the backing store.
+ * @example
+ * ```typescript
+ * const cache = new VercelKVCache({
+ *   ttl: 3600, // Optional: Cache entries will expire after 1 hour
+ * });
+ *
+ * // Initialize the OpenAI model with Vercel KV cache for caching responses
+ * const model = new ChatOpenAI({
+ *   cache,
+ * });
+ * await model.invoke("How are you today?");
+ * const cachedValues = await cache.lookup("How are you today?", "llmKey");
+ * ```
+ */
+export class VercelKVCache extends BaseCache {
+  private client: VercelKV;
+
+  private ttl?: number;
+
+  constructor(props: VercelKVCacheProps) {
+    super();
+    const { client, ttl } = props;
+    this.client = client ?? kv;
+    this.ttl = ttl;
+  }
+
+  /**
+   * Lookup LLM generations in cache by prompt and associated LLM key.
+   */
+  public async lookup(prompt: string, llmKey: string) {
+    let idx = 0;
+    let key = getCacheKey(prompt, llmKey, String(idx));
+    let value = await this.client.get<StoredGeneration | null>(key);
+    const generations: Generation[] = [];
+
+    while (value) {
+      generations.push(deserializeStoredGeneration(value));
+      idx += 1;
+      key = getCacheKey(prompt, llmKey, String(idx));
+      value = await this.client.get<StoredGeneration | null>(key);
+    }
+
+    return generations.length > 0 ? generations : null;
+  }
+
+  /**
+   * Update the cache with the given generations.
+   *
+   * Note this overwrites any existing generations for the given prompt and LLM key.
+   */
+  public async update(prompt: string, llmKey: string, value: Generation[]) {
+    for (let i = 0; i < value.length; i += 1) {
+      const key = getCacheKey(prompt, llmKey, String(i));
+      const serializedValue = JSON.stringify(serializeGeneration(value[i]));
+
+      if (this.ttl) {
+        await this.client.set(key, serializedValue, { ex: this.ttl });
+      } else {
+        await this.client.set(key, serializedValue);
+      }
+    }
+  }
+}