feat(community): Add VercelKVcache (#7447)

Co-authored-by: jacoblee93 <[email protected]>

Author: Zamoca42

docs/core_docs/docs/how_to/chat_model_caching.mdx

@@ -98,6 +98,64 @@ import RedisCacheExample from "@examples/cache/chat_models/redis.ts";
 
 <CodeBlock language="typescript">{RedisCacheExample}</CodeBlock>
 
+## Caching with Upstash Redis
+
+LangChain provides an Upstash Redis-based cache. Like the Redis-based cache, this cache is useful if you want to share the cache across multiple processes or servers. The Upstash Redis client uses HTTP and supports edge environments. To use it, you'll need to install the `@upstash/redis` package:
+
+```bash npm2yarn
+npm install @upstash/redis
+```
+
+You'll also need an [Upstash account](https://docs.upstash.com/redis#create-account) and a [Redis database](https://docs.upstash.com/redis#create-a-database) to connect to. Once you've done that, retrieve your REST URL and REST token.
+
+Then, you can pass a `cache` option when you instantiate the LLM. For example:
+
+import UpstashRedisCacheExample from "@examples/cache/chat_models/upstash_redis.ts";
+
+<CodeBlock language="typescript">{UpstashRedisCacheExample}</CodeBlock>
+
+You can also directly pass in a previously created [@upstash/redis](https://docs.upstash.com/redis/sdks/javascriptsdk/overview) client instance:
+
+import AdvancedUpstashRedisCacheExample from "@examples/cache/chat_models/upstash_redis_advanced.ts";
+
+<CodeBlock language="typescript">{AdvancedUpstashRedisCacheExample}</CodeBlock>
+
+## Caching with Vercel KV
+
+LangChain provides an Vercel KV-based cache. Like the Redis-based cache, this cache is useful if you want to share the cache across multiple processes or servers. The Vercel KV client uses HTTP and supports edge environments. To use it, you'll need to install the `@vercel/kv` package:
+
+```bash npm2yarn
+npm install @vercel/kv
+```
+
+You'll also need an Vercel account and a [KV database](https://vercel.com/docs/storage/vercel-kv/kv-reference) to connect to. Once you've done that, retrieve your REST URL and REST token.
+
+Then, you can pass a `cache` option when you instantiate the LLM. For example:
+
+import VercelKVCacheExample from "@examples/cache/chat_models/vercel_kv.ts";
+
+<CodeBlock language="typescript">{VercelKVCacheExample}</CodeBlock>
+
+## Caching with Cloudflare KV
+
+:::info
+This integration is only supported in Cloudflare Workers.
+:::
+
+If you're deploying your project as a Cloudflare Worker, you can use LangChain's Cloudflare KV-powered LLM cache.
+
+For information on how to set up KV in Cloudflare, see [the official documentation](https://developers.cloudflare.com/kv/).
+
+**Note:** If you are using TypeScript, you may need to install types if they aren't already present:
+
+```bash npm2yarn
+npm install -S @cloudflare/workers-types
+```
+
+import CloudflareExample from "@examples/cache/chat_models/cloudflare_kv.ts";
+
+<CodeBlock language="typescript">{CloudflareExample}</CodeBlock>
+
 ## Caching on the File System
 
 :::warning

docs/core_docs/docs/how_to/llm_caching.mdx

@@ -161,6 +161,22 @@ import AdvancedUpstashRedisCacheExample from "@examples/cache/upstash_redis_adva
 
 <CodeBlock language="typescript">{AdvancedUpstashRedisCacheExample}</CodeBlock>
 
+## Caching with Vercel KV
+
+LangChain provides an Vercel KV-based cache. Like the Redis-based cache, this cache is useful if you want to share the cache across multiple processes or servers. The Vercel KV client uses HTTP and supports edge environments. To use it, you'll need to install the `@vercel/kv` package:
+
+```bash npm2yarn
+npm install @vercel/kv
+```
+
+You'll also need an Vercel account and a [KV database](https://vercel.com/docs/storage/vercel-kv/kv-reference) to connect to. Once you've done that, retrieve your REST URL and REST token.
+
+Then, you can pass a `cache` option when you instantiate the LLM. For example:
+
+import VercelKVCacheExample from "@examples/cache/vercel_kv.ts";
+
+<CodeBlock language="typescript">{VercelKVCacheExample}</CodeBlock>
+
 ## Caching with Cloudflare KV
 
 :::info

examples/src/cache/chat_models/vercel_kv.ts

@@ -0,0 +1,17 @@
+import { ChatOpenAI } from "@langchain/openai";
+import { VercelKVCache } from "@langchain/community/caches/vercel_kv";
+import { createClient } from "@vercel/kv";
+
+// See https://vercel.com/docs/storage/vercel-kv/kv-reference#createclient-example for connection options
+const cache = new VercelKVCache({
+  client: createClient({
+    url: "VERCEL_KV_API_URL",
+    token: "VERCEL_KV_API_TOKEN",
+  }),
+  ttl: 3600,
+});
+
+const model = new ChatOpenAI({
+  model: "gpt-4o-mini",
+  cache,
+});

examples/src/cache/vercel_kv.ts

@@ -0,0 +1,14 @@
+import { OpenAI } from "@langchain/openai";
+import { VercelKVCache } from "@langchain/community/caches/vercel_kv";
+import { createClient } from "@vercel/kv";
+
+// See https://vercel.com/docs/storage/vercel-kv/kv-reference#createclient-example for connection options
+const cache = new VercelKVCache({
+  client: createClient({
+    url: "VERCEL_KV_API_URL",
+    token: "VERCEL_KV_API_TOKEN",
+  }),
+  ttl: 3600,
+});
+
+const model = new OpenAI({ cache });

libs/langchain-community/.gitignore

@@ -714,6 +714,10 @@ caches/upstash_redis.cjs
 caches/upstash_redis.js
 caches/upstash_redis.d.ts
 caches/upstash_redis.d.cts
+caches/vercel_kv.cjs
+caches/vercel_kv.js
+caches/vercel_kv.d.ts
+caches/vercel_kv.d.cts
 graphs/neo4j_graph.cjs
 graphs/neo4j_graph.js
 graphs/neo4j_graph.d.ts

libs/langchain-community/langchain.config.js

@@ -222,6 +222,7 @@ export const config = {
     "caches/ioredis": "caches/ioredis",
     "caches/momento": "caches/momento",
     "caches/upstash_redis": "caches/upstash_redis",
+    "caches/vercel_kv": "caches/vercel_kv",
     // graphs
     "graphs/neo4j_graph": "graphs/neo4j_graph",
     "graphs/memgraph_graph": "graphs/memgraph_graph",
@@ -453,9 +454,12 @@ export const config = {
     "structured_query/supabase",
     "structured_query/vectara",
     "retrievers/zep_cloud",
+    // cache
     "cache/cloudflare_kv",
     "cache/momento",
     "cache/upstash_redis",
+    "cache/vercel_kv",
+    //graphs
     "graphs/neo4j_graph",
     "graphs/memgraph_graph",
     // document_compressors

libs/langchain-community/package.json

@@ -2334,6 +2334,15 @@
       "import": "./caches/upstash_redis.js",
       "require": "./caches/upstash_redis.cjs"
     },
+    "./caches/vercel_kv": {
+      "types": {
+        "import": "./caches/vercel_kv.d.ts",
+        "require": "./caches/vercel_kv.d.cts",
+        "default": "./caches/vercel_kv.d.ts"
+      },
+      "import": "./caches/vercel_kv.js",
+      "require": "./caches/vercel_kv.cjs"
+    },
     "./graphs/neo4j_graph": {
       "types": {
         "import": "./graphs/neo4j_graph.d.ts",
@@ -3900,6 +3909,10 @@
     "caches/upstash_redis.js",
     "caches/upstash_redis.d.ts",
     "caches/upstash_redis.d.cts",
+    "caches/vercel_kv.cjs",
+    "caches/vercel_kv.js",
+    "caches/vercel_kv.d.ts",
+    "caches/vercel_kv.d.cts",
     "graphs/neo4j_graph.cjs",
     "graphs/neo4j_graph.js",
     "graphs/neo4j_graph.d.ts",

libs/langchain-community/src/caches/tests/vercel_kv.int.test.ts

@@ -0,0 +1,34 @@
+/* eslint-disable no-process-env */
+import { ChatOpenAI } from "@langchain/openai";
+import { createClient } from "@vercel/kv";
+import { VercelKVCache } from "../vercel_kv.js";
+
+test("VercelKVCache works with ChatOpenAI", async () => {
+  if (
+    !process.env.VERCEL_KV_API_URL ||
+    !process.env.VERCEL_KV_API_TOKEN ||
+    !process.env.OPENAI_API_KEY
+  ) {
+    throw new Error("Missing Vercel KV API URL, token, or OpenAI API key");
+  }
+
+  const vercelKVCache = new VercelKVCache({
+    client: createClient({
+      url: process.env.VERCEL_KV_API_URL,
+      token: process.env.VERCEL_KV_API_TOKEN,
+    }),
+    ttl: 60,
+  });
+
+  const chat = new ChatOpenAI({
+    temperature: 0,
+    cache: vercelKVCache,
+    maxTokens: 10,
+  });
+
+  const prompt = "What color is the sky?";
+  const result1 = await chat.invoke(prompt);
+  const result2 = await chat.invoke(prompt);
+
+  expect(result1).toEqual(result2);
+});

libs/langchain-community/src/caches/vercel_kv.ts

@@ -0,0 +1,87 @@
+import { kv, type VercelKV } from "@vercel/kv";
+
+import { Generation } from "@langchain/core/outputs";
+import {
+  BaseCache,
+  deserializeStoredGeneration,
+  getCacheKey,
+  serializeGeneration,
+} from "@langchain/core/caches";
+import { StoredGeneration } from "@langchain/core/messages";
+
+export type VercelKVCacheProps = {
+  /**
+   * An existing Vercel KV client
+   */
+  client?: VercelKV;
+  /**
+   * Time-to-live (TTL) for cached items in seconds
+   */
+  ttl?: number;
+};
+
+/**
+ * A cache that uses Vercel KV as the backing store.
+ * @example
+ * ```typescript
+ * const cache = new VercelKVCache({
+ *   ttl: 3600, // Optional: Cache entries will expire after 1 hour
+ * });
+ *
+ * // Initialize the OpenAI model with Vercel KV cache for caching responses
+ * const model = new ChatOpenAI({
+ *   cache,
+ * });
+ * await model.invoke("How are you today?");
+ * const cachedValues = await cache.lookup("How are you today?", "llmKey");
+ * ```
+ */
+export class VercelKVCache extends BaseCache {
+  private client: VercelKV;
+
+  private ttl?: number;
+
+  constructor(props: VercelKVCacheProps) {
+    super();
+    const { client, ttl } = props;
+    this.client = client ?? kv;
+    this.ttl = ttl;
+  }
+
+  /**
+   * Lookup LLM generations in cache by prompt and associated LLM key.
+   */
+  public async lookup(prompt: string, llmKey: string) {
+    let idx = 0;
+    let key = getCacheKey(prompt, llmKey, String(idx));
+    let value = await this.client.get<StoredGeneration | null>(key);
+    const generations: Generation[] = [];
+
+    while (value) {
+      generations.push(deserializeStoredGeneration(value));
+      idx += 1;
+      key = getCacheKey(prompt, llmKey, String(idx));
+      value = await this.client.get<StoredGeneration | null>(key);
+    }
+
+    return generations.length > 0 ? generations : null;
+  }
+
+  /**
+   * Update the cache with the given generations.
+   *
+   * Note this overwrites any existing generations for the given prompt and LLM key.
+   */
+  public async update(prompt: string, llmKey: string, value: Generation[]) {
+    for (let i = 0; i < value.length; i += 1) {
+      const key = getCacheKey(prompt, llmKey, String(i));
+      const serializedValue = JSON.stringify(serializeGeneration(value[i]));
+
+      if (this.ttl) {
+        await this.client.set(key, serializedValue, { ex: this.ttl });
+      } else {
+        await this.client.set(key, serializedValue);
+      }
+    }
+  }
+}

libs/langchain-community/src/load/import_map.ts

@@ -67,6 +67,7 @@ export * as caches__cloudflare_kv from "../caches/cloudflare_kv.js";
 export * as caches__ioredis from "../caches/ioredis.js";
 export * as caches__momento from "../caches/momento.js";
 export * as caches__upstash_redis from "../caches/upstash_redis.js";
+export * as caches__vercel_kv from "../caches/vercel_kv.js";
 export * as stores__doc__base from "../stores/doc/base.js";
 export * as stores__doc__gcs from "../stores/doc/gcs.js";
 export * as stores__doc__in_memory from "../stores/doc/in_memory.js";