core[minor],google-common[minor]: Add support for generic objects in prompts, gemini audio/video docs

langchain-core/src/prompts/chat.ts

@@ -706,7 +706,13 @@ function _coerceMessagePromptTemplateLike(
   const message = coerceMessageLikeToMessage(messagePromptTemplateLike);
   let templateData:
     | string
-    | (string | _TextTemplateParam | _ImageTemplateParam)[];
+    | (
+        | string
+        | _TextTemplateParam
+        | _ImageTemplateParam
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        | Record<string, any>
+      )[];
 
   if (typeof message.content === "string") {
     templateData = message.content;
@@ -718,7 +724,7 @@ function _coerceMessagePromptTemplateLike(
       } else if ("image_url" in item) {
         return { image_url: item.image_url };
       } else {
-        throw new Error("Invalid message content");
+        return item;
       }
     });
   }

langchain-core/src/prompts/tests/chat.test.ts

@@ -568,3 +568,29 @@ test("Multi-modal, multi part chat prompt works with instances of BaseMessage",
   });
   expect(messages).toMatchSnapshot();
 });
+
+test.only("Gemini can understand audio", async () => {
+  const audioBase64 = `fs.readFileSync(audioPath, "base64");`;
+
+  const prompt = ChatPromptTemplate.fromMessages([
+    new MessagesPlaceholder("audio"),
+  ]);
+
+  const pInvoke = await prompt.invoke({
+    audio: new HumanMessage({
+      content: [
+        {
+          type: "audio",
+          data: {
+            url: `data:audio/mp3;base64,${audioBase64}`,
+          },
+        },
+        {
+          type: "text",
+          text: "Summarize this audio. Be very concise.",
+        },
+      ],
+    }),
+  });
+  console.log(JSON.stringify(pInvoke, null, 2));
+});

libs/langchain-google-common/src/utils/common.ts

@@ -24,6 +24,7 @@ export function copyAIModelParamsInto(
   const model = options?.model ?? params?.model ?? target.model;
   ret.modelName =
     model ?? options?.modelName ?? params?.modelName ?? target.modelName;
+  ret.model = model;
   ret.temperature =
     options?.temperature ?? params?.temperature ?? target.temperature;
   ret.maxOutputTokens =

libs/langchain-google-common/src/utils/gemini.ts

@@ -32,6 +32,18 @@ import type {
 } from "../types.js";
 import { GoogleAISafetyError } from "./safety.js";
 
+const extractMimeType = (
+  str: string
+): { mimeType: string; data: string } | null => {
+  if (str.startsWith("data:")) {
+    return {
+      mimeType: str.split(":")[1].split(";")[0],
+      data: str.split(",")[1],
+    };
+  }
+  return null;
+};
+
 function messageContentText(
   content: MessageContentText
 ): GeminiPartText | null {
@@ -51,17 +63,14 @@ function messageContentImageUrl(
     typeof content.image_url === "string"
       ? content.image_url
       : content.image_url.url;
-
   if (!url) {
     throw new Error("Missing Image URL");
   }
 
-  if (url.startsWith("data:")) {
+  const mineTypeAndData = extractMimeType(url);
+  if (mineTypeAndData) {
     return {
-      inlineData: {
-        mimeType: url.split(":")[1].split(";")[0],
-        data: url.split(",")[1],
-      },
+      inlineData: mineTypeAndData,
     };
   } else {
     // FIXME - need some way to get mime type
@@ -74,6 +83,32 @@ function messageContentImageUrl(
   }
 }
 
+function messageContentToAudio(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  content: Record<string, any>
+): GeminiPartInlineData | GeminiPartFileData {
+  const { url } = content.data;
+
+  if (!url) {
+    throw new Error("Missing Audio URL");
+  }
+
+  const mineTypeAndData = extractMimeType(url);
+  if (mineTypeAndData) {
+    return {
+      inlineData: mineTypeAndData,
+    };
+  } else {
+    // FIXME - need some way to get mime type
+    return {
+      fileData: {
+        mimeType: "audio/mpeg",
+        fileUri: url,
+      },
+    };
+  }
+}
+
 export function messageContentToParts(content: MessageContent): GeminiPart[] {
   // Convert a string to a text type MessageContent if needed
   const messageContent: MessageContent =
@@ -101,6 +136,11 @@ export function messageContentToParts(content: MessageContent): GeminiPart[] {
             return messageContentImageUrl(content as MessageContentImageUrl);
           }
           break;
+        case "audio":
+          if ("data" in content) {
+            return messageContentToAudio(content);
+          }
+          break;
         default:
           throw new Error(
             `Unsupported type received while converting message to message parts`

libs/langchain-google-gauth/src/tests/chat_models.audio.int.test.ts

@@ -0,0 +1,46 @@
+import fs from "fs";
+import {
+  ChatPromptTemplate,
+  MessagesPlaceholder,
+} from "@langchain/core/prompts";
+import { HumanMessage } from "@langchain/core/messages";
+import { ChatGoogle } from "../chat_models.js";
+
+function convertMp3ToBase64(filePath: string): string {
+  const fileData = fs.readFileSync(filePath);
+  const base64String = Buffer.from(fileData).toString("base64");
+  return base64String;
+}
+
+test("Gemini can understand audio", async () => {
+  const model = new ChatGoogle({
+    model: "gemini-1.5-pro-preview-0409",
+    temperature: 0,
+  });
+
+  const audioPath = "./src/tests/data/audio.mp3";
+  const audioBase64 = convertMp3ToBase64(audioPath);
+
+  const prompt = ChatPromptTemplate.fromMessages([
+    new MessagesPlaceholder("audio"),
+  ]);
+
+  const chain = prompt.pipe(model);
+  const response = await chain.invoke({
+    audio: new HumanMessage({
+      content: [
+        {
+          type: "audio",
+          data: {
+            url: `data:audio/mp3;base64,${audioBase64}`,
+          },
+        },
+        {
+          type: "text",
+          text: "Summarize this audio. Be very concise.",
+        },
+      ],
+    }),
+  });
+  console.log("response", response);
+});