Skip to content

Commit de2f5fa

Browse files
bracesproulbracesproul
committed
core[minor],google-common[minor]: Add generic object prompt template
1 parent 1c270f4 commit de2f5fa

File tree

10 files changed

+352
-10
lines changed

10 files changed

+352
-10
lines changed

‎langchain-core/src/prompt_values.ts

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,3 +175,50 @@ export class ImagePromptValue extends BasePromptValue {
175175
];
176176
}
177177
}
178+
179+
export interface GenericObjectPromptValueFields {
180+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
181+
data: Record<string, any>;
182+
}
183+
184+
/**
185+
* Class that represents a generic object prompt value.
186+
*/
187+
export class GenericObjectPromptValue extends BasePromptValue {
188+
lc_namespace = ["langchain_core", "prompt_values"];
189+
190+
lc_serializable = true;
191+
192+
static lc_name() {
193+
return "GenericObjectPromptValue";
194+
}
195+
196+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
197+
data: Record<string, any>;
198+
199+
/** @ignore */
200+
value: string;
201+
202+
203+
constructor(fields: GenericObjectPromptValueFields) {
204+
super(fields);
205+
this.data = fields.data;
206+
}
207+
208+
toString() {
209+
return JSON.stringify(this.data, null, 2)
210+
}
211+
212+
toChatMessages() {
213+
return [
214+
new HumanMessage({
215+
content: [
216+
{
217+
type: "generic",
218+
data: this.data,
219+
},
220+
],
221+
}),
222+
];
223+
}
224+
}

‎langchain-core/src/prompts/chat.ts

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import {
2828
import { PromptTemplate, type ParamsFromFString } from "./prompt.js";
2929
import { ImagePromptTemplate } from "./image.js";
3030
import { parseFString } from "./template.js";
31+
import { GenericObjectPromptTemplate } from "./generic_object.js";
3132

3233
/**
3334
* Abstract class that serves as a base for creating message prompt
@@ -344,6 +345,11 @@ interface _ImageTemplateParam {
344345
image_url?: string | Record<string, any>;
345346
}
346347

348+
interface _GenericObjectTemplateParam {
349+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
350+
data: Record<string, any>;
351+
}
352+
347353
type MessageClass =
348354
| typeof HumanMessage
349355
| typeof AIMessage
@@ -381,6 +387,10 @@ class _StringImageMessagePromptTemplate<
381387
| MessageStringPromptTemplateFields<
382388
InputValues<Extract<keyof RunInput, string>>
383389
>
390+
| GenericObjectPromptTemplate<
391+
InputValues<Extract<keyof RunInput, string>>,
392+
string
393+
>
384394
>;
385395

386396
protected messageClass?: MessageClass;
@@ -455,14 +465,14 @@ class _StringImageMessagePromptTemplate<
455465
}
456466

457467
static fromTemplate(
458-
template: string | Array<string | _TextTemplateParam | _ImageTemplateParam>,
468+
template: string | Array<string | _TextTemplateParam | _ImageTemplateParam | _GenericObjectTemplateParam>,
459469
additionalOptions?: Record<string, unknown>
460470
) {
461471
if (typeof template === "string") {
462472
return new this(PromptTemplate.fromTemplate(template));
463473
}
464474
const prompt: Array<
465-
PromptTemplate<InputValues> | ImagePromptTemplate<InputValues>
475+
PromptTemplate<InputValues> | ImagePromptTemplate<InputValues> | GenericObjectPromptTemplate<InputValues>
466476
> = [];
467477
for (const item of template) {
468478
if (
@@ -519,6 +529,9 @@ class _StringImageMessagePromptTemplate<
519529
throw new Error("Invalid image template");
520530
}
521531
prompt.push(imgTemplateObject);
532+
} else if (typeof item === "object" && "data" in item) {
533+
const genericTemplate = item.data;
534+
prompt.push(new GenericObjectPromptTemplate<InputValues>({ template: genericTemplate, inputVariables: [] }));
522535
}
523536
}
524537
return new this({ prompt, additionalOptions });
@@ -559,6 +572,13 @@ class _StringImageMessagePromptTemplate<
559572
inputs as TypedPromptInputValues<RunInput>
560573
);
561574
content.push({ type: "image_url", image_url: formatted });
575+
// eslint-disable-next-line no-instanceof/no-instanceof
576+
} else if (prompt instanceof GenericObjectPromptTemplate) {
577+
const formatted = await prompt.format(
578+
inputs as TypedPromptInputValues<RunInput>
579+
);
580+
console.log("formatted", formatted)
581+
content.push({ type: "generic", data: formatted });
562582
}
563583
}
564584

@@ -706,7 +726,7 @@ function _coerceMessagePromptTemplateLike(
706726
const message = coerceMessageLikeToMessage(messagePromptTemplateLike);
707727
let templateData:
708728
| string
709-
| (string | _TextTemplateParam | _ImageTemplateParam)[];
729+
| (string | _TextTemplateParam | _ImageTemplateParam | _GenericObjectTemplateParam)[];
710730

711731
if (typeof message.content === "string") {
712732
templateData = message.content;
@@ -717,8 +737,12 @@ function _coerceMessagePromptTemplateLike(
717737
return { text: item.text };
718738
} else if ("image_url" in item) {
719739
return { image_url: item.image_url };
740+
} else if ("data" in item) {
741+
return { data: item.data };
720742
} else {
721-
throw new Error("Invalid message content");
743+
throw new Error(
744+
"Invalid message content"
745+
);
722746
}
723747
});
724748
}

‎langchain-core/src/prompts/generic_object.ts

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
import { MessageContent } from "../messages/index.js";
2+
import {
3+
GenericObjectPromptValue,
4+
StringPromptValue,
5+
} from "../prompt_values.js";
6+
import type { InputValues, PartialValues } from "../utils/types/index.js";
7+
import {
8+
BasePromptTemplate,
9+
BasePromptTemplateInput,
10+
TypedPromptInputValues,
11+
} from "./base.js";
12+
import { TemplateFormat, checkValidTemplate } from "./template.js";
13+
14+
/**
15+
* Inputs to create a {@link GenericObjectPromptTemplate}
16+
* @augments BasePromptTemplateInput
17+
*/
18+
export interface GenericObjectPromptTemplateInput<
19+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
20+
RunInput extends InputValues = any,
21+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
22+
PartialVariableName extends string = any
23+
> extends BasePromptTemplateInput<RunInput, PartialVariableName> {
24+
/**
25+
* The prompt template
26+
*/
27+
template: Record<string, unknown>;
28+
29+
/**
30+
* The format of the prompt template. Options are 'f-string'
31+
*
32+
* @defaultValue 'f-string'
33+
*/
34+
templateFormat?: TemplateFormat;
35+
36+
/**
37+
* Whether or not to try validating the template on initialization
38+
*
39+
* @defaultValue `true`
40+
*/
41+
validateTemplate?: boolean;
42+
}
43+
44+
/**
45+
* A generic prompt template for a multimodal model.
46+
*/
47+
export class GenericObjectPromptTemplate<
48+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
49+
RunInput extends InputValues = any,
50+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
51+
PartialVariableName extends string = any
52+
> extends BasePromptTemplate<RunInput, StringPromptValue, PartialVariableName> {
53+
static lc_name() {
54+
return "GenericObjectPromptTemplate";
55+
}
56+
57+
lc_namespace = ["langchain_core", "prompts", "generic_object"];
58+
59+
template: Record<string, unknown>;
60+
61+
templateFormat: TemplateFormat = "f-string";
62+
63+
validateTemplate = true;
64+
65+
constructor(input: GenericObjectPromptTemplateInput<RunInput, PartialVariableName>) {
66+
super(input);
67+
this.template = input.template;
68+
this.templateFormat = input.templateFormat ?? this.templateFormat;
69+
this.validateTemplate = input.validateTemplate ?? this.validateTemplate;
70+
71+
if (this.validateTemplate) {
72+
let totalInputVariables: string[] = this.inputVariables;
73+
if (this.partialVariables) {
74+
totalInputVariables = totalInputVariables.concat(
75+
Object.keys(this.partialVariables)
76+
);
77+
}
78+
checkValidTemplate(
79+
[
80+
{ type: "generic", data: this.template },
81+
] as unknown as MessageContent,
82+
this.templateFormat,
83+
totalInputVariables
84+
);
85+
}
86+
}
87+
88+
_getPromptType(): "prompt" {
89+
return "prompt";
90+
}
91+
92+
/**
93+
* Partially applies values to the prompt template.
94+
* @param values The values to be partially applied to the prompt template.
95+
* @returns A new instance of GenericObjectPromptTemplate with the partially applied values.
96+
*/
97+
async partial<NewPartialVariableName extends string>(
98+
values: PartialValues<NewPartialVariableName>
99+
) {
100+
const newInputVariables = this.inputVariables.filter(
101+
(iv) => !(iv in values)
102+
) as Exclude<Extract<keyof RunInput, string>, NewPartialVariableName>[];
103+
const newPartialVariables = {
104+
...(this.partialVariables ?? {}),
105+
...values,
106+
} as PartialValues<PartialVariableName | NewPartialVariableName>;
107+
const promptDict = {
108+
...this,
109+
inputVariables: newInputVariables,
110+
partialVariables: newPartialVariables,
111+
};
112+
return new GenericObjectPromptTemplate<
113+
InputValues<
114+
Exclude<Extract<keyof RunInput, string>, NewPartialVariableName>
115+
>
116+
>(promptDict);
117+
}
118+
119+
/**
120+
* Formats the prompt template with the provided values.
121+
* @param values The values to be used to format the prompt template.
122+
* @returns A promise that resolves to a string which is the formatted prompt.
123+
*/
124+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
125+
async format<FormatOutput = Record<string, any>>(
126+
values: TypedPromptInputValues<RunInput>
127+
): Promise<FormatOutput> {
128+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
129+
const formatted: Record<string, any> = {};
130+
for (const [key, value] of Object.entries(this.template)) {
131+
if (typeof value === "string") {
132+
formatted[key] = value.replace(/{([^{}]*)}/g, (match, group) => {
133+
const replacement = values[group];
134+
return typeof replacement === "string" ||
135+
typeof replacement === "number"
136+
? String(replacement)
137+
: match;
138+
});
139+
} else {
140+
formatted[key] = value;
141+
}
142+
}
143+
const data = Object.values(values).length ? values : formatted;
144+
145+
return data as FormatOutput;
146+
}
147+
148+
/**
149+
* Formats the prompt given the input values and returns a formatted
150+
* prompt value.
151+
* @param values The input values to format the prompt.
152+
* @returns A Promise that resolves to a formatted prompt value.
153+
*/
154+
async formatPromptValue(
155+
values: TypedPromptInputValues<RunInput>
156+
): Promise<GenericObjectPromptValue> {
157+
const formattedPrompt = await this.format(values);
158+
return new GenericObjectPromptValue({
159+
data: formattedPrompt,
160+
});
161+
}
162+
}

‎langchain-core/src/prompts/template.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,8 @@ export const checkValidTemplate = (
135135
const imageUrl = message.image_url.url;
136136
renderTemplate(imageUrl, templateFormat, dummyInputs);
137137
}
138+
} else if (message.type === "generic") {
139+
// no-op
138140
} else {
139141
throw new Error(
140142
`Invalid message template received. ${JSON.stringify(

‎langchain-core/src/prompts/tests/chat.test.ts

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -568,3 +568,28 @@ test("Multi-modal, multi part chat prompt works with instances of BaseMessage",
568568
});
569569
expect(messages).toMatchSnapshot();
570570
});
571+
572+
test.only("Gemini can understand audio", async () => {
573+
const audioBase64 = `fs.readFileSync(audioPath, "base64");`
574+
575+
const prompt = ChatPromptTemplate.fromMessages([
576+
[
577+
"human",
578+
[
579+
{
580+
type: "generic",
581+
data: {
582+
url: `data:audio/mp3;base64,${audioBase64}`,
583+
},
584+
},
585+
{
586+
type: "text",
587+
text: "Summarize this audio. Be very concise.",
588+
},
589+
],
590+
],
591+
]);
592+
593+
const pInvoke = await prompt.invoke({});
594+
console.log(JSON.stringify(pInvoke, null, 2))
595+
});

‎libs/langchain-google-common/src/utils/common.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ export function copyAIModelParamsInto(
2424
const model = options?.model ?? params?.model ?? target.model;
2525
ret.modelName =
2626
model ?? options?.modelName ?? params?.modelName ?? target.modelName;
27+
ret.model = model;
2728
ret.temperature =
2829
options?.temperature ?? params?.temperature ?? target.temperature;
2930
ret.maxOutputTokens =

0 commit comments

Comments
 (0)