Skip to content

Commit 40a6364

Browse files
benjamincburnsbenjamincburns
committed
feat(google): implement standard content blocks
1 parent 01397b7 commit 40a6364

File tree

6 files changed

+456
-21
lines changed

6 files changed

+456
-21
lines changed

‎libs/langchain-google-common/src/types-anthropic.ts

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,16 @@ export interface AnthropicMessageContentText
1616
export interface AnthropicMessageContentImage
1717
extends AnthropicMessageContentBase {
1818
type: "image";
19-
source: {
20-
type: "base64" | string;
21-
media_type: string;
22-
data: string;
23-
};
19+
source:
20+
| {
21+
type: "base64" | string;
22+
media_type?: string;
23+
data: string;
24+
}
25+
| {
26+
type: "url" | string;
27+
url: string;
28+
};
2429
}
2530

2631
export interface AnthropicMessageContentThinking
@@ -30,6 +35,50 @@ export interface AnthropicMessageContentThinking
3035
signature: string;
3136
}
3237

38+
export interface AnthropicMessageContentDocument
39+
extends AnthropicMessageContentBase {
40+
type: "document";
41+
source:
42+
| {
43+
type: "base64" | "text" | string;
44+
media_type?: "application/pdf" | "text/plain" | string;
45+
data: string;
46+
}
47+
| {
48+
type: "url" | string;
49+
url: string;
50+
}
51+
| {
52+
type: "content" | string;
53+
content: {
54+
type: "image" | string;
55+
source:
56+
| {
57+
type: "base64" | string;
58+
data: string;
59+
media_type?:
60+
| "image/jpeg"
61+
| "image/png"
62+
| "image/gif"
63+
| "image/webp"
64+
| string;
65+
}
66+
| {
67+
type: "url" | string;
68+
url: string;
69+
}
70+
| {
71+
type: "text" | string;
72+
text: string;
73+
};
74+
}[];
75+
};
76+
citations?: {
77+
enabled?: boolean;
78+
};
79+
context?: string;
80+
title?: string;
81+
}
3382
export interface AnthropicMessageContentRedactedThinking
3483
extends AnthropicMessageContentBase {
3584
type: "redacted_thinking";

‎libs/langchain-google-common/src/utils/anthropic.ts

Lines changed: 248 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ import {
1414
AIMessageFields,
1515
AIMessageChunkFields,
1616
AIMessage,
17+
StandardContentBlockConverter,
18+
StandardImageBlock,
19+
StandardTextBlock,
20+
StandardFileBlock,
21+
DataContentBlock,
22+
isDataContentBlock,
23+
convertToProviderContentBlock,
24+
parseBase64DataUrl,
1725
} from "@langchain/core/messages";
1826
import {
1927
ToolCall,
@@ -22,13 +30,15 @@ import {
2230
} from "@langchain/core/messages/tool";
2331
import {
2432
AnthropicAPIConfig,
33+
AnthropicCacheControl,
2534
AnthropicContent,
2635
AnthropicContentRedactedThinking,
2736
AnthropicContentText,
2837
AnthropicContentThinking,
2938
AnthropicContentToolUse,
3039
AnthropicMessage,
3140
AnthropicMessageContent,
41+
AnthropicMessageContentDocument,
3242
AnthropicMessageContentImage,
3343
AnthropicMessageContentRedactedThinking,
3444
AnthropicMessageContentText,
@@ -528,21 +538,248 @@ export function getAnthropicAPI(config?: AnthropicAPIConfig): GoogleAIAPI {
528538
}
529539
}
530540

541+
const anthropicContentConverter: StandardContentBlockConverter<{
542+
text: AnthropicMessageContentText;
543+
image: AnthropicMessageContentImage;
544+
file: AnthropicMessageContentDocument;
545+
}> = {
546+
providerName: "anthropic",
547+
548+
fromStandardTextBlock(
549+
block: StandardTextBlock
550+
): AnthropicMessageContentText {
551+
return {
552+
type: "text",
553+
text: block.text,
554+
...("cache_control" in (block.metadata ?? {})
555+
? {
556+
cache_control: block.metadata!
557+
.cache_control as AnthropicCacheControl,
558+
}
559+
: {}),
560+
};
561+
},
562+
563+
fromStandardImageBlock(
564+
block: StandardImageBlock
565+
): AnthropicMessageContentImage {
566+
if (block.source_type === "url") {
567+
const data = parseBase64DataUrl({
568+
dataUrl: block.url,
569+
asTypedArray: false,
570+
});
571+
if (data) {
572+
return {
573+
type: "image",
574+
source: {
575+
type: "base64",
576+
data: data.data,
577+
media_type: data.mime_type,
578+
},
579+
...("cache_control" in (block.metadata ?? {})
580+
? { cache_control: block.metadata!.cache_control }
581+
: {}),
582+
} as AnthropicMessageContentImage;
583+
} else {
584+
return {
585+
type: "image",
586+
source: {
587+
type: "url",
588+
url: block.url,
589+
media_type: block.mime_type ?? "",
590+
},
591+
...("cache_control" in (block.metadata ?? {})
592+
? { cache_control: block.metadata!.cache_control }
593+
: {}),
594+
} as AnthropicMessageContentImage;
595+
}
596+
} else {
597+
if (block.source_type === "base64") {
598+
return {
599+
type: "image",
600+
source: {
601+
type: "base64",
602+
data: block.data,
603+
media_type: block.mime_type ?? "",
604+
},
605+
...("cache_control" in (block.metadata ?? {})
606+
? { cache_control: block.metadata!.cache_control }
607+
: {}),
608+
} as AnthropicMessageContentImage;
609+
} else {
610+
throw new Error(
611+
`Unsupported image source type: ${block.source_type}`
612+
);
613+
}
614+
}
615+
},
616+
617+
fromStandardFileBlock(
618+
block: StandardFileBlock
619+
): AnthropicMessageContentDocument {
620+
const mime_type = (block.mime_type ?? "").split(";")[0];
621+
622+
if (block.source_type === "url") {
623+
if (mime_type === "application/pdf" || mime_type === "") {
624+
return {
625+
type: "document",
626+
source: {
627+
type: "url",
628+
url: block.url,
629+
media_type: block.mime_type ?? "",
630+
},
631+
...("cache_control" in (block.metadata ?? {})
632+
? {
633+
cache_control: block.metadata!
634+
.cache_control as AnthropicCacheControl,
635+
}
636+
: {}),
637+
...("citations" in (block.metadata ?? {})
638+
? {
639+
citations: block.metadata!.citations as { enabled?: boolean },
640+
}
641+
: {}),
642+
...("context" in (block.metadata ?? {})
643+
? { context: block.metadata!.context as string }
644+
: {}),
645+
...(block.metadata?.title ||
646+
block.metadata?.filename ||
647+
block.metadata?.name
648+
? {
649+
title: (block.metadata?.title ||
650+
block.metadata?.filename ||
651+
block.metadata?.name) as string,
652+
}
653+
: {}),
654+
};
655+
}
656+
throw new Error(
657+
`Unsupported file mime type for file url source: ${block.mime_type}`
658+
);
659+
} else if (block.source_type === "text") {
660+
if (mime_type === "text/plain" || mime_type === "") {
661+
return {
662+
type: "document",
663+
source: {
664+
type: "text",
665+
data: block.text,
666+
media_type: block.mime_type ?? "",
667+
},
668+
...("cache_control" in (block.metadata ?? {})
669+
? {
670+
cache_control: block.metadata!
671+
.cache_control as AnthropicCacheControl,
672+
}
673+
: {}),
674+
...("citations" in (block.metadata ?? {})
675+
? {
676+
citations: block.metadata!.citations as { enabled?: boolean },
677+
}
678+
: {}),
679+
...("context" in (block.metadata ?? {})
680+
? { context: block.metadata!.context as string }
681+
: {}),
682+
...("title" in (block.metadata ?? {})
683+
? { title: block.metadata!.title as string }
684+
: {}),
685+
};
686+
} else {
687+
throw new Error(
688+
`Unsupported file mime type for file text source: ${block.mime_type}`
689+
);
690+
}
691+
} else if (block.source_type === "base64") {
692+
if (mime_type === "application/pdf" || mime_type === "") {
693+
return {
694+
type: "document",
695+
source: {
696+
type: "base64",
697+
data: block.data,
698+
media_type: "application/pdf",
699+
},
700+
...("cache_control" in (block.metadata ?? {})
701+
? {
702+
cache_control: block.metadata!
703+
.cache_control as AnthropicCacheControl,
704+
}
705+
: {}),
706+
...("citations" in (block.metadata ?? {})
707+
? {
708+
citations: block.metadata!.citations as { enabled?: boolean },
709+
}
710+
: {}),
711+
...("context" in (block.metadata ?? {})
712+
? { context: block.metadata!.context as string }
713+
: {}),
714+
...("title" in (block.metadata ?? {})
715+
? { title: block.metadata!.title as string }
716+
: {}),
717+
};
718+
} else if (
719+
["image/jpeg", "image/png", "image/gif", "image/webp"].includes(
720+
mime_type
721+
)
722+
) {
723+
return {
724+
type: "document",
725+
source: {
726+
type: "content",
727+
content: [
728+
{
729+
type: "image",
730+
source: {
731+
type: "base64",
732+
data: block.data,
733+
media_type: mime_type as
734+
| "image/jpeg"
735+
| "image/png"
736+
| "image/gif"
737+
| "image/webp",
738+
},
739+
},
740+
],
741+
},
742+
...("cache_control" in (block.metadata ?? {})
743+
? {
744+
cache_control: block.metadata!
745+
.cache_control as AnthropicCacheControl,
746+
}
747+
: {}),
748+
...("citations" in (block.metadata ?? {})
749+
? {
750+
citations: block.metadata!.citations as { enabled?: boolean },
751+
}
752+
: {}),
753+
...("context" in (block.metadata ?? {})
754+
? { context: block.metadata!.context as string }
755+
: {}),
756+
...("title" in (block.metadata ?? {})
757+
? { title: block.metadata!.title as string }
758+
: {}),
759+
};
760+
} else {
761+
throw new Error(
762+
`Unsupported file mime type for file base64 source: ${block.mime_type}`
763+
);
764+
}
765+
} else {
766+
throw new Error(`Unsupported file source type: ${block.source_type}`);
767+
}
768+
},
769+
};
770+
531771
function contentToAnthropicContent(
532-
content: MessageContent
772+
content: MessageContent | DataContentBlock[]
533773
): AnthropicMessageContent[] {
534-
const ret: AnthropicMessageContent[] = [];
535-
536774
const ca =
537775
typeof content === "string" ? [{ type: "text", text: content }] : content;
538-
ca.forEach((complex) => {
539-
const ac = contentComplexToAnthropicContent(complex);
540-
if (ac) {
541-
ret.push(ac);
542-
}
543-
});
544-
545-
return ret;
776+
return ca
777+
.map((complex) =>
778+
isDataContentBlock(complex)
779+
? convertToProviderContentBlock(complex, anthropicContentConverter)
780+
: contentComplexToAnthropicContent(complex)
781+
)
782+
.filter(Boolean) as AnthropicMessageContent[];
546783
}
547784

548785
function toolCallToAnthropicContent(

0 commit comments

Comments
 (0)