Skip to content

Commit 3aac90e

Browse files
authored
new parser for json + thinking output + tests using deepseek r1 on groq (#195)
1 parent 3bd1719 commit 3aac90e

File tree

10 files changed

+132
-33
lines changed

10 files changed

+132
-33
lines changed

.changeset/little-cougars-bow.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@instructor-ai/instructor": minor
3+
---
4+
5+
adding a new mode to support parsing thinking blocks out of markdown json responses (R1)

.github/workflows/test-pr.yml

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ jobs:
2323
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
2424
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
2525
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
26+
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
2627
steps:
2728
- uses: actions/checkout@v3
2829
with:

.github/workflows/test.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
1616
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
1717
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
18-
18+
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
1919
steps:
2020
- uses: actions/checkout@v3
2121
with:

bun.lockb

0 Bytes
Binary file not shown.

package.json

+5-2
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,17 @@
4444
"outputs",
4545
"zod"
4646
],
47-
"author": "Jason Liu",
47+
"contributors": [
48+
"Dimitri Kennedy",
49+
"Jason Liu"
50+
],
4851
"license": "MIT",
4952
"bugs": {
5053
"url": "https://github.com/instructor-ai/instructor-js/issues"
5154
},
5255
"homepage": "https://github.com/instructor-ai/instructor-js#readme",
5356
"dependencies": {
54-
"zod-stream": "2.0.0",
57+
"zod-stream": "3.0.0",
5558
"zod-validation-error": "^3.4.0"
5659
},
5760
"peerDependencies": {

src/constants/providers.ts

+14-4
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
import { omit } from "@/lib"
22
import OpenAI from "openai"
33
import { z } from "zod"
4-
import { withResponseModel, MODE as ZMODE, type Mode } from "zod-stream"
4+
import { thinkingJsonParser, withResponseModel, MODE as ZMODE } from "zod-stream"
5+
6+
import { Mode } from "../types"
7+
8+
export const MODE: typeof ZMODE = ZMODE
9+
10+
export const MODE_TO_RESPONSE_PARSER = {
11+
[MODE.THINKING_MD_JSON]: thinkingJsonParser
12+
}
513

6-
export const MODE = ZMODE
714
export const PROVIDERS = {
815
OAI: "OAI",
916
ANYSCALE: "ANYSCALE",
@@ -12,6 +19,7 @@ export const PROVIDERS = {
1219
GROQ: "GROQ",
1320
OTHER: "OTHER"
1421
} as const
22+
1523
export type Provider = keyof typeof PROVIDERS
1624

1725
export const PROVIDER_SUPPORTED_MODES: {
@@ -98,7 +106,8 @@ export const PROVIDER_SUPPORTED_MODES_BY_MODEL = {
98106
[MODE.TOOLS]: ["*"],
99107
[MODE.JSON]: ["*"],
100108
[MODE.MD_JSON]: ["*"],
101-
[MODE.JSON_SCHEMA]: ["*"]
109+
[MODE.JSON_SCHEMA]: ["*"],
110+
[MODE.THINKING_MD_JSON]: ["*"]
102111
},
103112
[PROVIDERS.OAI]: {
104113
[MODE.FUNCTIONS]: ["*"],
@@ -122,6 +131,7 @@ export const PROVIDER_SUPPORTED_MODES_BY_MODEL = {
122131
},
123132
[PROVIDERS.GROQ]: {
124133
[MODE.TOOLS]: ["*"],
125-
[MODE.MD_JSON]: ["*"]
134+
[MODE.MD_JSON]: ["*"],
135+
[MODE.THINKING_MD_JSON]: ["deepseek-r1-distill-llama-70b"]
126136
}
127137
}

src/instructor.ts

+18-7
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,19 @@ import {
1111
import OpenAI from "openai"
1212
import { Stream } from "openai/streaming"
1313
import { z, ZodError } from "zod"
14-
import ZodStream, { OAIResponseParser, OAIStream, withResponseModel, type Mode } from "zod-stream"
14+
import ZodStream, { OAIResponseParser, OAIStream, withResponseModel } from "zod-stream"
1515
import { fromZodError } from "zod-validation-error"
1616

1717
import {
18+
MODE_TO_RESPONSE_PARSER,
1819
NON_OAI_PROVIDER_URLS,
1920
Provider,
2021
PROVIDER_PARAMS_TRANSFORMERS,
2122
PROVIDER_SUPPORTED_MODES,
2223
PROVIDERS
2324
} from "./constants/providers"
2425
import { iterableTee } from "./lib"
25-
import { ClientTypeChatCompletionParams, CompletionMeta } from "./types"
26+
import { ClientTypeChatCompletionParams, CompletionMeta, Mode } from "./types"
2627

2728
const MAX_RETRIES_DEFAULT = 0
2829

@@ -68,6 +69,7 @@ class Instructor<C> {
6869
: this.client?.baseURL.includes(NON_OAI_PROVIDER_URLS.TOGETHER) ? PROVIDERS.TOGETHER
6970
: this.client?.baseURL.includes(NON_OAI_PROVIDER_URLS.OAI) ? PROVIDERS.OAI
7071
: this.client?.baseURL.includes(NON_OAI_PROVIDER_URLS.ANTHROPIC) ? PROVIDERS.ANTHROPIC
72+
: this.client?.baseURL.includes(NON_OAI_PROVIDER_URLS.GROQ) ? PROVIDERS.GROQ
7173
: PROVIDERS.OTHER
7274
: PROVIDERS.OTHER
7375

@@ -187,13 +189,22 @@ class Instructor<C> {
187189
throw error
188190
}
189191

190-
const parsedCompletion = OAIResponseParser(
191-
completion as OpenAI.Chat.Completions.ChatCompletion
192-
)
192+
const responseParser = MODE_TO_RESPONSE_PARSER?.[this.mode] ?? OAIResponseParser
193+
const parsedCompletion = responseParser(completion as OpenAI.Chat.Completions.ChatCompletion)
193194

194195
try {
195-
const data = JSON.parse(parsedCompletion) as z.infer<T> & { _meta?: CompletionMeta }
196-
return { ...data, _meta: { usage: completion?.usage ?? undefined } }
196+
const responseJson = parsedCompletion.json ?? parsedCompletion
197+
const data = JSON.parse(responseJson) as z.infer<T> & {
198+
_meta?: CompletionMeta
199+
thinking?: string
200+
}
201+
return {
202+
...data,
203+
_meta: {
204+
usage: completion?.usage ?? undefined,
205+
thinking: parsedCompletion?.thinking ?? undefined
206+
}
207+
}
197208
} catch (error) {
198209
this.log(
199210
"error",

src/types/index.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,10 @@ export type LogLevel = "debug" | "info" | "warn" | "error"
6161

6262
export type CompletionMeta = Partial<ZCompletionMeta> & {
6363
usage?: OpenAI.CompletionUsage
64+
thinking?: string
6465
}
6566

66-
export type Mode = ZMode
67+
export type Mode = ZMode | "THINKING_MD_JSON"
6768

6869
export type ResponseModel<T extends z.AnyZodObject> = ZResponseModel<T>
6970

tests/deepseek.test.ts

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import Instructor from "@/index"
2+
import { describe, expect, test } from "bun:test"
3+
import OpenAI from "openai"
4+
import { z } from "zod"
5+
6+
const textBlock = `
7+
In our recent online meeting, participants from various backgrounds joined to discuss the upcoming tech conference. The names and contact details of the participants were as follows:
8+
9+
- Name: John Doe, Email: [email protected], Twitter: @TechGuru44
10+
- Name: Jane Smith, Email: [email protected], Twitter: @DigitalDiva88
11+
- Name: Alex Johnson, Email: [email protected], Twitter: @CodeMaster2023
12+
13+
During the meeting, we agreed on several key points. The conference will be held on March 15th, 2024, at the Grand Tech Arena located at 4521 Innovation Drive. Dr. Emily Johnson, a renowned AI researcher, will be our keynote speaker.
14+
15+
The budget for the event is set at $50,000, covering venue costs, speaker fees, and promotional activities. Each participant is expected to contribute an article to the conference blog by February 20th.
16+
17+
A follow-up meeting is scheduled for January 25th at 3 PM GMT to finalize the agenda and confirm the list of speakers.
18+
`
19+
20+
const ExtractionValuesSchema = z.object({
21+
users: z
22+
.array(
23+
z.object({
24+
name: z.string(),
25+
email: z.string(),
26+
twitter: z.string()
27+
})
28+
)
29+
.min(3),
30+
conference: z.object({
31+
date: z.string(),
32+
venue: z.string(),
33+
budget: z.number(),
34+
keynoteSpeaker: z.string()
35+
}),
36+
nextMeeting: z.object({
37+
date: z.string(),
38+
time: z.string(),
39+
timezone: z.string()
40+
})
41+
})
42+
43+
describe("thinking parser - live tests", () => {
44+
test("should parse r1 response with thinking tags", async () => {
45+
const groq = new OpenAI({
46+
apiKey: process.env["GROQ_API_KEY"] ?? undefined,
47+
baseURL: "https://api.groq.com/openai/v1"
48+
})
49+
50+
const client = Instructor({
51+
client: groq,
52+
mode: "THINKING_MD_JSON",
53+
debug: true
54+
})
55+
56+
const result = await client.chat.completions.create({
57+
messages: [{ role: "user", content: textBlock }],
58+
model: "deepseek-r1-distill-llama-70b",
59+
response_model: { schema: ExtractionValuesSchema, name: "Extract" },
60+
max_retries: 4
61+
})
62+
63+
console.log("result", result)
64+
65+
expect(result._meta?.thinking).toBeDefined()
66+
expect(typeof result._meta?.thinking).toBe("string")
67+
68+
expect(result.users).toHaveLength(3)
69+
expect(result.users[0]).toHaveProperty("name")
70+
expect(result.users[0]).toHaveProperty("email")
71+
expect(result.users[0]).toHaveProperty("twitter")
72+
73+
expect(result.conference).toBeDefined()
74+
expect(result.conference.budget).toBe(50000)
75+
expect(result.conference.keynoteSpeaker).toBe("Dr. Emily Johnson")
76+
77+
expect(result.nextMeeting).toBeDefined()
78+
expect(result.nextMeeting.timezone).toBe("GMT")
79+
})
80+
})

tests/mode.test.ts

+6-18
Original file line numberDiff line numberDiff line change
@@ -128,23 +128,11 @@ describe("Modes", async () => {
128128
const testCases = createTestCases()
129129

130130
for await (const { model, mode, provider, defaultMessage } of testCases) {
131-
if (provider !== PROVIDERS.GROQ) {
132-
test(`${provider}: Should return extracted name and age for model ${model} and mode ${mode}`, async () => {
133-
const user = await extractUser(model, mode, provider, defaultMessage)
134-
135-
expect(user.name).toEqual("Jason Liu")
136-
expect(user.age).toEqual(30)
137-
})
138-
} else {
139-
test.todo(
140-
`${provider}: Should return extracted name and age for model ${model} and mode ${mode}`,
141-
async () => {
142-
const user = await extractUser(model, mode, provider, defaultMessage)
143-
144-
expect(user.name).toEqual("Jason Liu")
145-
expect(user.age).toEqual(30)
146-
}
147-
)
148-
}
131+
test(`${provider}: Should return extracted name and age for model ${model} and mode ${mode}`, async () => {
132+
const user = await extractUser(model, mode, provider, defaultMessage)
133+
134+
expect(user.name).toEqual("Jason Liu")
135+
expect(user.age).toEqual(30)
136+
})
149137
}
150138
})

0 commit comments

Comments
 (0)