Testing Typescript inference (#72)

Author: ethanleifer

.changeset/plenty-eggs-grow.md

@@ -0,0 +1,5 @@
+---
+"@instructor-ai/instructor": patch
+---
+
+Implements testing for typescript inference

bun.lockb

@@ -1,2 +1,5 @@
 [install]
-peer = true
+peer = true
+
+[test]
+coverage = true

bunfig.toml

@@ -49,6 +49,7 @@
   "homepage": "https://github.com/instructor-ai/instructor-js#readme",
   "dependencies": {
     "schema-stream": "1.6.0",
+    "ts-inference-check": "^0.3.0",
     "zod-to-json-schema": "^3.22.3",
     "zod-validation-error": "^2.1.0"
   },

package.json

@@ -293,7 +293,6 @@ class Instructor {
         params: P
       ): ReturnTypeBasedOnParams<P> => {
         if ("response_model" in params) {
-          console.log(params.response_model.name)
           return this.chatCompletion(params) as ReturnTypeBasedOnParams<P>
         } else {
           return this.chatCompletionWithoutModel(params) as ReturnTypeBasedOnParams<P>

src/instructor.ts

@@ -0,0 +1,106 @@
+// cases of type inference testing:
+// 1. no response_model, stream
+// 2. no response_model, no stream
+// 3. response_model, stream, no max_retries
+// 4. response_model, no stream, no max_retries
+// 5. response_model, stream, max_retries
+// 6. response_model, no stream, max_retries
+
+import Instructor from "@/instructor"
+import { describe, expect, test } from "bun:test"
+import OpenAI from "openai"
+import { Stream } from "openai/streaming.mjs"
+import { type } from "ts-inference-check"
+import { z } from "zod"
+
+describe("Inference Checking", () => {
+  const UserSchema = z.object({
+    age: z.number(),
+    name: z.string()
+  })
+
+  const oai = new OpenAI({
+    apiKey: process.env.OPENAI_API_KEY ?? undefined,
+    organization: process.env.OPENAI_ORG_ID ?? undefined
+  })
+
+  const client = Instructor({
+    client: oai,
+    mode: "FUNCTIONS"
+  })
+
+  test("no response_model, no stream", async () => {
+    const user = await client.chat.completions.create({
+      messages: [{ role: "user", content: "Jason Liu is 30 years old" }],
+      model: "gpt-3.5-turbo",
+      seed: 1,
+      stream: false
+    })
+
+    expect(type(user).strictly.is<OpenAI.Chat.ChatCompletion>(true)).toBe(true)
+  })
+
+  test("no response_model, stream", async () => {
+    const userStream = await client.chat.completions.create({
+      messages: [{ role: "user", content: "Jason Liu is 30 years old" }],
+      model: "gpt-3.5-turbo",
+      seed: 1,
+      stream: true
+    })
+
+    expect(type(userStream).strictly.is<Stream<OpenAI.Chat.ChatCompletionChunk>>(true)).toBe(true)
+  })
+
+  test("response_model, no stream", async () => {
+    const user = await client.chat.completions.create({
+      messages: [{ role: "user", content: "Jason Liu is 30 years old" }],
+      model: "gpt-3.5-turbo",
+      response_model: { schema: UserSchema, name: "User" },
+      seed: 1,
+      stream: false
+    })
+
+    expect(type(user).strictly.is<z.infer<typeof UserSchema>>(true)).toBe(true)
+  })
+
+  test("response_model, stream", async () => {
+    const userStream = await client.chat.completions.create({
+      messages: [{ role: "user", content: "Jason Liu is 30 years old" }],
+      model: "gpt-3.5-turbo",
+      response_model: { schema: UserSchema, name: "User" },
+      seed: 1,
+      stream: true
+    })
+
+    expect(
+      type(userStream).strictly.is<AsyncGenerator<z.infer<typeof UserSchema>, void, unknown>>(true)
+    ).toBe(true)
+  })
+
+  test("response_model, stream, max_retries", async () => {
+    const userStream = await client.chat.completions.create({
+      messages: [{ role: "user", content: "Jason Liu is 30 years old" }],
+      model: "gpt-3.5-turbo",
+      response_model: { schema: UserSchema, name: "User" },
+      seed: 1,
+      stream: true,
+      max_retries: 3
+    })
+
+    expect(
+      type(userStream).strictly.is<AsyncGenerator<z.infer<typeof UserSchema>, void, unknown>>(true)
+    ).toBe(true)
+  })
+
+  test("response_model, no stream, max_retries", async () => {
+    const user = await client.chat.completions.create({
+      messages: [{ role: "user", content: "Jason Liu is 30 years old" }],
+      model: "gpt-3.5-turbo",
+      response_model: { schema: UserSchema, name: "User" },
+      seed: 1,
+      max_retries: 3
+    })
+
+    expect(type(user).strictly.is<z.infer<typeof UserSchema>>(true)).toBe(true)
+  })
+})