diff --git a/.github/workflows/standard-tests.yml b/.github/workflows/standard-tests.yml new file mode 100644 index 000000000000..5411831078b7 --- /dev/null +++ b/.github/workflows/standard-tests.yml @@ -0,0 +1,23 @@ +name: Standard Tests (Integration) + +on: + workflow_dispatch: + schedule: + - cron: '0 13 * * *' + +jobs: + openai: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable --mode=skip-build + - name: Run standard tests (integration) + run: yarn test:standard:int --filter=@langchain/openai + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/langchain-core/src/language_models/chat_models.ts b/langchain-core/src/language_models/chat_models.ts index 64924bbebd47..d78bb04b2e0e 100644 --- a/langchain-core/src/language_models/chat_models.ts +++ b/langchain-core/src/language_models/chat_models.ts @@ -263,7 +263,7 @@ export abstract class BaseChatModel< } } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { return { ls_model_type: "chat", ls_stop: options.stop, diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index 9b270ad37752..ab1acebd0ca7 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -518,7 +518,7 @@ export class ChatAnthropicMessages< this.clientOptions = fields?.clientOptions ?? {}; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "openai", diff --git a/libs/langchain-cohere/src/chat_models.ts b/libs/langchain-cohere/src/chat_models.ts index 1aeca0175581..d88145ae8ba1 100644 --- a/libs/langchain-cohere/src/chat_models.ts +++ b/libs/langchain-cohere/src/chat_models.ts @@ -146,7 +146,7 @@ export class ChatCohere< this.streaming = fields?.streaming ?? this.streaming; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "cohere", diff --git a/libs/langchain-community/src/chat_models/fireworks.ts b/libs/langchain-community/src/chat_models/fireworks.ts index 65d6a6588c31..2fb7a62bd118 100644 --- a/libs/langchain-community/src/chat_models/fireworks.ts +++ b/libs/langchain-community/src/chat_models/fireworks.ts @@ -104,7 +104,7 @@ export class ChatFireworks extends ChatOpenAI { this.apiKey = fireworksApiKey; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = super.getLsParams(options); params.ls_provider = "fireworks"; return params; diff --git a/libs/langchain-community/src/chat_models/ollama.ts b/libs/langchain-community/src/chat_models/ollama.ts index 3fa78fa9c6b2..7c037864498a 100644 --- a/libs/langchain-community/src/chat_models/ollama.ts +++ b/libs/langchain-community/src/chat_models/ollama.ts @@ -177,7 +177,7 @@ export class ChatOllama this.format = fields.format; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "ollama", diff --git a/libs/langchain-community/src/chat_models/togetherai.ts b/libs/langchain-community/src/chat_models/togetherai.ts index 5488e6cf61fe..b377024406a1 100644 --- a/libs/langchain-community/src/chat_models/togetherai.ts +++ b/libs/langchain-community/src/chat_models/togetherai.ts @@ -116,7 +116,7 @@ export class ChatTogetherAI extends ChatOpenAI { }); } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = super.getLsParams(options); params.ls_provider = "together"; return params; diff --git a/libs/langchain-google-common/src/chat_models.ts b/libs/langchain-google-common/src/chat_models.ts index 1a27b3cd2cab..4f65b13a46b8 100644 --- a/libs/langchain-google-common/src/chat_models.ts +++ b/libs/langchain-google-common/src/chat_models.ts @@ -231,7 +231,7 @@ export abstract class ChatGoogleBase this.buildConnection(fields ?? {}, client); } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "google_vertexai", diff --git a/libs/langchain-google-genai/src/chat_models.ts b/libs/langchain-google-genai/src/chat_models.ts index 22b76ad5d4e2..73b54d22bfab 100644 --- a/libs/langchain-google-genai/src/chat_models.ts +++ b/libs/langchain-google-genai/src/chat_models.ts @@ -308,7 +308,7 @@ export class ChatGoogleGenerativeAI ); } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { return { ls_provider: "google_genai", ls_model_name: this.model, diff --git a/libs/langchain-mistralai/src/chat_models.ts b/libs/langchain-mistralai/src/chat_models.ts index 4762ddf75127..ab9d9bff360b 100644 --- a/libs/langchain-mistralai/src/chat_models.ts +++ b/libs/langchain-mistralai/src/chat_models.ts @@ -409,7 +409,7 @@ export class ChatMistralAI< this.model = this.modelName; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "mistral", diff --git a/libs/langchain-openai/package.json b/libs/langchain-openai/package.json index f841c46aa719..14b342daac48 100644 --- a/libs/langchain-openai/package.json +++ b/libs/langchain-openai/package.json @@ -16,26 +16,21 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/openai", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rimraf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -50,6 +45,7 @@ "@azure/identity": "^4.2.0", "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "dpdm": "^3.12.0", diff --git a/libs/langchain-openai/src/azure/chat_models.ts b/libs/langchain-openai/src/azure/chat_models.ts index 224a3e1ccf44..96c8b0ae769e 100644 --- a/libs/langchain-openai/src/azure/chat_models.ts +++ b/libs/langchain-openai/src/azure/chat_models.ts @@ -50,7 +50,7 @@ export class AzureChatOpenAI extends ChatOpenAI { super(newFields); } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = super.getLsParams(options); params.ls_provider = "azure"; return params; diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index f68fa5fdce72..78ace7e3d13c 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -491,7 +491,7 @@ export class ChatOpenAI< }; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "openai", diff --git a/libs/langchain-openai/src/tests/chat_models.standard.int.test.ts b/libs/langchain-openai/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..9c0c4e0d90d2 --- /dev/null +++ b/libs/langchain-openai/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,51 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatOpenAI, ChatOpenAICallOptions } from "../chat_models.js"; + +class ChatOpenAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatOpenAICallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.OPENAI_API_KEY) { + throw new Error( + "OPENAI_API_KEY must be set to run standard integration tests." + ); + } + super({ + Cls: ChatOpenAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "gpt-3.5-turbo", + }, + }); + } + + async testToolMessageHistoriesListContent() { + console.warn( + "ChatOpenAI testToolMessageHistoriesListContent test known failure. Skipping..." + ); + } + + async testUsageMetadataStreaming() { + // ChatOpenAI does not support streaming tokens by + // default, so we must pass in a call option to + // enable streaming tokens. + const callOptions: ChatOpenAI["ParsedCallOptions"] = { + stream_options: { + include_usage: true, + }, + }; + await super.testUsageMetadataStreaming(callOptions); + } +} + +const testClass = new ChatOpenAIStandardIntegrationTests(); + +test("ChatOpenAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-openai/src/tests/chat_models.standard.test.ts b/libs/langchain-openai/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..7b794cdccf1e --- /dev/null +++ b/libs/langchain-openai/src/tests/chat_models.standard.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatOpenAI, ChatOpenAICallOptions } from "../chat_models.js"; + +class ChatOpenAIStandardUnitTests extends ChatModelUnitTests< + ChatOpenAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatOpenAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.OPENAI_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.OPENAI_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.OPENAI_API_KEY = "test"; + } +} + +const testClass = new ChatOpenAIStandardUnitTests(); + +test("ChatOpenAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-standard-tests/.eslintrc.cjs b/libs/langchain-standard-tests/.eslintrc.cjs new file mode 100644 index 000000000000..51a16939ff9f --- /dev/null +++ b/libs/langchain-standard-tests/.eslintrc.cjs @@ -0,0 +1,67 @@ +module.exports = { + extends: [ + "airbnb-base", + "eslint:recommended", + "prettier", + "plugin:@typescript-eslint/recommended", + ], + parserOptions: { + ecmaVersion: 12, + parser: "@typescript-eslint/parser", + project: "./tsconfig.json", + sourceType: "module", + }, + plugins: ["@typescript-eslint", "no-instanceof"], + ignorePatterns: [ + ".eslintrc.cjs", + "scripts", + "node_modules", + "dist", + "dist-cjs", + "*.js", + "*.cjs", + "*.d.ts", + ], + rules: { + "no-process-env": 2, + "no-instanceof/no-instanceof": 2, + "@typescript-eslint/explicit-module-boundary-types": 0, + "@typescript-eslint/no-empty-function": 0, + "@typescript-eslint/no-shadow": 0, + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/no-explicit-any": "warn", + "@typescript-eslint/no-use-before-define": ["error", "nofunc"], + "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }], + "@typescript-eslint/no-floating-promises": "error", + "@typescript-eslint/no-misused-promises": "error", + camelcase: 0, + "class-methods-use-this": 0, + "import/extensions": [2, "ignorePackages"], + "import/no-extraneous-dependencies": [ + "error", + { devDependencies: ["**/*.test.ts"] }, + ], + "import/no-unresolved": 0, + "import/prefer-default-export": 0, + "keyword-spacing": "error", + "max-classes-per-file": 0, + "max-len": 0, + "no-await-in-loop": 0, + "no-bitwise": 0, + "no-console": 0, + "no-restricted-syntax": 0, + "no-shadow": 0, + "no-continue": 0, + "no-void": 0, + "no-underscore-dangle": 0, + "no-use-before-define": 0, + "no-useless-constructor": 0, + "no-return-await": 0, + "consistent-return": 0, + "no-else-return": 0, + "func-names": 0, + "no-lonely-if": 0, + "prefer-rest-params": 0, + "new-cap": ["error", { properties: false, capIsNew: false }], + }, +}; diff --git a/libs/langchain-standard-tests/.gitignore b/libs/langchain-standard-tests/.gitignore new file mode 100644 index 000000000000..c10034e2f1be --- /dev/null +++ b/libs/langchain-standard-tests/.gitignore @@ -0,0 +1,7 @@ +index.cjs +index.js +index.d.ts +index.d.cts +node_modules +dist +.yarn diff --git a/libs/langchain-standard-tests/.prettierrc b/libs/langchain-standard-tests/.prettierrc new file mode 100644 index 000000000000..ba08ff04f677 --- /dev/null +++ b/libs/langchain-standard-tests/.prettierrc @@ -0,0 +1,19 @@ +{ + "$schema": "https://json.schemastore.org/prettierrc", + "printWidth": 80, + "tabWidth": 2, + "useTabs": false, + "semi": true, + "singleQuote": false, + "quoteProps": "as-needed", + "jsxSingleQuote": false, + "trailingComma": "es5", + "bracketSpacing": true, + "arrowParens": "always", + "requirePragma": false, + "insertPragma": false, + "proseWrap": "preserve", + "htmlWhitespaceSensitivity": "css", + "vueIndentScriptAndStyle": false, + "endOfLine": "lf" +} diff --git a/libs/langchain-standard-tests/.release-it.json b/libs/langchain-standard-tests/.release-it.json new file mode 100644 index 000000000000..522ee6abf705 --- /dev/null +++ b/libs/langchain-standard-tests/.release-it.json @@ -0,0 +1,10 @@ +{ + "github": { + "release": true, + "autoGenerate": true, + "tokenRef": "GITHUB_TOKEN_RELEASE" + }, + "npm": { + "versionArgs": ["--workspaces-update=false"] + } +} diff --git a/libs/langchain-standard-tests/LICENSE b/libs/langchain-standard-tests/LICENSE new file mode 100644 index 000000000000..8cd8f501eb49 --- /dev/null +++ b/libs/langchain-standard-tests/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2023 LangChain + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/libs/langchain-standard-tests/README.md b/libs/langchain-standard-tests/README.md new file mode 100644 index 000000000000..5b2b678af672 --- /dev/null +++ b/libs/langchain-standard-tests/README.md @@ -0,0 +1,94 @@ +# LangChain.js Standard Tests + +This package contains the base standard tests for LangChain.js. It includes unit, and integration test classes. +This package is not intended to be used outside of the LangChain.js project, and thus it is not published to npm. + +At the moment, we only have support for standard tets for chat models. + +## Usage + +Each LangChain.js integration should contain both unit and integration standard tests. +The package should have `@langchain/standard-tests` as a dev workspace dependency like so: + +`package.json`: +```json +{ + "devDependencies": { + "@langchain/standard-tests": "workspace:*" + } +} +``` + +To use the standard tests, you could create two files: + +- `src/tests/chat_models.standard.test.ts` - chat model unit tests +- `src/tests/chat_models.standard.int.test.ts` - chat model integration tests + +Your unit test file should look like this: + +`chat_models.standard.test.ts`: +```typescript +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { MyChatModel, MyChatModelCallOptions } from "../chat_models.js"; + +class MyChatModelStandardUnitTests extends ChatModelUnitTests< + MyChatModelCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: MyChatModel, + chatModelHasToolCalling: true, // Set to true if the model has tool calling support + chatModelHasStructuredOutput: true, // Set to true if the model has withStructuredOutput support + constructorArgs: {}, // Any additional constructor args + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.CHAT_MODEL_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.CHAT_MODEL_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.CHAT_MODEL_API_KEY = "test"; + } +} + +const testClass = new MyChatModelStandardUnitTests(); + +test("MyChatModelStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); +``` + +To use the standard tests, extend the `ChatModelUnitTests` class, passing in your chat model's call options and message chunk types. Super the constructor with your chat model class, any additional constructor args, and set `chatModelHasToolCalling` and `chatModelHasStructuredOutput` flags if supported. + +Set the model env var in the constructor directly to `process.env` for the tests to run properly. You can optionally override test methods to replace or add code before/after the test runs. + +Run all tests by calling `.runTests()`, which returns `true` if all tests pass, `false` otherwise. Tests are called in `try`/`catch` blocks, so failing tests are caught and marked as failed, but the rest still run. + +For integration tests, extend `ChatModelIntegrationTests` instead. Integration tests have an optional arg for all methods (except `withStructuredOutput`) to pass in "invoke" time call options. For example, in the OpenAI integration test: + +```typescript +async testUsageMetadataStreaming() { + // ChatOpenAI does not support streaming tokens by + // default, so we must pass in a call option to + // enable streaming tokens. + const callOptions: ChatOpenAI["ParsedCallOptions"] = { + stream_options: { + include_usage: true, + }, + }; + await super.testUsageMetadataStreaming(callOptions); +} +``` + +This overrides the base `testUsageMetadataStreaming` to pass a `callOptions` arg enabling streaming tokens. diff --git a/libs/langchain-standard-tests/jest.config.cjs b/libs/langchain-standard-tests/jest.config.cjs new file mode 100644 index 000000000000..994826496bc5 --- /dev/null +++ b/libs/langchain-standard-tests/jest.config.cjs @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest/presets/default-esm", + testEnvironment: "./jest.env.cjs", + modulePathIgnorePatterns: ["dist/", "docs/"], + moduleNameMapper: { + "^(\\.{1,2}/.*)\\.js$": "$1", + }, + transform: { + "^.+\\.tsx?$": ["@swc/jest"], + }, + transformIgnorePatterns: [ + "/node_modules/", + "\\.pnp\\.[^\\/]+$", + "./scripts/jest-setup-after-env.js", + ], + setupFiles: ["dotenv/config"], + testTimeout: 20_000, + passWithNoTests: true, + collectCoverageFrom: ["src/**/*.ts"], +}; diff --git a/libs/langchain-standard-tests/jest.env.cjs b/libs/langchain-standard-tests/jest.env.cjs new file mode 100644 index 000000000000..2ccedccb8672 --- /dev/null +++ b/libs/langchain-standard-tests/jest.env.cjs @@ -0,0 +1,12 @@ +const { TestEnvironment } = require("jest-environment-node"); + +class AdjustedTestEnvironmentToSupportFloat32Array extends TestEnvironment { + constructor(config, context) { + // Make `instanceof Float32Array` return true in tests + // to avoid https://github.com/xenova/transformers.js/issues/57 and https://github.com/jestjs/jest/issues/2549 + super(config, context); + this.global.Float32Array = Float32Array; + } +} + +module.exports = AdjustedTestEnvironmentToSupportFloat32Array; diff --git a/libs/langchain-standard-tests/langchain.config.js b/libs/langchain-standard-tests/langchain.config.js new file mode 100644 index 000000000000..25a18707a316 --- /dev/null +++ b/libs/langchain-standard-tests/langchain.config.js @@ -0,0 +1,21 @@ +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * @param {string} relativePath + * @returns {string} + */ +function abs(relativePath) { + return resolve(dirname(fileURLToPath(import.meta.url)), relativePath); +} + +export const config = { + internals: [/node\:/, /@langchain\/core\//], + entrypoints: { + index: "index", + }, + tsConfigPath: resolve("./tsconfig.json"), + cjsSource: "./dist-cjs", + cjsDestination: "./dist", + abs, +}; diff --git a/libs/langchain-standard-tests/package.json b/libs/langchain-standard-tests/package.json new file mode 100644 index 000000000000..794199bf70be --- /dev/null +++ b/libs/langchain-standard-tests/package.json @@ -0,0 +1,84 @@ +{ + "name": "@langchain/standard-tests", + "version": "0.0.0", + "description": "Standard tests for LangChain.js", + "type": "module", + "engines": { + "node": ">=18" + }, + "main": "./index.js", + "types": "./index.d.ts", + "repository": { + "type": "git", + "url": "git@github.com:langchain-ai/langchainjs.git" + }, + "homepage": "https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-standard-tests/", + "scripts": { + "build": "yarn turbo:command build:internal --filter=@langchain/standard-tests", + "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", + "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", + "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", + "lint": "yarn lint:eslint && yarn lint:dpdm", + "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", + "clean": "rm -rf .turbo dist/", + "prepack": "yarn build", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "format": "prettier --config .prettierrc --write \"src\"", + "format:check": "prettier --config .prettierrc --check \"src\"" + }, + "author": "LangChain", + "license": "MIT", + "dependencies": { + "@jest/globals": "^29.5.0", + "@langchain/core": "workspace:*", + "zod": "^3.22.4" + }, + "devDependencies": { + "@langchain/scripts": "workspace:*", + "@swc/core": "^1.3.90", + "@swc/jest": "^0.2.29", + "@tsconfig/recommended": "^1.0.3", + "@typescript-eslint/eslint-plugin": "^6.12.0", + "@typescript-eslint/parser": "^6.12.0", + "dotenv": "^16.3.1", + "dpdm": "^3.12.0", + "eslint": "^8.33.0", + "eslint-config-airbnb-base": "^15.0.0", + "eslint-config-prettier": "^8.6.0", + "eslint-plugin-import": "^2.27.5", + "eslint-plugin-no-instanceof": "^1.0.1", + "eslint-plugin-prettier": "^4.2.1", + "jest": "^29.5.0", + "jest-environment-node": "^29.6.4", + "prettier": "^2.8.3", + "release-it": "^15.10.1", + "rollup": "^4.5.2", + "ts-jest": "^29.1.0", + "typescript": "^5.4.5" + }, + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": { + "import": "./index.d.ts", + "require": "./index.d.cts", + "default": "./index.d.ts" + }, + "import": "./index.js", + "require": "./index.cjs" + }, + "./package.json": "./package.json" + }, + "files": [ + "dist/", + "index.cjs", + "index.js", + "index.d.ts", + "index.d.cts" + ] +} diff --git a/libs/langchain-standard-tests/scripts/jest-setup-after-env.js b/libs/langchain-standard-tests/scripts/jest-setup-after-env.js new file mode 100644 index 000000000000..778cf7437a20 --- /dev/null +++ b/libs/langchain-standard-tests/scripts/jest-setup-after-env.js @@ -0,0 +1,3 @@ +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; + +afterAll(awaitAllCallbacks); diff --git a/libs/langchain-standard-tests/src/base.ts b/libs/langchain-standard-tests/src/base.ts new file mode 100644 index 000000000000..504391c2e3ae --- /dev/null +++ b/libs/langchain-standard-tests/src/base.ts @@ -0,0 +1,45 @@ +import { + BaseChatModel, + BaseChatModelCallOptions, +} from "@langchain/core/language_models/chat_models"; +import { BaseMessageChunk } from "@langchain/core/messages"; + +export type RecordStringAny = Record; + +export type BaseChatModelConstructor< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk +> = new (...args: any[]) => BaseChatModel; + +export type BaseChatModelsTestsFields< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk +> = { + Cls: BaseChatModelConstructor; + chatModelHasToolCalling: boolean; + chatModelHasStructuredOutput: boolean; + constructorArgs: RecordStringAny; +}; + +export class BaseChatModelsTests< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk +> implements BaseChatModelsTestsFields +{ + Cls: BaseChatModelConstructor; + + chatModelHasToolCalling: boolean; + + chatModelHasStructuredOutput: boolean; + + constructorArgs: RecordStringAny; + + constructor( + fields: BaseChatModelsTestsFields + ) { + this.Cls = fields.Cls; + this.chatModelHasToolCalling = fields.chatModelHasToolCalling; + this.chatModelHasStructuredOutput = fields.chatModelHasStructuredOutput; + this.constructorArgs = fields.constructorArgs; + } +} diff --git a/libs/langchain-standard-tests/src/index.ts b/libs/langchain-standard-tests/src/index.ts new file mode 100644 index 000000000000..57ff0dc1b045 --- /dev/null +++ b/libs/langchain-standard-tests/src/index.ts @@ -0,0 +1,2 @@ +export * from "./unit_tests/chat_models.js"; +export * from "./integration_tests/chat_models.js"; diff --git a/libs/langchain-standard-tests/src/integration_tests/chat_models.ts b/libs/langchain-standard-tests/src/integration_tests/chat_models.ts new file mode 100644 index 000000000000..11c1f9183805 --- /dev/null +++ b/libs/langchain-standard-tests/src/integration_tests/chat_models.ts @@ -0,0 +1,437 @@ +import { expect } from "@jest/globals"; +import { BaseChatModelCallOptions } from "@langchain/core/language_models/chat_models"; +import { + AIMessage, + AIMessageChunk, + BaseMessageChunk, + HumanMessage, + ToolMessage, + UsageMetadata, +} from "@langchain/core/messages"; +import { z } from "zod"; +import { StructuredTool } from "@langchain/core/tools"; +import { BaseChatModelsTests, BaseChatModelsTestsFields } from "../base.js"; + +const adderSchema = /* #__PURE__ */ z + .object({ + a: z.number().int().describe("The first integer to add."), + b: z.number().int().describe("The second integer to add."), + }) + .describe("Add two integers"); + +class AdderTool extends StructuredTool { + name = "AdderTool"; + + description = adderSchema.description ?? "description"; + + schema = adderSchema; + + async _call(input: z.infer) { + const sum = input.a + input.b; + return JSON.stringify({ result: sum }); + } +} + +export abstract class ChatModelIntegrationTests< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk +> extends BaseChatModelsTests { + constructor( + fields: BaseChatModelsTestsFields + ) { + super(fields); + } + + async testInvoke( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + const result = await chatModel.invoke("Hello", callOptions); + expect(result).toBeDefined(); + expect(result).toBeInstanceOf(AIMessage); + expect(typeof result.content).toBe("string"); + expect(result.content.length).toBeGreaterThan(0); + } + + async testStream( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + let numChars = 0; + + for await (const token of await chatModel.stream("Hello", callOptions)) { + expect(token).toBeDefined(); + expect(token).toBeInstanceOf(AIMessageChunk); + expect(typeof token.content).toBe("string"); + numChars += token.content.length; + } + + expect(numChars).toBeGreaterThan(0); + } + + async testBatch( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + const batchResults = await chatModel.batch(["Hello", "Hey"], callOptions); + expect(batchResults).toBeDefined(); + expect(Array.isArray(batchResults)).toBe(true); + expect(batchResults.length).toBe(2); + for (const result of batchResults) { + expect(result).toBeDefined(); + expect(result).toBeInstanceOf(AIMessage); + expect(typeof result.content).toBe("string"); + expect(result.content.length).toBeGreaterThan(0); + } + } + + async testConversation( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + const messages = [ + new HumanMessage("hello"), + new AIMessage("hello"), + new HumanMessage("how are you"), + ]; + const result = await chatModel.invoke(messages, callOptions); + expect(result).toBeDefined(); + expect(result).toBeInstanceOf(AIMessage); + expect(typeof result.content).toBe("string"); + expect(result.content.length).toBeGreaterThan(0); + } + + async testUsageMetadata( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + const result = await chatModel.invoke("Hello", callOptions); + expect(result).toBeDefined(); + expect(result).toBeInstanceOf(AIMessage); + if (!("usage_metadata" in result)) { + throw new Error("result is not an instance of AIMessage"); + } + const usageMetadata = result.usage_metadata as UsageMetadata; + expect(usageMetadata).toBeDefined(); + expect(typeof usageMetadata.input_tokens).toBe("number"); + expect(typeof usageMetadata.output_tokens).toBe("number"); + expect(typeof usageMetadata.total_tokens).toBe("number"); + } + + async testUsageMetadataStreaming( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + let finalChunks: AIMessageChunk | undefined; + for await (const chunk of await chatModel.stream("Hello", callOptions)) { + expect(chunk).toBeDefined(); + expect(chunk).toBeInstanceOf(AIMessageChunk); + if (!finalChunks) { + finalChunks = chunk; + } else { + finalChunks = finalChunks.concat(chunk); + } + } + if (!finalChunks) { + throw new Error("finalChunks is undefined"); + } + const usageMetadata = finalChunks.usage_metadata; + expect(usageMetadata).toBeDefined(); + if (!usageMetadata) { + throw new Error("usageMetadata is undefined"); + } + expect(typeof usageMetadata.input_tokens).toBe("number"); + expect(typeof usageMetadata.output_tokens).toBe("number"); + expect(typeof usageMetadata.total_tokens).toBe("number"); + } + + /** + * Test that message histories are compatible with string tool contents + * (e.g. OpenAI). + * @returns {Promise} + */ + async testToolMessageHistoriesStringContent( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + if (!this.chatModelHasToolCalling) { + console.log("Test requires tool calling. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + const adderTool = new AdderTool(); + if (!model.bindTools) { + throw new Error( + "bindTools undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.bindTools([adderTool]); + const functionName = adderTool.name; + const functionArgs = { a: 1, b: 2 }; + + const functionId = "abc123"; + const functionResult = await adderTool.invoke(functionArgs); + + const messagesStringContent = [ + new HumanMessage("What is 1 + 2"), + // string content (e.g. OpenAI) + new AIMessage({ + content: "", + tool_calls: [ + { + name: functionName, + args: functionArgs, + id: functionId, + }, + ], + }), + new ToolMessage(functionResult, functionId, functionName), + ]; + + const resultStringContent = await modelWithTools.invoke( + messagesStringContent, + callOptions + ); + expect(resultStringContent).toBeInstanceOf(AIMessage); + } + + /** + * Test that message histories are compatible with list tool contents + * (e.g. Anthropic). + * @returns {Promise} + */ + async testToolMessageHistoriesListContent( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + if (!this.chatModelHasToolCalling) { + console.log("Test requires tool calling. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + const adderTool = new AdderTool(); + if (!model.bindTools) { + throw new Error( + "bindTools undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.bindTools([adderTool]); + const functionName = adderTool.name; + const functionArgs = { a: 1, b: 2 }; + + const functionId = "abc123"; + const functionResult = await adderTool.invoke(functionArgs); + + const messagesListContent = [ + new HumanMessage("What is 1 + 2"), + // List content (e.g., Anthropic) + new AIMessage({ + content: [ + { type: "text", text: "some text" }, + { + type: "tool_use", + id: functionId, + name: functionName, + input: functionArgs, + }, + ], + tool_calls: [ + { + name: functionName, + args: functionArgs, + id: functionId, + }, + ], + }), + new ToolMessage(functionResult, functionId, functionName), + ]; + + const resultListContent = await modelWithTools.invoke( + messagesListContent, + callOptions + ); + expect(resultListContent).toBeInstanceOf(AIMessage); + } + + /** + * Test that model can process few-shot examples with tool calls. + * @returns {Promise} + */ + async testStructuredFewShotExamples( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + if (!this.chatModelHasToolCalling) { + console.log("Test requires tool calling. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + const adderTool = new AdderTool(); + if (!model.bindTools) { + throw new Error("bindTools undefined. Cannot test few-shot examples."); + } + const modelWithTools = model.bindTools([adderTool]); + const functionName = adderTool.name; + const functionArgs = { a: 1, b: 2 }; + + const functionId = "abc123"; + const functionResult = await adderTool.invoke(functionArgs); + + const messagesStringContent = [ + new HumanMessage("What is 1 + 2"), + new AIMessage({ + content: "", + tool_calls: [ + { + name: functionName, + args: functionArgs, + id: functionId, + }, + ], + }), + new ToolMessage(functionResult, functionId, functionName), + new AIMessage(functionResult), + new HumanMessage("What is 3 + 4"), + ]; + + const resultStringContent = await modelWithTools.invoke( + messagesStringContent, + callOptions + ); + expect(resultStringContent).toBeInstanceOf(AIMessage); + } + + async testWithStructuredOutput() { + if (!this.chatModelHasStructuredOutput) { + console.log("Test requires withStructuredOutput. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + if (!model.withStructuredOutput) { + throw new Error( + "withStructuredOutput undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.withStructuredOutput(adderSchema); + + const resultStringContent = await modelWithTools.invoke("What is 1 + 2"); + expect(resultStringContent.a).toBeDefined(); + expect([1, 2].includes(resultStringContent.a)).toBeTruthy(); + expect(resultStringContent.b).toBeDefined(); + expect([1, 2].includes(resultStringContent.b)).toBeTruthy(); + } + + async testWithStructuredOutputIncludeRaw() { + if (!this.chatModelHasStructuredOutput) { + console.log("Test requires withStructuredOutput. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + if (!model.withStructuredOutput) { + throw new Error( + "withStructuredOutput undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.withStructuredOutput(adderSchema, { + includeRaw: true, + }); + + const resultStringContent = await modelWithTools.invoke("What is 1 + 2"); + expect(resultStringContent.raw).toBeInstanceOf(AIMessage); + expect(resultStringContent.parsed.a).toBeDefined(); + expect([1, 2].includes(resultStringContent.parsed.a)).toBeTruthy(); + expect(resultStringContent.parsed.b).toBeDefined(); + expect([1, 2].includes(resultStringContent.parsed.b)).toBeTruthy(); + } + + /** + * Run all unit tests for the chat model. + * Each test is wrapped in a try/catch block to prevent the entire test suite from failing. + * If a test fails, the error is logged to the console, and the test suite continues. + * @returns {boolean} + */ + async runTests(): Promise { + let allTestsPassed = true; + + try { + await this.testInvoke(); + } catch (e: any) { + allTestsPassed = false; + console.error("testInvoke failed", e); + } + + try { + await this.testStream(); + } catch (e: any) { + allTestsPassed = false; + console.error("testStream failed", e); + } + + try { + await this.testBatch(); + } catch (e: any) { + allTestsPassed = false; + console.error("testBatch failed", e); + } + + try { + await this.testConversation(); + } catch (e: any) { + allTestsPassed = false; + console.error("testConversation failed", e); + } + + try { + await this.testUsageMetadata(); + } catch (e: any) { + allTestsPassed = false; + console.error("testUsageMetadata failed", e); + } + + try { + await this.testUsageMetadataStreaming(); + } catch (e: any) { + allTestsPassed = false; + console.error("testUsageMetadataStreaming failed", e); + } + + try { + await this.testToolMessageHistoriesStringContent(); + } catch (e: any) { + allTestsPassed = false; + console.error("testToolMessageHistoriesStringContent failed", e); + } + + try { + await this.testToolMessageHistoriesListContent(); + } catch (e: any) { + allTestsPassed = false; + console.error("testToolMessageHistoriesListContent failed", e); + } + + try { + await this.testStructuredFewShotExamples(); + } catch (e: any) { + allTestsPassed = false; + console.error("testStructuredFewShotExamples failed", e); + } + + try { + await this.testWithStructuredOutput(); + } catch (e: any) { + allTestsPassed = false; + console.error("testWithStructuredOutput failed", e); + } + + try { + await this.testWithStructuredOutputIncludeRaw(); + } catch (e: any) { + allTestsPassed = false; + console.error("testWithStructuredOutputIncludeRaw failed", e); + } + + return allTestsPassed; + } +} diff --git a/libs/langchain-standard-tests/src/unit_tests/chat_models.ts b/libs/langchain-standard-tests/src/unit_tests/chat_models.ts new file mode 100644 index 000000000000..74fafe0813e9 --- /dev/null +++ b/libs/langchain-standard-tests/src/unit_tests/chat_models.ts @@ -0,0 +1,160 @@ +import { expect } from "@jest/globals"; +import { + BaseChatModelCallOptions, + LangSmithParams, +} from "@langchain/core/language_models/chat_models"; +import { BaseMessageChunk } from "@langchain/core/messages"; +import { z } from "zod"; +import { StructuredTool } from "@langchain/core/tools"; +import { + BaseChatModelsTests, + BaseChatModelsTestsFields, + RecordStringAny, +} from "../base.js"; + +const person = /* #__PURE__ */ z + .object({ + name: z.string().describe("Name of the person"), + age: z.number().int().positive().describe("Age of the person"), + }) + .describe("A person"); + +class PersonTool extends StructuredTool { + name = "PersonTool"; + + description = person.description ?? "description"; + + schema = person; + + async _call(input: z.infer) { + return JSON.stringify(input); + } +} + +export abstract class ChatModelUnitTests< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk +> extends BaseChatModelsTests { + constructor( + fields: BaseChatModelsTestsFields + ) { + const standardChatModelParams: RecordStringAny = { + temperature: 0, + maxTokens: 100, + timeout: 60, + stopSequences: [], + maxRetries: 2, + }; + super({ + ...fields, + constructorArgs: { + ...standardChatModelParams, + ...fields.constructorArgs, + }, + }); + } + + testChatModelInit() { + const chatModel = new this.Cls(this.constructorArgs); + expect(chatModel).toBeDefined(); + } + + testChatModelInitApiKey() { + const params = { ...this.constructorArgs, apiKey: "test" }; + const chatModel = new this.Cls(params); + expect(chatModel).toBeDefined(); + } + + testChatModelInitStreaming() { + const params = { ...this.constructorArgs, streaming: true }; + const chatModel = new this.Cls(params); + expect(chatModel).toBeDefined(); + } + + testChatModelWithBindTools() { + if (!this.chatModelHasToolCalling) { + return; + } + const chatModel = new this.Cls(this.constructorArgs); + expect(chatModel.bindTools?.([new PersonTool()])).toBeDefined(); + } + + testChatModelWithStructuredOutput() { + if (!this.chatModelHasStructuredOutput) { + return; + } + const chatModel = new this.Cls(this.constructorArgs); + expect((chatModel as any).withStructuredOutput?.(person)).toBeDefined(); + } + + testStandardParams() { + const expectedParams: LangSmithParams = { + ls_provider: "string", + ls_model_name: "string", + ls_model_type: "chat", + ls_temperature: 0, + ls_max_tokens: 0, + ls_stop: ["Array"], + }; + const chatModel = new this.Cls(this.constructorArgs); + + const lsParams = chatModel.getLsParams({} as any); + expect(lsParams).toBeDefined(); + expect(Object.keys(lsParams).sort()).toEqual( + Object.keys(expectedParams).sort() + ); + } + + /** + * Run all unit tests for the chat model. + * Each test is wrapped in a try/catch block to prevent the entire test suite from failing. + * If a test fails, the error is logged to the console, and the test suite continues. + * @returns {boolean} + */ + runTests(): boolean { + let allTestsPassed = true; + try { + this.testChatModelInit(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelInit failed", e); + } + + try { + this.testChatModelInitApiKey(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelInitApiKey failed", e); + } + + try { + this.testChatModelInitStreaming(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelInitStreaming failed", e); + } + + try { + this.testChatModelWithBindTools(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelWithBindTools failed", e); + } + + try { + this.testChatModelWithStructuredOutput(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelWithStructuredOutput failed", e); + } + + try { + this.testStandardParams(); + } catch (e: any) { + allTestsPassed = false; + console.error("testStandardParams failed", e); + } + + return allTestsPassed; + } +} diff --git a/libs/langchain-standard-tests/tsconfig.cjs.json b/libs/langchain-standard-tests/tsconfig.cjs.json new file mode 100644 index 000000000000..510d791e34ab --- /dev/null +++ b/libs/langchain-standard-tests/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false + }, + "exclude": ["node_modules", "dist", "docs", "**/tests"] +} diff --git a/libs/langchain-standard-tests/tsconfig.json b/libs/langchain-standard-tests/tsconfig.json new file mode 100644 index 000000000000..446862d6ffc6 --- /dev/null +++ b/libs/langchain-standard-tests/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "@tsconfig/recommended", + "compilerOptions": { + "outDir": "../dist", + "rootDir": "./src", + "target": "ES2021", + "lib": ["ES2021", "ES2022.Object", "DOM"], + "module": "nodenext", + "moduleResolution": "nodenext", + "esModuleInterop": true, + "declaration": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "useDefineForClassFields": true, + "strictPropertyInitialization": false, + "allowJs": true, + "strict": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "docs"] +} diff --git a/libs/langchain-standard-tests/turbo.json b/libs/langchain-standard-tests/turbo.json new file mode 100644 index 000000000000..d024cee15c81 --- /dev/null +++ b/libs/langchain-standard-tests/turbo.json @@ -0,0 +1,11 @@ +{ + "extends": ["//"], + "pipeline": { + "build": { + "outputs": ["**/dist/**"] + }, + "build:internal": { + "dependsOn": ["^build:internal"] + } + } +} diff --git a/package.json b/package.json index 82c1d5abb96b..3aaa400da2d4 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,9 @@ "test:int:deps:down": "docker compose -f test-int-deps-docker-compose.yml down", "test:ranges:docker": "docker compose -f dependency_range_tests/docker-compose.yml up --force-recreate", "test:exports:docker": "docker compose -f environment_tests/docker-compose.yml up --force-recreate", + "test:standard:unit": "turbo test:standard:unit", + "test:standard:int": "turbo test:standard:int", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "example": "yarn workspace examples start", "precommit": "turbo precommit", "docs": "yarn workspace core_docs start", diff --git a/turbo.json b/turbo.json index 0f82b8628676..f69adb6cf3dc 100644 --- a/turbo.json +++ b/turbo.json @@ -32,6 +32,18 @@ "test:integration": { "dependsOn": ["^build", "build"] }, + "test:standard:unit": { + "outputs": [], + "dependsOn": ["^build"] + }, + "test:standard:int": { + "outputs": [], + "dependsOn": ["^build"] + }, + "test:standard": { + "outputs": [], + "dependsOn": ["^build"] + }, "clean": { "dependsOn": ["^clean"] }, diff --git a/yarn.lock b/yarn.lock index 3ab88496cd0e..b1e6fb935076 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10043,6 +10043,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.5 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 dpdm: ^3.12.0 @@ -10206,6 +10207,37 @@ __metadata: languageName: unknown linkType: soft +"@langchain/standard-tests@workspace:*, @langchain/standard-tests@workspace:libs/langchain-standard-tests": + version: 0.0.0-use.local + resolution: "@langchain/standard-tests@workspace:libs/langchain-standard-tests" + dependencies: + "@jest/globals": ^29.5.0 + "@langchain/core": "workspace:*" + "@langchain/scripts": "workspace:*" + "@swc/core": ^1.3.90 + "@swc/jest": ^0.2.29 + "@tsconfig/recommended": ^1.0.3 + "@typescript-eslint/eslint-plugin": ^6.12.0 + "@typescript-eslint/parser": ^6.12.0 + dotenv: ^16.3.1 + dpdm: ^3.12.0 + eslint: ^8.33.0 + eslint-config-airbnb-base: ^15.0.0 + eslint-config-prettier: ^8.6.0 + eslint-plugin-import: ^2.27.5 + eslint-plugin-no-instanceof: ^1.0.1 + eslint-plugin-prettier: ^4.2.1 + jest: ^29.5.0 + jest-environment-node: ^29.6.4 + prettier: ^2.8.3 + release-it: ^15.10.1 + rollup: ^4.5.2 + ts-jest: ^29.1.0 + typescript: ^5.4.5 + zod: ^3.22.4 + languageName: unknown + linkType: soft + "@langchain/textsplitters@workspace:*, @langchain/textsplitters@workspace:libs/langchain-textsplitters, @langchain/textsplitters@~0.0.0": version: 0.0.0-use.local resolution: "@langchain/textsplitters@workspace:libs/langchain-textsplitters"