core[minor]: Add ID field to document (#5893)

* core[minor]: Add ID field to document

* add more docstring

* chore: lint files

* update typeorm id document field

* fix tests

Author: bracesproul

langchain-core/src/documents/document.ts

@@ -5,6 +5,14 @@ export interface DocumentInput<
   pageContent: string;
 
   metadata?: Metadata;
+
+  /**
+   * An optional identifier for the document.
+   *
+   * Ideally this should be unique across the document collection and formatted
+   * as a UUID, but this will not be enforced.
+   */
+  id?: string;
 }
 
 export interface DocumentInterface<
@@ -14,6 +22,14 @@ export interface DocumentInterface<
   pageContent: string;
 
   metadata: Metadata;
+
+  /**
+   * An optional identifier for the document.
+   *
+   * Ideally this should be unique across the document collection and formatted
+   * as a UUID, but this will not be enforced.
+   */
+  id?: string;
 }
 
 /**
@@ -28,9 +44,21 @@ export class Document<
 
   metadata: Metadata;
 
+  // The ID field is optional at the moment.
+  // It will likely become required in a future major release after
+  // it has been adopted by enough vectorstore implementations.
+  /**
+   * An optional identifier for the document.
+   *
+   * Ideally this should be unique across the document collection and formatted
+   * as a UUID, but this will not be enforced.
+   */
+  id?: string;
+
   constructor(fields: DocumentInput<Metadata>) {
     this.pageContent =
       fields.pageContent !== undefined ? fields.pageContent.toString() : "";
     this.metadata = fields.metadata ?? ({} as Metadata);
+    this.id = fields.id;
   }
 }

langchain/src/document_loaders/tests/csv-blob.test.ts

@@ -46,6 +46,7 @@ test("Test CSV loader from blob", async () => {
   expect(docs.length).toBe(2);
   expect(docs[0]).toMatchInlineSnapshot(`
     Document {
+      "id": undefined,
       "metadata": {
         "blobType": "text/csv",
         "line": 1,
@@ -57,6 +58,7 @@ test("Test CSV loader from blob", async () => {
   `);
   expect(docs[1]).toMatchInlineSnapshot(`
     Document {
+      "id": undefined,
       "metadata": {
         "blobType": "text/csv",
         "line": 2,

langchain/src/document_loaders/tests/json-blob.test.ts

@@ -39,6 +39,7 @@ test("Test JSON loader from blob", async () => {
   expect(docs.length).toBe(2);
   expect(docs[0]).toMatchInlineSnapshot(`
     Document {
+      "id": undefined,
       "metadata": {
         "blobType": "application/json",
         "line": 1,
@@ -49,6 +50,7 @@ test("Test JSON loader from blob", async () => {
   `);
   expect(docs[1]).toMatchInlineSnapshot(`
     Document {
+      "id": undefined,
       "metadata": {
         "blobType": "application/json",
         "line": 2,
@@ -87,6 +89,7 @@ test("Test JSON loader from blob", async () => {
   expect(docs.length).toBe(10);
   expect(docs[0]).toMatchInlineSnapshot(`
     Document {
+      "id": undefined,
       "metadata": {
         "blobType": "application/json",
         "line": 1,
@@ -97,6 +100,7 @@ test("Test JSON loader from blob", async () => {
   `);
   expect(docs[1]).toMatchInlineSnapshot(`
     Document {
+      "id": undefined,
       "metadata": {
         "blobType": "application/json",
         "line": 2,

langchain/src/document_loaders/tests/jsonl-blob.test.ts

@@ -40,6 +40,7 @@ test("Test JSONL loader from blob", async () => {
   expect(docs.length).toBe(2);
   expect(docs[0]).toMatchInlineSnapshot(`
     Document {
+      "id": undefined,
       "metadata": {
         "blobType": "application/jsonl+json",
         "line": 1,
@@ -50,6 +51,7 @@ test("Test JSONL loader from blob", async () => {
   `);
   expect(docs[1]).toMatchInlineSnapshot(`
     Document {
+      "id": undefined,
       "metadata": {
         "blobType": "application/jsonl+json",
         "line": 2,

libs/langchain-community/src/vectorstores/typeorm.ts

@@ -23,8 +23,6 @@ export interface TypeORMVectorStoreArgs {
  */
 export class TypeORMVectorStoreDocument extends Document {
   embedding: string;
-
-  id?: string;
 }
 
 const defaultDocumentTableName = "documents";