feat: support node path and file object\ and browser file object

Author: amirai21

examples/studio/conversational-rag/rag-engine.ts

@@ -1,5 +1,5 @@
 import { AI21 } from 'ai21';
-import { FileResponse, UploadFileResponse } from '../../../src/types/rag';
+import { FilePathOrFileObject, FileResponse, UploadFileResponse } from '../../../src/types/rag';
 
 async function waitForFileProcessing(
   client: AI21,
@@ -18,21 +18,14 @@ async function waitForFileProcessing(
   }
 }
 
-async function uploadQueryUpdateDelete() {
+async function uploadQueryUpdateDelete(fileInput) {
   const client = new AI21({ apiKey: process.env.AI21_API_KEY });
   try {
     const uploadFileResponse: UploadFileResponse = await client.ragEngine.create(
-      '/Users/amirkoblyansky/Documents/ukraine.txt',
+      fileInput,
       { path: 'test10' },
     );
 
-    // const fileContent = Buffer.from('This is the content of the file.');
-    // const dummyFile = new File([fileContent], 'example.txt', { type: 'text/plain' });
-
-    // // Use the File object in the create method
-    // const uploadFileResponse: UploadFileResponse = await client.ragEngine.create(dummyFile, {
-    //   path: 'test10',
-    // });
 
     const fileId = uploadFileResponse.fileId;
     let file: FileResponse = await waitForFileProcessing(client, fileId);
@@ -59,6 +52,10 @@ async function listFiles() {
   console.log(files);
 }
 
-uploadQueryUpdateDelete().catch(console.error);
+const filePath = '/Users/amirkoblyansky/Documents/ukraine.txt'
+const fileContent = Buffer.from('This is the content of the file.');
+const dummyFile = new File([fileContent], 'example.txt', { type: 'text/plain' });
+
+uploadQueryUpdateDelete(dummyFile).catch(console.error);
 
-listFiles().catch(console.error);
+// listFiles().catch(console.error);

src/APIClient.ts

@@ -15,7 +15,8 @@ import { Fetch } from 'fetch';
 import { createReadStream } from 'fs';
 import { basename as getBasename } from 'path';
 import FormData from 'form-data';
-import { FilePathOrFileObject } from 'types/rag';
+import { FilePathOrFileObject } from './types/rag';
+import { createFormData, getBoundary, appendBodyToFormData } from './files/form-utils';
 
 const validatePositiveInteger = (name: string, n: unknown): number => {
   if (typeof n !== 'number' || !Number.isInteger(n)) {
@@ -27,33 +28,6 @@ const validatePositiveInteger = (name: string, n: unknown): number => {
   return n;
 };
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const appendBodyToFormData = (formData: FormData, body: Record<string, any>): void => {
-  for (const [key, value] of Object.entries(body)) {
-    if (Array.isArray(value)) {
-      value.forEach((item) => formData.append(key, item));
-    } else {
-      formData.append(key, value);
-    }
-  }
-};
-
-
-function makeFormDataFromFilePath(filePath: string): FormData {
-  const formData = new FormData();
-  const fileStream = createReadStream(filePath);
-  const fileName = getBasename(filePath);
-
-  formData.append('file', fileStream, fileName);
-  return formData;
-}
-
-function makeFormDataFromFileObject(file: File): FormData {
-  const formData = new FormData();
-  formData.append('file', file);
-  return formData;
-}
-
 export abstract class APIClient {
   protected baseURL: string;
   protected maxRetries: number;
@@ -92,16 +66,8 @@ export abstract class APIClient {
     return this.makeRequest('delete', path, opts);
   }
 
-  upload<Req, Rsp>(path: string, file: FilePathOrFileObject, opts?: RequestOptions<Req>): Promise<Rsp> {
-    let formData: FormData;
-
-    if (typeof file === 'string') {
-      formData = makeFormDataFromFilePath(file);
-    } else if (file instanceof File) {
-      formData = makeFormDataFromFileObject(file);
-    } else {
-      throw new AI21Error('Invalid file type for upload');
-    }
+  async upload<Req, Rsp>(path: string, file: FilePathOrFileObject, opts?: RequestOptions<Req>): Promise<Rsp> {
+    const formData = await createFormData(file);
 
     if (opts?.body) {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -110,7 +76,7 @@ export abstract class APIClient {
 
     const headers = {
       ...opts?.headers,
-      'Content-Type': `multipart/form-data; boundary=${formData.getBoundary()}`,
+      'Content-Type': `multipart/form-data; boundary=${await getBoundary(formData)}`,
     };
 
     const options: FinalRequestOptions = {
@@ -224,3 +190,4 @@ export abstract class APIClient {
     );
   }
 }
+

src/files/form-utils.ts

@@ -0,0 +1,77 @@
+import { FilePathOrFileObject } from 'types/rag';
+import * as Runtime from '../runtime';
+
+export type UnifiedFormData = FormData | import('form-data');
+
+import { Readable } from 'stream';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const appendBodyToFormData = (formData: UnifiedFormData, body: Record<string, any>): void => {
+    for (const [key, value] of Object.entries(body)) {
+      if (Array.isArray(value)) {
+        value.forEach((item) => formData.append(key, item));
+      } else {
+        formData.append(key, value);
+      }
+    }
+  };
+
+// Convert WHATWG ReadableStream to Node.js Readable Stream
+function convertReadableStream(whatwgStream: ReadableStream): Readable {
+    const reader = whatwgStream.getReader();
+
+    return new Readable({
+        async read() {
+            const { done, value } = await reader.read();
+            if (done) {
+                this.push(null);
+            } else {
+                this.push(value);
+            }
+        },
+    });
+}
+
+
+export async function createFormData(file: FilePathOrFileObject): Promise<UnifiedFormData> {
+
+    if (Runtime.isBrowser) {
+        const formData = new FormData();
+
+        if (file instanceof window.File) {
+            formData.append('file', file);
+        } else {
+            throw new Error('Unsupported file type in browser');
+        }
+ 
+        return formData;
+    } else { // Node environment:
+
+        const { default: FormDataNode } = await import('form-data');
+        const formData = new FormDataNode();
+
+        if (typeof file === 'string') {
+            const { createReadStream } = await import('fs');
+            formData.append('file', createReadStream(file), { filename: file.split('/').pop() });
+        } else if (Buffer.isBuffer(file)) {
+            formData.append('file', file, { filename: 'TODO - add filename to buffer flow' });
+        } else if (file instanceof File) {
+            const nodeStream = convertReadableStream(file.stream());
+            formData.append('file', nodeStream, file.name);
+        } else {
+            throw new Error('Unsupported file type in Node.js');
+        }
+
+        return formData;
+    }
+}
+
+export async function getBoundary(formData: UnifiedFormData): Promise<string | undefined> {
+    if (!Runtime.isBrowser) {
+        const { default: FormDataNode } = await import('form-data');
+        if (formData instanceof FormDataNode) {
+            return formData.getBoundary();
+        }
+    }
+    return undefined;
+}

vite.config.ts

@@ -10,7 +10,7 @@ export default defineConfig({
       formats: ['es', 'cjs', 'umd'],
     },
     rollupOptions: {
-      external: ['node-fetch'],
+      external: ['node-fetch', 'fs', 'path', 'stream', 'form-data'],
       output: {
         globals: {
           'node-fetch': 'fetch',