Skip to content

Commit 02b40df

Browse files
amirai21asafgardin
amirai21
authored and
asafgardin
committed
feat: functioning browser upload
1 parent c055ca2 commit 02b40df

File tree

9 files changed

+73
-79
lines changed

9 files changed

+73
-79
lines changed

‎examples/studio/conversational-rag/rag-engine.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@ async function waitForFileProcessing(
1818
}
1919
}
2020

21-
async function uploadQueryUpdateDelete(fileInput) {
21+
async function uploadQueryUpdateDelete(fileInput, label) {
2222
const client = new AI21({ apiKey: process.env.AI21_API_KEY });
2323
try {
2424
const uploadFileResponse: UploadFileResponse = await client.ragEngine.create(fileInput, {
25-
path: 'test10',
25+
path: label,
2626
});
2727

2828
const fileId = uploadFileResponse.fileId;
@@ -54,7 +54,7 @@ const filePath = '/Users/amirkoblyansky/Documents/ukraine.txt';
5454
const fileContent = Buffer.from('This is the content of the file.');
5555
const dummyFile = new File([fileContent], 'example.txt', { type: 'text/plain' });
5656

57-
uploadQueryUpdateDelete(filePath).catch(console.error);
58-
uploadQueryUpdateDelete(dummyFile).catch(console.error);
57+
uploadQueryUpdateDelete(filePath, "abc123").catch(console.error);
58+
uploadQueryUpdateDelete(dummyFile, "test2").catch(console.error);
5959

6060
listFiles().catch(console.error);

‎src/APIClient.ts

Lines changed: 19 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,12 @@ import {
88
HTTPMethod,
99
Headers,
1010
CrossPlatformResponse,
11+
UnifiedFormData,
1112
} from './types';
1213
import { AI21EnvConfig } from './EnvConfig';
1314
import { createFetchInstance } from './runtime';
1415
import { Fetch } from 'fetch';
15-
import { createReadStream } from 'fs';
16-
import { basename as getBasename } from 'path';
17-
import FormData from 'form-data';
18-
import { FilePathOrFileObject } from './types/rag';
19-
import { createFormData, getBoundary, appendBodyToFormData } from './files/form-utils';
16+
import { getBoundary, appendBodyToFormData } from './files/form-utils';
2017

2118
const validatePositiveInteger = (name: string, n: unknown): number => {
2219
if (typeof n !== 'number' || !Number.isInteger(n)) {
@@ -66,17 +63,16 @@ export abstract class APIClient {
6663
return this.makeRequest('delete', path, opts);
6764
}
6865

69-
async upload<Req, Rsp>(path: string, file: FilePathOrFileObject, opts?: RequestOptions<Req>): Promise<Rsp> {
70-
const formData = await createFormData(file);
71-
66+
async upload<Req, Rsp>(path: string, formData: UnifiedFormData, opts?: RequestOptions<Req>): Promise<Rsp> {
7267
if (opts?.body) {
7368
// eslint-disable-next-line @typescript-eslint/no-explicit-any
7469
appendBodyToFormData(formData, opts.body as Record<string, any>);
7570
}
7671

72+
const boundary = await getBoundary(formData);
7773
const headers = {
7874
...opts?.headers,
79-
'Content-Type': `multipart/form-data; boundary=${await getBoundary(formData)}`,
75+
'Content-Type': `multipart/form-data; boundary=${boundary}`,
8076
};
8177

8278
const options: FinalRequestOptions = {
@@ -89,43 +85,6 @@ export abstract class APIClient {
8985
return this.performRequest(options).then((response) => this.fetch.handleResponse<Rsp>(response) as Rsp);
9086
}
9187

92-
protected makeFormDataRequest<Req>(
93-
path: string,
94-
filePath: string,
95-
opts?: RequestOptions<Req>,
96-
): FinalRequestOptions {
97-
const formData = new FormData();
98-
const fileStream = createReadStream(filePath);
99-
const fileName = getBasename(filePath);
100-
101-
formData.append('file', fileStream, fileName);
102-
103-
if (opts?.body) {
104-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
105-
const body = opts.body as Record<string, any>;
106-
for (const [key, value] of Object.entries(body)) {
107-
if (Array.isArray(value)) {
108-
value.forEach((item) => formData.append(key, item));
109-
} else {
110-
formData.append(key, value);
111-
}
112-
}
113-
}
114-
115-
const headers = {
116-
...opts?.headers,
117-
'Content-Type': `multipart/form-data; boundary=${formData.getBoundary()}`,
118-
};
119-
120-
const options: FinalRequestOptions = {
121-
method: 'post',
122-
path: path,
123-
body: formData,
124-
headers,
125-
};
126-
return options;
127-
}
128-
12988
protected getUserAgent(): string {
13089
const platform =
13190
this.isRunningInBrowser() ?
@@ -141,6 +100,20 @@ export abstract class APIClient {
141100
'User-Agent': this.getUserAgent(),
142101
...this.authHeaders(opts),
143102
};
103+
// if (opts?.body instanceof FormData) {
104+
// return {
105+
// Accept: 'application/json',
106+
// 'User-Agent': this.getUserAgent(),
107+
// ...this.authHeaders(opts),
108+
// };
109+
// } else {
110+
// return {
111+
// Accept: 'application/json',
112+
// 'Content-Type': 'application/json',
113+
// 'User-Agent': this.getUserAgent(),
114+
// ...this.authHeaders(opts),
115+
// };
116+
// }
144117
}
145118

146119
// eslint-disable-next-line @typescript-eslint/no-unused-vars

‎src/fetch/BrowserFetch.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@ import { Stream, BrowserSSEDecoder } from '../streaming';
55
export class BrowserFetch extends BaseFetch {
66
call(url: string, options: FinalRequestOptions): Promise<CrossPlatformResponse> {
77
const controller = new AbortController();
8+
const body = options.body instanceof FormData ? options.body : JSON.stringify(options.body);
89

910
return fetch(url, {
1011
method: options.method,
1112
headers: options?.headers ? (options.headers as HeadersInit) : undefined,
12-
body: options?.body ? JSON.stringify(options.body) : undefined,
13+
body,
1314
signal: controller.signal,
1415
});
1516
}

‎src/files/form-utils.ts

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
import { FilePathOrFileObject } from 'types/rag';
22
import * as Runtime from '../runtime';
33

4-
export type UnifiedFormData = FormData | import('form-data');
5-
6-
import { Readable } from 'stream';
4+
import { UnifiedFormData } from 'types';
75

86
// eslint-disable-next-line @typescript-eslint/no-explicit-any
97
export const appendBodyToFormData = (formData: UnifiedFormData, body: Record<string, any>): void => {
@@ -17,22 +15,30 @@ export const appendBodyToFormData = (formData: UnifiedFormData, body: Record<str
1715
};
1816

1917
// Convert WHATWG ReadableStream to Node.js Readable Stream
20-
function convertReadableStream(whatwgStream: ReadableStream): Readable {
21-
const reader = whatwgStream.getReader();
22-
23-
return new Readable({
24-
async read() {
25-
const { done, value } = await reader.read();
26-
if (done) {
27-
this.push(null);
28-
} else {
29-
this.push(value);
30-
}
31-
},
32-
});
18+
class NodeReadableStream {
19+
async convertReadableStream(whatwgStream: ReadableStream): Promise<NodeJS.ReadableStream> {
20+
if (Runtime.isNode) {
21+
const { Readable } = await import('stream'); // Inline import of the stream module
22+
const reader = whatwgStream.getReader();
23+
24+
return new Readable({
25+
async read() {
26+
const { done, value } = await reader.read();
27+
if (done) {
28+
this.push(null);
29+
} else {
30+
this.push(value);
31+
}
32+
},
33+
});
34+
} else {
35+
throw new Error('convertReadableStream is not supported in the browser environment.');
36+
}
37+
}
3338
}
3439

35-
export async function createFormData(file: FilePathOrFileObject): Promise<UnifiedFormData> {
40+
export class CreateFormData {
41+
async createFormData(file: FilePathOrFileObject): Promise<UnifiedFormData> {
3642
if (Runtime.isBrowser) {
3743
const formData = new FormData();
3844

@@ -45,28 +51,28 @@ export async function createFormData(file: FilePathOrFileObject): Promise<Unifie
4551
return formData;
4652
} else {
4753
// Node environment:
48-
4954
const { default: FormDataNode } = await import('form-data');
5055
const formData = new FormDataNode();
5156

5257
if (typeof file === 'string') {
53-
const { createReadStream } = await import('fs');
54-
formData.append('file', createReadStream(file), { filename: file.split('/').pop() });
58+
const fs = (await import('fs')).default;
59+
formData.append('file', fs.createReadStream(file), { filename: file.split('/').pop() });
5560
} else if (Buffer.isBuffer(file)) {
5661
formData.append('file', file, { filename: 'TODO - add filename to buffer flow' });
5762
} else if (file instanceof File) {
58-
const nodeStream = convertReadableStream(file.stream());
63+
const nodeStream = await new NodeReadableStream().convertReadableStream(file.stream());
5964
formData.append('file', nodeStream, file.name);
6065
} else {
6166
throw new Error('Unsupported file type in Node.js');
6267
}
6368

6469
return formData;
70+
}
6571
}
6672
}
6773

6874
export async function getBoundary(formData: UnifiedFormData): Promise<string | undefined> {
69-
if (!Runtime.isBrowser) {
75+
if (Runtime.isNode) {
7076
const { default: FormDataNode } = await import('form-data');
7177
if (formData instanceof FormDataNode) {
7278
return formData.getBoundary();

‎src/files/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export * from './form-utils';

‎src/resources/rag/ragEngine.ts

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,33 @@ import {
88
FilePathOrFileObject,
99
} from '../../types/rag';
1010
import { FileResponse } from 'types/rag/FileResponse';
11+
import { CreateFormData } from '../../files';
1112

1213
const RAG_ENGINE_PATH = '/library/files';
1314

1415
export class RAGEngine extends APIResource {
15-
create(file: string, body: UploadFileRequest, options?: Models.RequestOptions): Promise<UploadFileResponse>;
16+
async create(
17+
file: string,
18+
body: UploadFileRequest,
19+
options?: Models.RequestOptions,
20+
): Promise<UploadFileResponse>;
1621

17-
create(file: File, body: UploadFileRequest, options?: Models.RequestOptions): Promise<UploadFileResponse>;
22+
async create(
23+
file: File,
24+
body: UploadFileRequest,
25+
options?: Models.RequestOptions,
26+
): Promise<UploadFileResponse>;
1827

19-
create(
28+
async create(
2029
file: FilePathOrFileObject,
2130
body: UploadFileRequest,
2231
options?: Models.RequestOptions,
2332
): Promise<UploadFileResponse> {
24-
return this.client.upload<UploadFileRequest, UploadFileResponse>(RAG_ENGINE_PATH, file, {
33+
const formData = await new CreateFormData().createFormData(file);
34+
return this.client.upload<Models.UnifiedFormData, UploadFileResponse>(RAG_ENGINE_PATH, formData, {
2535
body: body,
2636
...options,
27-
} as Models.RequestOptions<UploadFileRequest>) as Promise<UploadFileResponse>;
37+
} as Models.RequestOptions<Models.UnifiedFormData>) as Promise<UploadFileResponse>;
2838
}
2939

3040
get(fileId: string, options?: Models.RequestOptions): Promise<FileResponse> {

‎src/types/API.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,5 @@ export type Headers = Record<string, string | null | undefined>;
2929
// Platforms specific types for NodeJS and Browser
3030
export type CrossPlatformResponse = Response | import('node-fetch').Response;
3131
export type CrossPlatformReadableStream = ReadableStream<Uint8Array> | import('stream/web').ReadableStream;
32+
33+
export type UnifiedFormData = FormData | import('form-data');

‎src/types/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ export {
3535
type DefaultQuery,
3636
type Headers,
3737
type CrossPlatformResponse,
38+
type UnifiedFormData,
3839
} from './API';
3940

4041
export {

‎vite.config.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export default defineConfig({
1010
formats: ['es', 'cjs', 'umd'],
1111
},
1212
rollupOptions: {
13-
external: ['node-fetch', 'fs', 'path', 'stream', 'form-data'],
13+
external: ['node-fetch'],
1414
output: {
1515
globals: {
1616
'node-fetch': 'fetch',

0 commit comments

Comments
 (0)