Skip to content

Commit 5d237eb

Browse files
maxtecheramaxtechera
committed
Update the documentstore with missing userId/organiationId
1 parent 5253be9 commit 5d237eb

File tree

7 files changed

+115
-36
lines changed

7 files changed

+115
-36
lines changed

‎apps/web/app/(Main UI)/(Studio Layout)/sidekick-studio/(main-layout)/document-stores/[id]/[name]/page.tsx renamed to ‎apps/web/app/(Main UI)/(Studio Layout)/sidekick-studio/(main-layout)/document-stores/[storeId]/[name]/page.tsx

File renamed without changes.

‎apps/web/app/(Main UI)/(Studio Layout)/sidekick-studio/(main-layout)/document-stores/[id]/page.tsx renamed to ‎apps/web/app/(Main UI)/(Studio Layout)/sidekick-studio/(main-layout)/document-stores/[storeId]/page.tsx

File renamed without changes.

‎packages/server/src/Interface.DocumentStore.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ export interface IDocumentStoreFileChunk {
3636
storeId: string
3737
pageContent: string
3838
metadata: string
39+
userId: string
40+
organizationId: string
3941
}
4042

4143
export interface IDocumentStoreFileChunkPagedResponse {

‎packages/server/src/controllers/documentstore/index.ts

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,10 @@ const processLoader = async (req: Request, res: Response, next: NextFunction) =>
234234
}
235235
const docLoaderId = req.params.loaderId
236236
const body = req.body
237-
const apiResponse = await documentStoreService.processLoaderMiddleware(body, docLoaderId)
237+
const apiResponse = await documentStoreService.processLoaderMiddleware(
238+
{ ...body, userId: req.user?.id!, organizationId: req.user?.organizationId! },
239+
docLoaderId
240+
)
238241
return res.json(apiResponse)
239242
} catch (error) {
240243
next(error)
@@ -265,7 +268,7 @@ const updateDocumentStore = async (req: Request, res: Response, next: NextFuncti
265268
const body = req.body
266269
const updateDocStore = new DocumentStore()
267270
Object.assign(updateDocStore, body)
268-
const apiResponse = await documentStoreService.updateDocumentStore(store, updateDocStore)
271+
const apiResponse = await documentStoreService.updateDocumentStore(store, updateDocStore, req.user?.id!, req.user?.organizationId!)
269272
return res.json(DocumentStoreDTO.fromEntity(apiResponse))
270273
} catch (error) {
271274
next(error)
@@ -297,7 +300,11 @@ const previewFileChunks = async (req: Request, res: Response, next: NextFunction
297300
}
298301
const body = req.body
299302
body.preview = true
300-
const apiResponse = await documentStoreService.previewChunksMiddleware(body)
303+
const apiResponse = await documentStoreService.previewChunksMiddleware({
304+
...body,
305+
userId: req.user?.id!,
306+
organizationId: req.user?.organizationId!
307+
})
301308
return res.json(apiResponse)
302309
} catch (error) {
303310
next(error)
@@ -319,7 +326,12 @@ const insertIntoVectorStore = async (req: Request, res: Response, next: NextFunc
319326
throw new Error('Error: documentStoreController.insertIntoVectorStore - body not provided!')
320327
}
321328
const body = req.body
322-
const apiResponse = await documentStoreService.insertIntoVectorStoreMiddleware(body)
329+
const apiResponse = await documentStoreService.insertIntoVectorStoreMiddleware(
330+
{ ...body, userId: req.user?.id!, organizationId: req.user?.organizationId! },
331+
true,
332+
req.user?.id!,
333+
req.user?.organizationId!
334+
)
323335
getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
324336
status: FLOWISE_COUNTER_STATUS.SUCCESS
325337
})
@@ -338,7 +350,11 @@ const queryVectorStore = async (req: Request, res: Response, next: NextFunction)
338350
throw new Error('Error: documentStoreController.queryVectorStore - body not provided!')
339351
}
340352
const body = req.body
341-
const apiResponse = await documentStoreService.queryVectorStore(body)
353+
const apiResponse = await documentStoreService.queryVectorStore(
354+
{ ...body, userId: req.user?.id!, organizationId: req.user?.organizationId! },
355+
req.user?.id!,
356+
req.user?.organizationId!
357+
)
342358
return res.json(apiResponse)
343359
} catch (error) {
344360
next(error)
@@ -353,7 +369,11 @@ const deleteVectorStoreFromStore = async (req: Request, res: Response, next: Nex
353369
`Error: documentStoreController.deleteVectorStoreFromStore - storeId not provided!`
354370
)
355371
}
356-
const apiResponse = await documentStoreService.deleteVectorStoreFromStore(req.params.storeId)
372+
const apiResponse = await documentStoreService.deleteVectorStoreFromStore(
373+
req.params.storeId,
374+
req.user?.id!,
375+
req.user?.organizationId!
376+
)
357377
return res.json(apiResponse)
358378
} catch (error) {
359379
next(error)
@@ -368,7 +388,13 @@ const saveVectorStoreConfig = async (req: Request, res: Response, next: NextFunc
368388
const body = req.body
369389
const appDataSource = getRunningExpressApp().AppDataSource
370390
const componentNodes = getRunningExpressApp().nodesPool.componentNodes
371-
const apiResponse = await documentStoreService.saveVectorStoreConfig(appDataSource, componentNodes, body)
391+
const apiResponse = await documentStoreService.saveVectorStoreConfig(
392+
appDataSource,
393+
body,
394+
true,
395+
req.user?.id!,
396+
req.user?.organizationId!
397+
)
372398
return res.json(apiResponse)
373399
} catch (error) {
374400
next(error)
@@ -381,7 +407,11 @@ const updateVectorStoreConfigOnly = async (req: Request, res: Response, next: Ne
381407
throw new Error('Error: documentStoreController.updateVectorStoreConfigOnly - body not provided!')
382408
}
383409
const body = req.body
384-
const apiResponse = await documentStoreService.updateVectorStoreConfigOnly(body)
410+
const apiResponse = await documentStoreService.updateVectorStoreConfigOnly({
411+
...body,
412+
userId: req.user?.id!,
413+
organizationId: req.user?.organizationId!
414+
})
385415
return res.json(apiResponse)
386416
} catch (error) {
387417
next(error)
@@ -390,7 +420,7 @@ const updateVectorStoreConfigOnly = async (req: Request, res: Response, next: Ne
390420

391421
const getEmbeddingProviders = async (req: Request, res: Response, next: NextFunction) => {
392422
try {
393-
const apiResponse = await documentStoreService.getEmbeddingProviders()
423+
const apiResponse = await documentStoreService.getEmbeddingProviders(req.user?.id!, req.user?.organizationId!)
394424
return res.json(apiResponse)
395425
} catch (error) {
396426
next(error)
@@ -428,7 +458,11 @@ const upsertDocStoreMiddleware = async (req: Request, res: Response, next: NextF
428458
}
429459
const body = req.body
430460
const files = (req.files as Express.Multer.File[]) || []
431-
const apiResponse = await documentStoreService.upsertDocStoreMiddleware(req.params.id, body, files)
461+
const apiResponse = await documentStoreService.upsertDocStoreMiddleware(
462+
req.params.id,
463+
{ ...body, userId: req.user?.id!, organizationId: req.user?.organizationId! },
464+
files
465+
)
432466
getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
433467
status: FLOWISE_COUNTER_STATUS.SUCCESS
434468
})
@@ -450,7 +484,11 @@ const refreshDocStoreMiddleware = async (req: Request, res: Response, next: Next
450484
)
451485
}
452486
const body = req.body
453-
const apiResponse = await documentStoreService.refreshDocStoreMiddleware(req.params.id, body)
487+
const apiResponse = await documentStoreService.refreshDocStoreMiddleware(req.params.id, {
488+
...body,
489+
userId: req.user?.id!,
490+
organizationId: req.user?.organizationId!
491+
})
454492
getRunningExpressApp().metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
455493
status: FLOWISE_COUNTER_STATUS.SUCCESS
456494
})

‎packages/server/src/database/entities/DocumentStoreFileChunk.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ export class DocumentStoreFileChunk implements IDocumentStoreFileChunk {
2525

2626
@Index()
2727
@Column({ type: 'uuid', nullable: true })
28-
userId?: string
28+
userId: string
2929

3030
@Index()
3131
@Column({ type: 'uuid', nullable: true })
32-
organizationId?: string
32+
organizationId: string
3333
}

‎packages/server/src/services/documentstore/index.ts

Lines changed: 61 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ const getDocumentStoreFileChunks = async (
289289
}
290290

291291
// Sync and refresh chunks for a specific loader or store
292-
const syncAndRefreshChunks = async (storeId: string, fileId: string, userId?: string, organizationId?: string) => {
292+
const syncAndRefreshChunks = async (storeId: string, fileId: string, userId: string, organizationId: string) => {
293293
try {
294294
const appServer = getRunningExpressApp()
295295
const componentNodes = appServer.nodesPool.componentNodes
@@ -460,7 +460,7 @@ const deleteDocumentStoreFileChunk = async (storeId: string, docId: string, chun
460460
}
461461
}
462462

463-
const deleteVectorStoreFromStore = async (storeId: string) => {
463+
const deleteVectorStoreFromStore = async (storeId: string, userId: string, organizationId: string) => {
464464
try {
465465
const appServer = getRunningExpressApp()
466466
const componentNodes = appServer.nodesPool.componentNodes
@@ -588,7 +588,12 @@ const editDocumentStoreFileChunk = async (
588588
}
589589
}
590590

591-
const updateDocumentStore = async (documentStore: DocumentStore, updatedDocumentStore: DocumentStore) => {
591+
const updateDocumentStore = async (
592+
documentStore: DocumentStore,
593+
updatedDocumentStore: DocumentStore,
594+
userId: string,
595+
organizationId: string
596+
) => {
592597
try {
593598
const appServer = getRunningExpressApp()
594599
const tmpUpdatedDocumentStore = appServer.AppDataSource.getRepository(DocumentStore).merge(documentStore, updatedDocumentStore)
@@ -602,7 +607,7 @@ const updateDocumentStore = async (documentStore: DocumentStore, updatedDocument
602607
}
603608
}
604609

605-
const _saveFileToStorage = async (fileBase64: string, entity: DocumentStore) => {
610+
const _saveFileToStorage = async (fileBase64: string, entity: DocumentStore, userId: string, organizationId: string) => {
606611
const splitDataURI = fileBase64.split(',')
607612
const filename = splitDataURI.pop()?.split(':')[1] ?? ''
608613
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
@@ -980,15 +985,15 @@ const _saveChunksToStorage = async (
980985
for (let j = 0; j < files.length; j++) {
981986
const file = files[j]
982987
if (re.test(file)) {
983-
const fileMetadata = await _saveFileToStorage(file, entity)
988+
const fileMetadata = await _saveFileToStorage(file, entity, data.userId, data.organizationId)
984989
fileNames.push(fileMetadata.name)
985990
filesWithMetadata.push(fileMetadata)
986991
}
987992
}
988993
if (fileNames.length) data.loaderConfig[keys[i]] = 'FILE-STORAGE::' + JSON.stringify(fileNames)
989994
} else if (re.test(input)) {
990995
const fileNames: string[] = []
991-
const fileMetadata = await _saveFileToStorage(input, entity)
996+
const fileMetadata = await _saveFileToStorage(input, entity, data.userId, data.organizationId)
992997
fileNames.push(fileMetadata.name)
993998
filesWithMetadata.push(fileMetadata)
994999
data.loaderConfig[keys[i]] = 'FILE-STORAGE::' + JSON.stringify(fileNames)
@@ -1024,7 +1029,9 @@ const _saveChunksToStorage = async (
10241029
id: uuidv4(),
10251030
chunkNo: index + 1,
10261031
pageContent: chunk.pageContent,
1027-
metadata: JSON.stringify(chunk.metadata)
1032+
metadata: JSON.stringify(chunk.metadata),
1033+
userId: data.userId,
1034+
organizationId: data.organizationId
10281035
}
10291036
const dChunk = appDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
10301037
await appDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
@@ -1139,7 +1146,13 @@ const updateVectorStoreConfigOnly = async (data: ICommonObject) => {
11391146
)
11401147
}
11411148
}
1142-
const saveVectorStoreConfig = async (appDataSource: DataSource, data: ICommonObject, isStrictSave = true) => {
1149+
const saveVectorStoreConfig = async (
1150+
appDataSource: DataSource,
1151+
data: ICommonObject,
1152+
isStrictSave = true,
1153+
userId: string,
1154+
organizationId: string
1155+
) => {
11431156
try {
11441157
const entity = await appDataSource.getRepository(DocumentStore).findOneBy({
11451158
id: data.storeId
@@ -1207,14 +1220,24 @@ export const insertIntoVectorStore = async ({
12071220
componentNodes,
12081221
telemetry,
12091222
data,
1210-
isStrictSave
1223+
isStrictSave,
1224+
userId,
1225+
organizationId
12111226
}: IExecuteVectorStoreInsert) => {
12121227
try {
1213-
const entity = await saveVectorStoreConfig(appDataSource, data, isStrictSave)
1228+
const entity = await saveVectorStoreConfig(appDataSource, data, isStrictSave, userId, organizationId)
12141229
entity.status = DocumentStoreStatus.UPSERTING
12151230
await appDataSource.getRepository(DocumentStore).save(entity)
12161231

1217-
const indexResult = await _insertIntoVectorStoreWorkerThread(appDataSource, componentNodes, telemetry, data, isStrictSave)
1232+
const indexResult = await _insertIntoVectorStoreWorkerThread(
1233+
appDataSource,
1234+
componentNodes,
1235+
telemetry,
1236+
data,
1237+
isStrictSave,
1238+
userId,
1239+
organizationId
1240+
)
12181241
return indexResult
12191242
} catch (error) {
12201243
throw new InternalFlowiseError(
@@ -1224,7 +1247,7 @@ export const insertIntoVectorStore = async ({
12241247
}
12251248
}
12261249

1227-
const insertIntoVectorStoreMiddleware = async (data: ICommonObject, isStrictSave = true) => {
1250+
const insertIntoVectorStoreMiddleware = async (data: ICommonObject, isStrictSave = true, userId: string, organizationId: string) => {
12281251
try {
12291252
const appServer = getRunningExpressApp()
12301253
const appDataSource = appServer.AppDataSource
@@ -1238,8 +1261,8 @@ const insertIntoVectorStoreMiddleware = async (data: ICommonObject, isStrictSave
12381261
data,
12391262
isStrictSave,
12401263
isVectorStoreInsert: true,
1241-
userId: data.userId,
1242-
organizationId: data.organizationId
1264+
userId,
1265+
organizationId
12431266
}
12441267

12451268
if (process.env.MODE === MODE.QUEUE) {
@@ -1270,10 +1293,12 @@ const _insertIntoVectorStoreWorkerThread = async (
12701293
componentNodes: IComponentNodes,
12711294
telemetry: Telemetry,
12721295
data: ICommonObject,
1273-
isStrictSave = true
1296+
isStrictSave = true,
1297+
userId: string,
1298+
organizationId: string
12741299
) => {
12751300
try {
1276-
const entity = await saveVectorStoreConfig(appDataSource, data, isStrictSave)
1301+
const entity = await saveVectorStoreConfig(appDataSource, data, isStrictSave, userId, organizationId)
12771302
let upsertHistory: Record<string, any> = {}
12781303
const chatflowid = data.storeId // fake chatflowid because this is not tied to any chatflow
12791304

@@ -1351,7 +1376,7 @@ const _insertIntoVectorStoreWorkerThread = async (
13511376
}
13521377

13531378
// Get all component nodes - Embeddings
1354-
const getEmbeddingProviders = async () => {
1379+
const getEmbeddingProviders = async (userId: string, organizationId: string) => {
13551380
try {
13561381
const dbResponse = await nodesService.getAllNodesForCategory('Embeddings')
13571382
return dbResponse.filter((node) => !node.tags?.includes('LlamaIndex'))
@@ -1390,13 +1415,15 @@ const getRecordManagerProviders = async () => {
13901415
}
13911416
}
13921417

1393-
const queryVectorStore = async (data: ICommonObject) => {
1418+
const queryVectorStore = async (data: ICommonObject, userId: string, organizationId: string) => {
13941419
try {
13951420
const appServer = getRunningExpressApp()
13961421
const componentNodes = appServer.nodesPool.componentNodes
13971422

13981423
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({
1399-
id: data.storeId
1424+
id: data.storeId,
1425+
userId,
1426+
organizationId
14001427
})
14011428
if (!entity) {
14021429
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Document store ${data.storeId} not found`)
@@ -1597,7 +1624,9 @@ const upsertDocStore = async (
15971624
storeId: string,
15981625
data: IDocumentStoreUpsertData,
15991626
files: Express.Multer.File[] = [],
1600-
isRefreshExisting = false
1627+
isRefreshExisting = false,
1628+
userId: string,
1629+
organizationId: string
16011630
) => {
16021631
const docId = data.docId
16031632
let metadata = {}
@@ -1683,7 +1712,7 @@ const upsertDocStore = async (
16831712
const docStoreBody = typeof data.docStore === 'string' ? JSON.parse(data.docStore) : data.docStore
16841713
const newDocumentStore = docStoreBody ?? { name: `Document Store ${Date.now().toString()}` }
16851714
const docStore = DocumentStoreDTO.toEntity(newDocumentStore)
1686-
const documentStore = appDataSource.getRepository(DocumentStore).create(docStore)
1715+
const documentStore = appDataSource.getRepository(DocumentStore).create({ ...docStore, userId, organizationId })
16871716
const dbResponse = await appDataSource.getRepository(DocumentStore).save(documentStore)
16881717
storeId = dbResponse.id
16891718
}
@@ -1872,7 +1901,17 @@ export const executeDocStoreUpsert = async ({
18721901
}: IExecuteDocStoreUpsert) => {
18731902
const results = []
18741903
for (const item of totalItems) {
1875-
const res = await upsertDocStore(appDataSource, componentNodes, telemetry, storeId, item, files, isRefreshAPI)
1904+
const res = await upsertDocStore(
1905+
appDataSource,
1906+
componentNodes,
1907+
telemetry,
1908+
storeId,
1909+
item,
1910+
files,
1911+
isRefreshAPI,
1912+
userId,
1913+
organizationId
1914+
)
18761915
results.push(res)
18771916
}
18781917
return isRefreshAPI ? results : results[0]

‎packages/ui/src/views/docstore/DocumentStoreDetail.jsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ const DocumentStoreDetails = () => {
151151
const [anchorEl, setAnchorEl] = useState(null)
152152
const open = Boolean(anchorEl)
153153

154-
const { id:storeId } = useParams()
154+
const { storeId } = useParams()
155155

156156
const openPreviewSettings = (id) => {
157157
navigate('/document-stores/' + storeId + '/' + id)

0 commit comments

Comments
 (0)