Skip to content

Commit f22bf89

Browse files
ahmosmanahmosman
authored
Merge branch 'main' into bugfix/web_crawler_search_params
2 parents 3739cf1 + adea2f0 commit f22bf89

File tree

25 files changed

+155
-73
lines changed

25 files changed

+155
-73
lines changed

‎packages/components/models.json

+4
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,10 @@
527527
"label": "open-mixtral-8x7b",
528528
"name": "open-mixtral-8x7b"
529529
},
530+
{
531+
"label": "open-mixtral-8x22b",
532+
"name": "open-mixtral-8x22b"
533+
},
530534
{
531535
"label": "mistral-small-2312",
532536
"name": "mistral-small-2312"

‎packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts

+9-2
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,19 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
7070
const output = nodeData.outputs?.output as string
7171

7272
const topK = (vectorStore as any)?.k ?? 4
73+
const _filter = (vectorStore as any)?.filter
7374

74-
const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK)
75+
// If it is already pre-defined in lc_kwargs, then don't pass it again
76+
const filter = vectorStore.lc_kwargs.filter ? undefined : _filter
77+
if (vectorStore.lc_kwargs.filter) {
78+
;(vectorStore as any).filter = vectorStore.lc_kwargs.filter
79+
}
80+
81+
const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK, filter)
7582
// eslint-disable-next-line no-console
7683
console.log('\x1b[94m\x1b[1m\n*****VectorStore Documents*****\n\x1b[0m\x1b[0m')
7784
// eslint-disable-next-line no-console
78-
console.log(docs)
85+
console.log(JSON.stringify(docs, null, 2))
7986

8087
if (output === 'document') {
8188
let finaldocs = []

‎packages/components/nodes/vectorstores/Chroma/Chroma.ts

+3
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,9 @@ class Chroma_VectorStores implements INode {
187187
return retriever
188188
} else if (output === 'vectorStore') {
189189
;(vectorStore as any).k = k
190+
if (chromaMetadataFilter) {
191+
;(vectorStore as any).filter = obj.filter
192+
}
190193
return vectorStore
191194
}
192195
return vectorStore

‎packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts

+3
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,9 @@ class Chroma_Existing_VectorStores implements INode {
117117
return retriever
118118
} else if (output === 'vectorStore') {
119119
;(vectorStore as any).k = k
120+
if (chromaMetadataFilter) {
121+
;(vectorStore as any).filter = obj.filter
122+
}
120123
return vectorStore
121124
}
122125
return vectorStore

‎packages/components/nodes/vectorstores/Milvus/Milvus.ts

+3
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,9 @@ class Milvus_VectorStores implements INode {
199199
return retriever
200200
} else if (output === 'vectorStore') {
201201
;(vectorStore as any).k = k
202+
if (milvusFilter) {
203+
;(vectorStore as any).filter = milvusFilter
204+
}
202205
return vectorStore
203206
}
204207
return vectorStore

‎packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts

+3
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,9 @@ class Milvus_Existing_VectorStores implements INode {
189189
return retriever
190190
} else if (output === 'vectorStore') {
191191
;(vectorStore as any).k = k
192+
if (milvusFilter) {
193+
;(vectorStore as any).filter = milvusFilter
194+
}
192195
return vectorStore
193196
}
194197
return vectorStore

‎packages/components/nodes/vectorstores/Pinecone/Pinecone.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ class Pinecone_VectorStores implements INode {
190190

191191
const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
192192

193-
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
193+
return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter)
194194
}
195195
}
196196

‎packages/components/nodes/vectorstores/Qdrant/Qdrant.ts

+38-6
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class Qdrant_VectorStores implements INode {
3232
constructor() {
3333
this.label = 'Qdrant'
3434
this.name = 'qdrant'
35-
this.version = 2.0
35+
this.version = 3.0
3636
this.type = 'Qdrant'
3737
this.icon = 'qdrant.png'
3838
this.category = 'Vector Stores'
@@ -86,6 +86,15 @@ class Qdrant_VectorStores implements INode {
8686
default: 1536,
8787
additionalParams: true
8888
},
89+
{
90+
label: 'Upsert Batch Size',
91+
name: 'batchSize',
92+
type: 'number',
93+
step: 1,
94+
description: 'Upsert in batches of size N',
95+
additionalParams: true,
96+
optional: true
97+
},
8998
{
9099
label: 'Similarity',
91100
name: 'qdrantSimilarity',
@@ -159,6 +168,7 @@ class Qdrant_VectorStores implements INode {
159168
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
160169
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
161170
const recordManager = nodeData.inputs?.recordManager
171+
const _batchSize = nodeData.inputs?.batchSize
162172

163173
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
164174
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
@@ -218,10 +228,21 @@ class Qdrant_VectorStores implements INode {
218228
}))
219229

220230
try {
221-
await client.upsert(collectionName, {
222-
wait: true,
223-
points
224-
})
231+
if (_batchSize) {
232+
const batchSize = parseInt(_batchSize, 10)
233+
for (let i = 0; i < points.length; i += batchSize) {
234+
const batchPoints = points.slice(i, i + batchSize)
235+
await client.upsert(collectionName, {
236+
wait: true,
237+
points: batchPoints
238+
})
239+
}
240+
} else {
241+
await client.upsert(collectionName, {
242+
wait: true,
243+
points
244+
})
245+
}
225246
} catch (e: any) {
226247
const error = new Error(`${e?.status ?? 'Undefined error code'} ${e?.message}: ${e?.data?.status?.error}`)
227248
throw error
@@ -257,7 +278,15 @@ class Qdrant_VectorStores implements INode {
257278

258279
return res
259280
} else {
260-
await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
281+
if (_batchSize) {
282+
const batchSize = parseInt(_batchSize, 10)
283+
for (let i = 0; i < finalDocs.length; i += batchSize) {
284+
const batch = finalDocs.slice(i, i + batchSize)
285+
await QdrantVectorStore.fromDocuments(batch, embeddings, dbConfig)
286+
}
287+
} else {
288+
await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
289+
}
261290
return { numAdded: finalDocs.length, addedDocs: finalDocs }
262291
}
263292
} catch (e) {
@@ -325,6 +354,9 @@ class Qdrant_VectorStores implements INode {
325354
return retriever
326355
} else if (output === 'vectorStore') {
327356
;(vectorStore as any).k = k
357+
if (queryFilter) {
358+
;(vectorStore as any).filter = retrieverConfig.filter
359+
}
328360
return vectorStore
329361
}
330362
return vectorStore

‎packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts

+3
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,9 @@ class Qdrant_Existing_VectorStores implements INode {
182182
return retriever
183183
} else if (output === 'vectorStore') {
184184
;(vectorStore as any).k = k
185+
if (queryFilter) {
186+
;(vectorStore as any).filter = retrieverConfig.filter
187+
}
185188
return vectorStore
186189
}
187190
return vectorStore

‎packages/components/nodes/vectorstores/Qdrant/Qdrant_Upsert.ts

+31-8
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class QdrantUpsert_VectorStores implements INode {
2626
constructor() {
2727
this.label = 'Qdrant Upsert Document'
2828
this.name = 'qdrantUpsert'
29-
this.version = 2.0
29+
this.version = 3.0
3030
this.type = 'Qdrant'
3131
this.icon = 'qdrant.png'
3232
this.category = 'Vector Stores'
@@ -71,6 +71,15 @@ class QdrantUpsert_VectorStores implements INode {
7171
default: 1536,
7272
additionalParams: true
7373
},
74+
{
75+
label: 'Upsert Batch Size',
76+
name: 'batchSize',
77+
type: 'number',
78+
step: 1,
79+
description: 'Upsert in batches of size N',
80+
additionalParams: true,
81+
optional: true
82+
},
7483
{
7584
label: 'Similarity',
7685
name: 'qdrantSimilarity',
@@ -132,6 +141,7 @@ class QdrantUpsert_VectorStores implements INode {
132141
const embeddings = nodeData.inputs?.embeddings as Embeddings
133142
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
134143
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
144+
const _batchSize = nodeData.inputs?.batchSize
135145

136146
const output = nodeData.outputs?.output as string
137147
const topK = nodeData.inputs?.topK as string
@@ -174,16 +184,29 @@ class QdrantUpsert_VectorStores implements INode {
174184
retrieverConfig.filter = typeof queryFilter === 'object' ? queryFilter : JSON.parse(queryFilter)
175185
}
176186

177-
const vectorStore = await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
187+
let vectorStore: QdrantVectorStore | undefined = undefined
188+
if (_batchSize) {
189+
const batchSize = parseInt(_batchSize, 10)
190+
for (let i = 0; i < finalDocs.length; i += batchSize) {
191+
const batch = finalDocs.slice(i, i + batchSize)
192+
vectorStore = await QdrantVectorStore.fromDocuments(batch, embeddings, dbConfig)
193+
}
194+
} else {
195+
vectorStore = await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
196+
}
178197

179-
if (output === 'retriever') {
180-
const retriever = vectorStore.asRetriever(retrieverConfig)
181-
return retriever
182-
} else if (output === 'vectorStore') {
183-
;(vectorStore as any).k = k
198+
if (vectorStore === undefined) {
199+
throw new Error('No documents to upsert')
200+
} else {
201+
if (output === 'retriever') {
202+
const retriever = vectorStore.asRetriever(retrieverConfig)
203+
return retriever
204+
} else if (output === 'vectorStore') {
205+
;(vectorStore as any).k = k
206+
return vectorStore
207+
}
184208
return vectorStore
185209
}
186-
return vectorStore
187210
}
188211
}
189212

‎packages/components/nodes/vectorstores/Supabase/Supabase.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ class Supabase_VectorStores implements INode {
186186

187187
const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, obj)
188188

189-
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
189+
return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter)
190190
}
191191
}
192192

‎packages/components/nodes/vectorstores/Supabase/Supabase_Exisiting.ts

+3
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,9 @@ class Supabase_Existing_VectorStores implements INode {
119119
return retriever
120120
} else if (output === 'vectorStore') {
121121
;(vectorStore as any).k = k
122+
if (supabaseMetadataFilter) {
123+
;(vectorStore as any).filter = obj.filter
124+
}
122125
return vectorStore
123126
}
124127
return vectorStore

‎packages/components/nodes/vectorstores/Vectara/Vectara.ts

+3
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,9 @@ class Vectara_VectorStores implements INode {
272272
return retriever
273273
} else if (output === 'vectorStore') {
274274
;(vectorStore as any).k = k
275+
if (vectaraMetadataFilter) {
276+
;(vectorStore as any).filter = vectaraFilter.filter
277+
}
275278
return vectorStore
276279
}
277280
return vectorStore

‎packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts

+3
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,9 @@ class VectaraExisting_VectorStores implements INode {
127127
return retriever
128128
} else if (output === 'vectorStore') {
129129
;(vectorStore as any).k = k
130+
if (vectaraMetadataFilter) {
131+
;(vectorStore as any).filter = vectaraFilter.filter
132+
}
130133
return vectorStore
131134
}
132135
return vectorStore

‎packages/components/nodes/vectorstores/VectorStoreUtils.ts

+12-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,19 @@
11
import { INodeData } from '../../src'
2+
import { VectorStore } from '@langchain/core/vectorstores'
23

3-
export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore: any) => {
4+
export const resolveVectorStoreOrRetriever = (
5+
nodeData: INodeData,
6+
vectorStore: VectorStore,
7+
metadataFilter?: string | object | undefined
8+
) => {
49
const output = nodeData.outputs?.output as string
510
const searchType = nodeData.outputs?.searchType as string
611
const topK = nodeData.inputs?.topK as string
712
const k = topK ? parseFloat(topK) : 4
813

14+
// If it is already pre-defined in lc_kwargs, then don't pass it again
15+
const filter = vectorStore?.lc_kwargs?.filter ? undefined : metadataFilter
16+
917
if (output === 'retriever') {
1018
if ('mmr' === searchType) {
1119
const fetchK = nodeData.inputs?.fetchK as string
@@ -15,17 +23,19 @@ export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore:
1523
return vectorStore.asRetriever({
1624
searchType: 'mmr',
1725
k: k,
26+
filter,
1827
searchKwargs: {
1928
fetchK: f,
2029
lambda: l
2130
}
2231
})
2332
} else {
2433
// "searchType" is "similarity"
25-
return vectorStore.asRetriever(k)
34+
return vectorStore.asRetriever(k, filter)
2635
}
2736
} else if (output === 'vectorStore') {
2837
;(vectorStore as any).k = k
38+
;(vectorStore as any).filter = filter
2939
return vectorStore
3040
}
3141
}

‎packages/components/nodes/vectorstores/Weaviate/Weaviate.ts

+12-1
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,13 @@ class Weaviate_VectorStores implements INode {
114114
type: 'number',
115115
additionalParams: true,
116116
optional: true
117+
},
118+
{
119+
label: 'Weaviate Search Filter',
120+
name: 'weaviateFilter',
121+
type: 'json',
122+
additionalParams: true,
123+
optional: true
117124
}
118125
]
119126
addMMRInputParams(this.inputs)
@@ -203,6 +210,7 @@ class Weaviate_VectorStores implements INode {
203210
const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
204211
const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
205212
const embeddings = nodeData.inputs?.embeddings as Embeddings
213+
let weaviateFilter = nodeData.inputs?.weaviateFilter
206214

207215
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
208216
const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
@@ -223,10 +231,13 @@ class Weaviate_VectorStores implements INode {
223231

224232
if (weaviateTextKey) obj.textKey = weaviateTextKey
225233
if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
234+
if (weaviateFilter) {
235+
weaviateFilter = typeof weaviateFilter === 'object' ? weaviateFilter : JSON.parse(weaviateFilter)
236+
}
226237

227238
const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj)
228239

229-
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
240+
return resolveVectorStoreOrRetriever(nodeData, vectorStore, weaviateFilter)
230241
}
231242
}
232243

‎packages/components/nodes/vectorstores/Zep/Zep.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ class Zep_VectorStores implements INode {
165165

166166
const vectorStore = await ZepExistingVS.fromExistingIndex(embeddings, zepConfig)
167167

168-
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
168+
return resolveVectorStoreOrRetriever(nodeData, vectorStore, zepConfig.filter)
169169
}
170170
}
171171

‎packages/components/nodes/vectorstores/Zep/Zep_Existing.ts

+3
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@ class Zep_Existing_VectorStores implements INode {
125125
return retriever
126126
} else if (output === 'vectorStore') {
127127
;(vectorStore as any).k = k
128+
if (zepMetadataFilter) {
129+
;(vectorStore as any).filter = zepConfig.filter
130+
}
128131
return vectorStore
129132
}
130133
return vectorStore

‎packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ class Zep_CloudVectorStores implements INode {
131131
}
132132
zepConfig.client = await ZepClient.init(zepConfig.apiKey)
133133
const vectorStore = await ZepExistingVS.init(zepConfig)
134-
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
134+
return resolveVectorStoreOrRetriever(nodeData, vectorStore, zepConfig.filter)
135135
}
136136
}
137137

0 commit comments

Comments
 (0)