diff --git a/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts b/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts index 366f9122f..6fc5032d4 100644 --- a/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts +++ b/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts @@ -70,12 +70,19 @@ class VectorStoreToDocument_DocumentLoaders implements INode { const output = nodeData.outputs?.output as string const topK = (vectorStore as any)?.k ?? 4 + const _filter = (vectorStore as any)?.filter - const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK) + // If it is already pre-defined in lc_kwargs, then don't pass it again + const filter = vectorStore.lc_kwargs.filter ? undefined : _filter + if (vectorStore.lc_kwargs.filter) { + ;(vectorStore as any).filter = vectorStore.lc_kwargs.filter + } + + const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK, filter) // eslint-disable-next-line no-console console.log('\x1b[94m\x1b[1m\n*****VectorStore Documents*****\n\x1b[0m\x1b[0m') // eslint-disable-next-line no-console - console.log(docs) + console.log(JSON.stringify(docs, null, 2)) if (output === 'document') { let finaldocs = [] diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma.ts b/packages/components/nodes/vectorstores/Chroma/Chroma.ts index a8722e611..bacf04455 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma.ts @@ -187,6 +187,9 @@ class Chroma_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (chromaMetadataFilter) { + ;(vectorStore as any).filter = obj.filter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts index 90c650c04..8f3f52ba6 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts @@ -117,6 +117,9 @@ class Chroma_Existing_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (chromaMetadataFilter) { + ;(vectorStore as any).filter = obj.filter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/Milvus/Milvus.ts b/packages/components/nodes/vectorstores/Milvus/Milvus.ts index c0818df45..bd850ee22 100644 --- a/packages/components/nodes/vectorstores/Milvus/Milvus.ts +++ b/packages/components/nodes/vectorstores/Milvus/Milvus.ts @@ -199,6 +199,9 @@ class Milvus_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (milvusFilter) { + ;(vectorStore as any).filter = milvusFilter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts b/packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts index 5077b449a..03a1bc9b8 100644 --- a/packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts +++ b/packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts @@ -189,6 +189,9 @@ class Milvus_Existing_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (milvusFilter) { + ;(vectorStore as any).filter = milvusFilter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts index 095e23726..8451ff647 100644 --- a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts @@ -190,7 +190,7 @@ class Pinecone_VectorStores implements INode { const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj) - return resolveVectorStoreOrRetriever(nodeData, vectorStore) + return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter) } } diff --git a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts index e67dd533b..82ec1170e 100644 --- a/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts +++ b/packages/components/nodes/vectorstores/Qdrant/Qdrant.ts @@ -354,6 +354,9 @@ class Qdrant_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (queryFilter) { + ;(vectorStore as any).filter = retrieverConfig.filter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts b/packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts index 61cd260ff..bcb80d34c 100644 --- a/packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts +++ b/packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts @@ -182,6 +182,9 @@ class Qdrant_Existing_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (queryFilter) { + ;(vectorStore as any).filter = retrieverConfig.filter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/Supabase/Supabase.ts b/packages/components/nodes/vectorstores/Supabase/Supabase.ts index b5472ef4f..c732f286f 100644 --- a/packages/components/nodes/vectorstores/Supabase/Supabase.ts +++ b/packages/components/nodes/vectorstores/Supabase/Supabase.ts @@ -186,7 +186,7 @@ class Supabase_VectorStores implements INode { const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, obj) - return resolveVectorStoreOrRetriever(nodeData, vectorStore) + return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter) } } diff --git a/packages/components/nodes/vectorstores/Supabase/Supabase_Exisiting.ts b/packages/components/nodes/vectorstores/Supabase/Supabase_Exisiting.ts index 03bd3437a..6d83ce751 100644 --- a/packages/components/nodes/vectorstores/Supabase/Supabase_Exisiting.ts +++ b/packages/components/nodes/vectorstores/Supabase/Supabase_Exisiting.ts @@ -119,6 +119,9 @@ class Supabase_Existing_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (supabaseMetadataFilter) { + ;(vectorStore as any).filter = obj.filter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/Vectara/Vectara.ts b/packages/components/nodes/vectorstores/Vectara/Vectara.ts index 7d0705075..061562beb 100644 --- a/packages/components/nodes/vectorstores/Vectara/Vectara.ts +++ b/packages/components/nodes/vectorstores/Vectara/Vectara.ts @@ -272,6 +272,9 @@ class Vectara_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (vectaraMetadataFilter) { + ;(vectorStore as any).filter = vectaraFilter.filter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts b/packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts index d8322de41..f648aa55d 100644 --- a/packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts +++ b/packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts @@ -127,6 +127,9 @@ class VectaraExisting_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (vectaraMetadataFilter) { + ;(vectorStore as any).filter = vectaraFilter.filter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/VectorStoreUtils.ts b/packages/components/nodes/vectorstores/VectorStoreUtils.ts index 0d92587f4..8b75337c1 100644 --- a/packages/components/nodes/vectorstores/VectorStoreUtils.ts +++ b/packages/components/nodes/vectorstores/VectorStoreUtils.ts @@ -1,11 +1,19 @@ import { INodeData } from '../../src' +import { VectorStore } from '@langchain/core/vectorstores' -export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore: any) => { +export const resolveVectorStoreOrRetriever = ( + nodeData: INodeData, + vectorStore: VectorStore, + metadataFilter?: string | object | undefined +) => { const output = nodeData.outputs?.output as string const searchType = nodeData.outputs?.searchType as string const topK = nodeData.inputs?.topK as string const k = topK ? parseFloat(topK) : 4 + // If it is already pre-defined in lc_kwargs, then don't pass it again + const filter = vectorStore?.lc_kwargs?.filter ? undefined : metadataFilter + if (output === 'retriever') { if ('mmr' === searchType) { const fetchK = nodeData.inputs?.fetchK as string @@ -15,6 +23,7 @@ export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore: return vectorStore.asRetriever({ searchType: 'mmr', k: k, + filter, searchKwargs: { fetchK: f, lambda: l @@ -22,10 +31,11 @@ export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore: }) } else { // "searchType" is "similarity" - return vectorStore.asRetriever(k) + return vectorStore.asRetriever(k, filter) } } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + ;(vectorStore as any).filter = filter return vectorStore } } diff --git a/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts b/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts index 139a02de1..044baf2c6 100644 --- a/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts +++ b/packages/components/nodes/vectorstores/Weaviate/Weaviate.ts @@ -114,6 +114,13 @@ class Weaviate_VectorStores implements INode { type: 'number', additionalParams: true, optional: true + }, + { + label: 'Weaviate Search Filter', + name: 'weaviateFilter', + type: 'json', + additionalParams: true, + optional: true } ] addMMRInputParams(this.inputs) @@ -203,6 +210,7 @@ class Weaviate_VectorStores implements INode { const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string const embeddings = nodeData.inputs?.embeddings as Embeddings + let weaviateFilter = nodeData.inputs?.weaviateFilter const credentialData = await getCredentialData(nodeData.credential ?? '', options) const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData) @@ -223,10 +231,13 @@ class Weaviate_VectorStores implements INode { if (weaviateTextKey) obj.textKey = weaviateTextKey if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, '')) + if (weaviateFilter) { + weaviateFilter = typeof weaviateFilter === 'object' ? weaviateFilter : JSON.parse(weaviateFilter) + } const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj) - return resolveVectorStoreOrRetriever(nodeData, vectorStore) + return resolveVectorStoreOrRetriever(nodeData, vectorStore, weaviateFilter) } } diff --git a/packages/components/nodes/vectorstores/Zep/Zep.ts b/packages/components/nodes/vectorstores/Zep/Zep.ts index 0cddf4d12..6f94ca920 100644 --- a/packages/components/nodes/vectorstores/Zep/Zep.ts +++ b/packages/components/nodes/vectorstores/Zep/Zep.ts @@ -165,7 +165,7 @@ class Zep_VectorStores implements INode { const vectorStore = await ZepExistingVS.fromExistingIndex(embeddings, zepConfig) - return resolveVectorStoreOrRetriever(nodeData, vectorStore) + return resolveVectorStoreOrRetriever(nodeData, vectorStore, zepConfig.filter) } } diff --git a/packages/components/nodes/vectorstores/Zep/Zep_Existing.ts b/packages/components/nodes/vectorstores/Zep/Zep_Existing.ts index 58f02797e..2095cd5c1 100644 --- a/packages/components/nodes/vectorstores/Zep/Zep_Existing.ts +++ b/packages/components/nodes/vectorstores/Zep/Zep_Existing.ts @@ -125,6 +125,9 @@ class Zep_Existing_VectorStores implements INode { return retriever } else if (output === 'vectorStore') { ;(vectorStore as any).k = k + if (zepMetadataFilter) { + ;(vectorStore as any).filter = zepConfig.filter + } return vectorStore } return vectorStore diff --git a/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts b/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts index 85c32ca73..0d516ce88 100644 --- a/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts +++ b/packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts @@ -131,7 +131,7 @@ class Zep_CloudVectorStores implements INode { } zepConfig.client = await ZepClient.init(zepConfig.apiKey) const vectorStore = await ZepExistingVS.init(zepConfig) - return resolveVectorStoreOrRetriever(nodeData, vectorStore) + return resolveVectorStoreOrRetriever(nodeData, vectorStore, zepConfig.filter) } }