Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bugfix/Missing Filter for VectorStore to Document (#2285) 

add filter for vector store to document
This commit is contained in:
Henry Heng 2024-04-29 22:25:40 +01:00 committed by GitHub
parent c4eb75ddde
commit 2b1273ca31
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 67 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts
View File

@ -70,12 +70,19 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
const output = nodeData.outputs?.output as string
const topK = (vectorStore as any)?.k ?? 4
const _filter = (vectorStore as any)?.filter
const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK)
// If it is already pre-defined in lc_kwargs, then don't pass it again
const filter = vectorStore.lc_kwargs.filter ? undefined : _filter
if (vectorStore.lc_kwargs.filter) {
;(vectorStore as any).filter = vectorStore.lc_kwargs.filter
}
const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK, filter)
// eslint-disable-next-line no-console
console.log('\x1b[94m\x1b[1m\n*****VectorStore Documents*****\n\x1b[0m\x1b[0m')
// eslint-disable-next-line no-console
console.log(docs)
console.log(JSON.stringify(docs, null, 2))
if (output === 'document') {
let finaldocs = []

3
packages/components/nodes/vectorstores/Chroma/Chroma.ts
View File

@ -187,6 +187,9 @@ class Chroma_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (chromaMetadataFilter) {
;(vectorStore as any).filter = obj.filter
}
return vectorStore
}
return vectorStore

3
packages/components/nodes/vectorstores/Chroma/Chroma_Existing.ts
View File

@ -117,6 +117,9 @@ class Chroma_Existing_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (chromaMetadataFilter) {
;(vectorStore as any).filter = obj.filter
}
return vectorStore
}
return vectorStore

3
packages/components/nodes/vectorstores/Milvus/Milvus.ts
View File

@ -199,6 +199,9 @@ class Milvus_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (milvusFilter) {
;(vectorStore as any).filter = milvusFilter
}
return vectorStore
}
return vectorStore

3
packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts
View File

@ -189,6 +189,9 @@ class Milvus_Existing_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (milvusFilter) {
;(vectorStore as any).filter = milvusFilter
}
return vectorStore
}
return vectorStore

2
packages/components/nodes/vectorstores/Pinecone/Pinecone.ts
View File

@ -190,7 +190,7 @@ class Pinecone_VectorStores implements INode {
const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter)
}
}

3
packages/components/nodes/vectorstores/Qdrant/Qdrant.ts
View File

@ -354,6 +354,9 @@ class Qdrant_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (queryFilter) {
;(vectorStore as any).filter = retrieverConfig.filter
}
return vectorStore
}
return vectorStore

3
packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts
View File

@ -182,6 +182,9 @@ class Qdrant_Existing_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (queryFilter) {
;(vectorStore as any).filter = retrieverConfig.filter
}
return vectorStore
}
return vectorStore

2
packages/components/nodes/vectorstores/Supabase/Supabase.ts
View File

@ -186,7 +186,7 @@ class Supabase_VectorStores implements INode {
const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, obj)
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter)
}
}

3
packages/components/nodes/vectorstores/Supabase/Supabase_Exisiting.ts
View File

@ -119,6 +119,9 @@ class Supabase_Existing_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (supabaseMetadataFilter) {
;(vectorStore as any).filter = obj.filter
}
return vectorStore
}
return vectorStore

3
packages/components/nodes/vectorstores/Vectara/Vectara.ts
View File

@ -272,6 +272,9 @@ class Vectara_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (vectaraMetadataFilter) {
;(vectorStore as any).filter = vectaraFilter.filter
}
return vectorStore
}
return vectorStore

3
packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts
View File

@ -127,6 +127,9 @@ class VectaraExisting_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (vectaraMetadataFilter) {
;(vectorStore as any).filter = vectaraFilter.filter
}
return vectorStore
}
return vectorStore

14
packages/components/nodes/vectorstores/VectorStoreUtils.ts
View File

@ -1,11 +1,19 @@
import { INodeData } from '../../src'
import { VectorStore } from '@langchain/core/vectorstores'
export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore: any) => {
export const resolveVectorStoreOrRetriever = (
nodeData: INodeData,
vectorStore: VectorStore,
metadataFilter?: string | object | undefined
) => {
const output = nodeData.outputs?.output as string
const searchType = nodeData.outputs?.searchType as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseFloat(topK) : 4
// If it is already pre-defined in lc_kwargs, then don't pass it again
const filter = vectorStore?.lc_kwargs?.filter ? undefined : metadataFilter
if (output === 'retriever') {
if ('mmr' === searchType) {
const fetchK = nodeData.inputs?.fetchK as string
@ -15,6 +23,7 @@ export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore:
return vectorStore.asRetriever({
searchType: 'mmr',
k: k,
filter,
searchKwargs: {
fetchK: f,
lambda: l
@ -22,10 +31,11 @@ export const resolveVectorStoreOrRetriever = (nodeData: INodeData, vectorStore:
})
} else {
// "searchType" is "similarity"
return vectorStore.asRetriever(k)
return vectorStore.asRetriever(k, filter)
}
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
;(vectorStore as any).filter = filter
return vectorStore
}
}

13
packages/components/nodes/vectorstores/Weaviate/Weaviate.ts
View File

@ -114,6 +114,13 @@ class Weaviate_VectorStores implements INode {
type: 'number',
additionalParams: true,
optional: true
},
{
label: 'Weaviate Search Filter',
name: 'weaviateFilter',
type: 'json',
additionalParams: true,
optional: true
}
]
addMMRInputParams(this.inputs)
@ -203,6 +210,7 @@ class Weaviate_VectorStores implements INode {
const weaviateTextKey = nodeData.inputs?.weaviateTextKey as string
const weaviateMetadataKeys = nodeData.inputs?.weaviateMetadataKeys as string
const embeddings = nodeData.inputs?.embeddings as Embeddings
let weaviateFilter = nodeData.inputs?.weaviateFilter
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const weaviateApiKey = getCredentialParam('weaviateApiKey', credentialData, nodeData)
@ -223,10 +231,13 @@ class Weaviate_VectorStores implements INode {
if (weaviateTextKey) obj.textKey = weaviateTextKey
if (weaviateMetadataKeys) obj.metadataKeys = JSON.parse(weaviateMetadataKeys.replace(/\s/g, ''))
if (weaviateFilter) {
weaviateFilter = typeof weaviateFilter === 'object' ? weaviateFilter : JSON.parse(weaviateFilter)
}
const vectorStore = await WeaviateStore.fromExistingIndex(embeddings, obj)
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
return resolveVectorStoreOrRetriever(nodeData, vectorStore, weaviateFilter)
}
}

2
packages/components/nodes/vectorstores/Zep/Zep.ts
View File

@ -165,7 +165,7 @@ class Zep_VectorStores implements INode {
const vectorStore = await ZepExistingVS.fromExistingIndex(embeddings, zepConfig)
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
return resolveVectorStoreOrRetriever(nodeData, vectorStore, zepConfig.filter)
}
}

3
packages/components/nodes/vectorstores/Zep/Zep_Existing.ts
View File

@ -125,6 +125,9 @@ class Zep_Existing_VectorStores implements INode {
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
if (zepMetadataFilter) {
;(vectorStore as any).filter = zepConfig.filter
}
return vectorStore
}
return vectorStore

2
packages/components/nodes/vectorstores/ZepCloud/ZepCloud.ts
View File

@ -131,7 +131,7 @@ class Zep_CloudVectorStores implements INode {
}
zepConfig.client = await ZepClient.init(zepConfig.apiKey)
const vectorStore = await ZepExistingVS.init(zepConfig)
return resolveVectorStoreOrRetriever(nodeData, vectorStore)
return resolveVectorStoreOrRetriever(nodeData, vectorStore, zepConfig.filter)
}
}