diff --git a/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts similarity index 73% rename from packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts rename to packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts index c11d8b115..6cb774144 100644 --- a/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts +++ b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts @@ -2,9 +2,10 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/I import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch' import { Embeddings } from 'langchain/embeddings/base' import { Document } from 'langchain/document' -import { Client } from '@opensearch-project/opensearch' +import { Client, RequestParams } from '@opensearch-project/opensearch' import { flatten } from 'lodash' import { getBaseClasses } from '../../../src/utils' +import { buildMetadataTerms } from './core' class OpenSearchUpsert_VectorStores implements INode { label: string @@ -95,9 +96,44 @@ class OpenSearchUpsert_VectorStores implements INode { const vectorStore = await OpenSearchVectorStore.fromDocuments(finalDocs, embeddings, { client, - indexName: indexName + indexName }) + vectorStore.similaritySearchVectorWithScore = async ( + query: number[], + k: number, + filter?: object | undefined + ): Promise<[Document, number][]> => { + const search: RequestParams.Search = { + index: indexName, + body: { + query: { + bool: { + filter: { bool: { must: buildMetadataTerms(filter) } }, + must: [ + { + knn: { + embedding: { vector: query, k } + } + } + ] + } + }, + size: k + } + } + + const { body } = await client.search(search) + + return body.hits.hits.map((hit: any) => [ + new Document({ + pageContent: hit._source.text, + metadata: hit._source.metadata + }), + hit._score + ]) + } + if (output === 'retriever') { const retriever = vectorStore.asRetriever(k) return retriever diff --git a/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts similarity index 62% rename from packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts rename to packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts index c8d09470a..aeab0e3f4 100644 --- a/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts +++ b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts @@ -1,8 +1,10 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch' import { Embeddings } from 'langchain/embeddings/base' -import { Client } from '@opensearch-project/opensearch' +import { Document } from 'langchain/document' +import { Client, RequestParams } from '@opensearch-project/opensearch' import { getBaseClasses } from '../../../src/utils' +import { buildMetadataTerms } from './core' class OpenSearch_Existing_VectorStores implements INode { label: string @@ -42,6 +44,13 @@ class OpenSearch_Existing_VectorStores implements INode { name: 'indexName', type: 'string' }, + { + label: 'OpenSearch Metadata Filter', + name: 'openSearchMetadataFilter', + type: 'json', + optional: true, + additionalParams: true + }, { label: 'Top K', name: 'topK', @@ -73,6 +82,7 @@ class OpenSearch_Existing_VectorStores implements INode { const output = nodeData.outputs?.output as string const topK = nodeData.inputs?.topK as string const k = topK ? parseFloat(topK) : 4 + const openSearchMetadataFilter = nodeData.inputs?.openSearchMetadataFilter const client = new Client({ nodes: [opensearchURL] @@ -83,6 +93,46 @@ class OpenSearch_Existing_VectorStores implements INode { indexName }) + vectorStore.similaritySearchVectorWithScore = async ( + query: number[], + k: number, + filter?: object | undefined + ): Promise<[Document, number][]> => { + if (openSearchMetadataFilter) { + const metadatafilter = + typeof openSearchMetadataFilter === 'object' ? openSearchMetadataFilter : JSON.parse(openSearchMetadataFilter) + filter = metadatafilter + } + const search: RequestParams.Search = { + index: indexName, + body: { + query: { + bool: { + filter: { bool: { must: buildMetadataTerms(filter) } }, + must: [ + { + knn: { + embedding: { vector: query, k } + } + } + ] + } + }, + size: k + } + } + + const { body } = await client.search(search) + + return body.hits.hits.map((hit: any) => [ + new Document({ + pageContent: hit._source.text, + metadata: hit._source.metadata + }), + hit._score + ]) + } + if (output === 'retriever') { const retriever = vectorStore.asRetriever(k) return retriever diff --git a/packages/components/nodes/vectorstores/OpenSearch/core.ts b/packages/components/nodes/vectorstores/OpenSearch/core.ts new file mode 100644 index 000000000..49a1d6e39 --- /dev/null +++ b/packages/components/nodes/vectorstores/OpenSearch/core.ts @@ -0,0 +1,8 @@ +export const buildMetadataTerms = (filter?: object): { term: Record }[] => { + if (filter == null) return [] + const result = [] + for (const [key, value] of Object.entries(filter)) { + result.push({ term: { [`metadata.${key}`]: value } }) + } + return result +} diff --git a/packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png b/packages/components/nodes/vectorstores/OpenSearch/opensearch.png similarity index 100% rename from packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png rename to packages/components/nodes/vectorstores/OpenSearch/opensearch.png diff --git a/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png b/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png deleted file mode 100644 index 3fdcfd3f0..000000000 Binary files a/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png and /dev/null differ