diff --git a/packages/components/credentials/ElasticsearchAPI.credential.ts b/packages/components/credentials/ElasticsearchAPI.credential.ts new file mode 100644 index 000000000..e377243d4 --- /dev/null +++ b/packages/components/credentials/ElasticsearchAPI.credential.ts @@ -0,0 +1,31 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class ElectricsearchAPI implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Elasticsearch API' + this.name = 'elasticsearchApi' + this.version = 1.0 + this.description = + 'Refer to official guide on how to get an API Key from ElasticSearch' + this.inputs = [ + { + label: 'Elasticsearch Endpoint', + name: 'endpoint', + type: 'string' + }, + { + label: 'Elasticsearch API ID', + name: 'apiKey', + type: 'password' + } + ] + } +} + +module.exports = { credClass: ElectricsearchAPI } diff --git a/packages/components/credentials/ElectricsearchUserPassword.credential.ts b/packages/components/credentials/ElectricsearchUserPassword.credential.ts new file mode 100644 index 000000000..2dd889373 --- /dev/null +++ b/packages/components/credentials/ElectricsearchUserPassword.credential.ts @@ -0,0 +1,31 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class ElasticSearchUserPassword implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'ElasticSearch User Password' + this.name = 'elasticSearchUserPassword' + this.version = 1.0 + this.description = + 'Refer to official guide on how to get User Password from ElasticSearch' + this.inputs = [ + { + label: 'ElasticSearch User', + name: 'elasticSearchUser', + type: 'string' + }, + { + label: 'ElasticSearch Password', + name: 'elasticSearchPassword', + type: 'password' + } + ] + } +} + +module.exports = { credClass: ElasticSearchUserPassword } diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts new file mode 100644 index 000000000..6e785c857 --- /dev/null +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts @@ -0,0 +1,111 @@ +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src' + +import { Client, ClientOptions } from '@elastic/elasticsearch' +import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch' + +class ElasicsearchExisting_VectorStores implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Elasticsearch Load Existing Index' + this.name = 'ElasticsearchIndex' + this.version = 1.0 + this.type = 'Elasticsearch' + this.icon = 'elasticsearch.png' + this.category = 'Vector Stores' + this.description = 'Load existing index from Elasticsearch (i.e: Document has been upserted)' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword'] + } + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Index Name', + name: 'indexName', + placeholder: '', + type: 'string' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Elasticsearch Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Elasticsearch Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)] + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const endPoint = getCredentialParam('endpoint', credentialData, nodeData) + const apiKey = getCredentialParam('apiKey', credentialData, nodeData) + const indexName = nodeData.inputs?.indexName as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const topK = nodeData.inputs?.topK as string + + const k = topK ? parseFloat(topK) : 4 + const output = nodeData.outputs?.output as string + + // eslint-disable-next-line no-console + console.log('EndPoint:: ' + endPoint + ', APIKey:: ' + apiKey + ', Index:: ' + indexName) + + const elasticSearchClientOptions: ClientOptions = { + node: endPoint, + auth: { + apiKey: apiKey + } + } + + const elasticSearchClientArgs: ElasticClientArgs = { + client: new Client(elasticSearchClientOptions), + indexName: indexName + } + + const vectorStore = await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs) + // eslint-disable-next-line no-console + console.log('vectorStore ::' + vectorStore._vectorstoreType()) + if (output === 'retriever') { + return vectorStore.asRetriever(k) + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: ElasicsearchExisting_VectorStores } diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts new file mode 100644 index 000000000..5a0065d54 --- /dev/null +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts @@ -0,0 +1,165 @@ +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src' + +import { Client, ClientOptions } from '@elastic/elasticsearch' +import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch' +import { flatten } from 'lodash' + +class ElasicsearchUpsert_VectorStores implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Elasticsearch Upsert Document' + this.name = 'ElasticsearchUpsert' + this.version = 1.0 + this.type = 'Elasticsearch' + this.icon = 'elasticsearch.png' + this.category = 'Vector Stores' + this.description = 'Upsert documents to Elasticsearch' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword'] + } + this.inputs = [ + { + label: 'Document', + name: 'document', + type: 'Document', + list: true + }, + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Index Name', + name: 'indexName', + placeholder: '', + type: 'string' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + }, + { + label: 'Similarity', + name: 'similarity', + description: 'Similarity measure used in Elasticsearch.', + type: 'options', + default: 'l2_norm', + options: [ + { + label: 'l2_norm', + name: 'l2_norm' + }, + { + label: 'dot_product', + name: 'dot_product' + }, + { + label: 'cosine', + name: 'cosine' + } + ], + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Elasticsearch Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Elasticsearch Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)] + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const endPoint = getCredentialParam('endpoint', credentialData, nodeData) + const apiKey = getCredentialParam('apiKey', credentialData, nodeData) + const docs = nodeData.inputs?.document as Document[] + const indexName = nodeData.inputs?.indexName as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const topK = nodeData.inputs?.topK as string + const k = topK ? parseFloat(topK) : 4 + const output = nodeData.outputs?.output as string + const similarityMeasure = nodeData.inputs?.similarityMeasure as string + + // eslint-disable-next-line no-console + console.log('EndPoint:: ' + endPoint + ', APIKey:: ' + apiKey + ', Index:: ' + indexName) + + const elasticSearchClientOptions: ClientOptions = { + node: endPoint, + auth: { + apiKey: apiKey + } + } + let vectorSearchOptions = {} + switch (similarityMeasure) { + case 'dot_product': + vectorSearchOptions = { + similarity: 'dot_product' + } + break + case 'cosine': + vectorSearchOptions = { + similarity: 'cosine' + } + break + default: + vectorSearchOptions = { + similarity: 'l2_norm' + } + } + const elasticSearchClientArgs: ElasticClientArgs = { + client: new Client(elasticSearchClientOptions), + indexName: indexName, + vectorSearchOptions: vectorSearchOptions + } + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + finalDocs.push(new Document(flattenDocs[i])) + } + + const vectorStore = await ElasticVectorSearch.fromDocuments(finalDocs, embeddings, elasticSearchClientArgs) + + if (output === 'retriever') { + return vectorStore.asRetriever(k) + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: ElasicsearchUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png b/packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png new file mode 100644 index 000000000..fdb668636 Binary files /dev/null and b/packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png differ diff --git a/packages/components/package.json b/packages/components/package.json index 936091062..3fdd8923a 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -19,6 +19,7 @@ "@aws-sdk/client-dynamodb": "^3.360.0", "@dqbd/tiktoken": "^1.0.7", "@getzep/zep-js": "^0.6.3", + "@elastic/elasticsearch": "^8.9.0", "@google-ai/generativelanguage": "^0.2.1", "@huggingface/inference": "^2.6.1", "@notionhq/client": "^2.2.8",