diff --git a/package.json b/package.json index e123cc681..13db19d14 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flowise", - "version": "1.3.7", + "version": "1.3.8", "private": true, "homepage": "https://flowiseai.com", "workspaces": [ diff --git a/packages/components/credentials/ElasticsearchAPI.credential.ts b/packages/components/credentials/ElasticsearchAPI.credential.ts new file mode 100644 index 000000000..fbba76f40 --- /dev/null +++ b/packages/components/credentials/ElasticsearchAPI.credential.ts @@ -0,0 +1,31 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class ElectricsearchAPI implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Elasticsearch API' + this.name = 'elasticsearchApi' + this.version = 1.0 + this.description = + 'Refer to official guide on how to get an API Key from ElasticSearch' + this.inputs = [ + { + label: 'Elasticsearch Endpoint', + name: 'endpoint', + type: 'string' + }, + { + label: 'Elasticsearch API Key', + name: 'apiKey', + type: 'password' + } + ] + } +} + +module.exports = { credClass: ElectricsearchAPI } diff --git a/packages/components/credentials/ElectricsearchUserPassword.credential.ts b/packages/components/credentials/ElectricsearchUserPassword.credential.ts new file mode 100644 index 000000000..6c47f7b18 --- /dev/null +++ b/packages/components/credentials/ElectricsearchUserPassword.credential.ts @@ -0,0 +1,36 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class ElasticSearchUserPassword implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'ElasticSearch User Password' + this.name = 'elasticSearchUserPassword' + this.version = 1.0 + this.description = + 'Refer to official guide on how to get User Password from ElasticSearch' + this.inputs = [ + { + label: 'Cloud ID', + name: 'cloudId', + type: 'string' + }, + { + label: 'ElasticSearch User', + name: 'username', + type: 'string' + }, + { + label: 'ElasticSearch Password', + name: 'password', + type: 'password' + } + ] + } +} + +module.exports = { credClass: ElasticSearchUserPassword } diff --git a/packages/components/nodes/vectorstores/Elasticsearch/ElasticSearchBase.ts b/packages/components/nodes/vectorstores/Elasticsearch/ElasticSearchBase.ts new file mode 100644 index 000000000..59294b7ea --- /dev/null +++ b/packages/components/nodes/vectorstores/Elasticsearch/ElasticSearchBase.ts @@ -0,0 +1,193 @@ +import { + getBaseClasses, + getCredentialData, + getCredentialParam, + ICommonObject, + INodeData, + INodeOutputsValue, + INodeParams +} from '../../../src' +import { Client, ClientOptions } from '@elastic/elasticsearch' +import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch' +import { Embeddings } from 'langchain/embeddings/base' +import { VectorStore } from 'langchain/vectorstores/base' +import { Document } from 'langchain/document' + +export abstract class ElasticSearchBase { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + + protected constructor() { + this.type = 'Elasticsearch' + this.icon = 'elasticsearch.png' + this.category = 'Vector Stores' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['elasticsearchApi', 'elasticSearchUserPassword'] + } + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Index Name', + name: 'indexName', + placeholder: '', + type: 'string' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + }, + { + label: 'Similarity', + name: 'similarity', + description: 'Similarity measure used in Elasticsearch.', + type: 'options', + default: 'l2_norm', + options: [ + { + label: 'l2_norm', + name: 'l2_norm' + }, + { + label: 'dot_product', + name: 'dot_product' + }, + { + label: 'cosine', + name: 'cosine' + } + ], + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Elasticsearch Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Elasticsearch Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(ElasticVectorSearch)] + } + ] + } + + abstract constructVectorStore( + embeddings: Embeddings, + elasticSearchClientArgs: ElasticClientArgs, + docs: Document>[] | undefined + ): Promise + + async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document>[] | undefined): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const endPoint = getCredentialParam('endpoint', credentialData, nodeData) + const cloudId = getCredentialParam('cloudId', credentialData, nodeData) + const indexName = nodeData.inputs?.indexName as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const topK = nodeData.inputs?.topK as string + const similarityMeasure = nodeData.inputs?.similarityMeasure as string + const k = topK ? parseFloat(topK) : 4 + const output = nodeData.outputs?.output as string + + const elasticSearchClientArgs = this.prepareClientArgs(endPoint, cloudId, credentialData, nodeData, similarityMeasure, indexName) + + const vectorStore = await this.constructVectorStore(embeddings, elasticSearchClientArgs, docs) + + if (output === 'retriever') { + return vectorStore.asRetriever(k) + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } + + protected prepareConnectionOptions( + endPoint: string | undefined, + cloudId: string | undefined, + credentialData: ICommonObject, + nodeData: INodeData + ) { + let elasticSearchClientOptions: ClientOptions = {} + if (endPoint) { + let apiKey = getCredentialParam('apiKey', credentialData, nodeData) + elasticSearchClientOptions = { + node: endPoint, + auth: { + apiKey: apiKey + } + } + } else if (cloudId) { + let username = getCredentialParam('username', credentialData, nodeData) + let password = getCredentialParam('password', credentialData, nodeData) + elasticSearchClientOptions = { + cloud: { + id: cloudId + }, + auth: { + username: username, + password: password + } + } + } + return elasticSearchClientOptions + } + + protected prepareClientArgs( + endPoint: string | undefined, + cloudId: string | undefined, + credentialData: ICommonObject, + nodeData: INodeData, + similarityMeasure: string, + indexName: string + ) { + let elasticSearchClientOptions = this.prepareConnectionOptions(endPoint, cloudId, credentialData, nodeData) + let vectorSearchOptions = {} + switch (similarityMeasure) { + case 'dot_product': + vectorSearchOptions = { + similarity: 'dot_product' + } + break + case 'cosine': + vectorSearchOptions = { + similarity: 'cosine' + } + break + default: + vectorSearchOptions = { + similarity: 'l2_norm' + } + } + const elasticSearchClientArgs: ElasticClientArgs = { + client: new Client(elasticSearchClientOptions), + indexName: indexName, + vectorSearchOptions: vectorSearchOptions + } + return elasticSearchClientArgs + } +} diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts new file mode 100644 index 000000000..94e45d742 --- /dev/null +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts @@ -0,0 +1,31 @@ +import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' + +import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch' +import { ElasticSearchBase } from './ElasticSearchBase' +import { VectorStore } from 'langchain/vectorstores/base' +import { Document } from 'langchain/document' + +class ElasicsearchExisting_VectorStores extends ElasticSearchBase implements INode { + constructor() { + super() + this.label = 'Elasticsearch Load Existing Index' + this.name = 'ElasticsearchIndex' + this.version = 1.0 + this.description = 'Load existing index from Elasticsearch (i.e: Document has been upserted)' + } + + async constructVectorStore( + embeddings: Embeddings, + elasticSearchClientArgs: ElasticClientArgs, + docs: Document>[] | undefined + ): Promise { + return await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs) + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + return super.init(nodeData, _, options, undefined) + } +} + +module.exports = { nodeClass: ElasicsearchExisting_VectorStores } diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts new file mode 100644 index 000000000..d4b79a5dd --- /dev/null +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts @@ -0,0 +1,55 @@ +import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' + +import { ElasticClientArgs, ElasticVectorSearch } from 'langchain/vectorstores/elasticsearch' +import { flatten } from 'lodash' +import { ElasticSearchBase } from './ElasticSearchBase' +import { VectorStore } from 'langchain/vectorstores/base' + +class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode { + constructor() { + super() + this.label = 'Elasticsearch Upsert Document' + this.name = 'ElasticsearchUpsert' + this.version = 1.0 + this.description = 'Upsert documents to Elasticsearch' + this.inputs.unshift({ + label: 'Document', + name: 'document', + type: 'Document', + list: true + }) + } + + async constructVectorStore( + embeddings: Embeddings, + elasticSearchClientArgs: ElasticClientArgs, + docs: Document>[] + ): Promise { + const vectorStore = new ElasticVectorSearch(embeddings, elasticSearchClientArgs) + await vectorStore.addDocuments(docs) + return vectorStore + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const docs = nodeData.inputs?.document as Document[] + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + finalDocs.push(new Document(flattenDocs[i])) + } + + // The following code is a workaround for a bug (Langchain Issue #1589) in the underlying library. + // Store does not support object in metadata and fail silently + finalDocs.forEach((d) => { + delete d.metadata.pdf + delete d.metadata.loc + }) + // end of workaround + return super.init(nodeData, _, options, flattenDocs) + } +} + +module.exports = { nodeClass: ElasicsearchUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png b/packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png new file mode 100644 index 000000000..fdb668636 Binary files /dev/null and b/packages/components/nodes/vectorstores/Elasticsearch/elasticsearch.png differ diff --git a/packages/components/nodes/vectorstores/Pinecone_Existing/Pinecone_Existing.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone_Existing.ts similarity index 97% rename from packages/components/nodes/vectorstores/Pinecone_Existing/Pinecone_Existing.ts rename to packages/components/nodes/vectorstores/Pinecone/Pinecone_Existing.ts index 2369165d8..e8536d8d9 100644 --- a/packages/components/nodes/vectorstores/Pinecone_Existing/Pinecone_Existing.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone_Existing.ts @@ -1,5 +1,5 @@ import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' -import { PineconeClient } from '@pinecone-database/pinecone' +import { Pinecone } from '@pinecone-database/pinecone' import { PineconeLibArgs, PineconeStore } from 'langchain/vectorstores/pinecone' import { Embeddings } from 'langchain/embeddings/base' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' @@ -95,8 +95,7 @@ class Pinecone_Existing_VectorStores implements INode { const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData) const pineconeEnv = getCredentialParam('pineconeEnv', credentialData, nodeData) - const client = new PineconeClient() - await client.init({ + const client = new Pinecone({ apiKey: pineconeApiKey, environment: pineconeEnv }) diff --git a/packages/components/nodes/vectorstores/Pinecone_Upsert/Pinecone_Upsert.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts similarity index 97% rename from packages/components/nodes/vectorstores/Pinecone_Upsert/Pinecone_Upsert.ts rename to packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts index 3d2a6497d..4a12f27b2 100644 --- a/packages/components/nodes/vectorstores/Pinecone_Upsert/Pinecone_Upsert.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts @@ -1,5 +1,5 @@ import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' -import { PineconeClient } from '@pinecone-database/pinecone' +import { Pinecone } from '@pinecone-database/pinecone' import { PineconeLibArgs, PineconeStore } from 'langchain/vectorstores/pinecone' import { Embeddings } from 'langchain/embeddings/base' import { Document } from 'langchain/document' @@ -96,8 +96,7 @@ class PineconeUpsert_VectorStores implements INode { const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData) const pineconeEnv = getCredentialParam('pineconeEnv', credentialData, nodeData) - const client = new PineconeClient() - await client.init({ + const client = new Pinecone({ apiKey: pineconeApiKey, environment: pineconeEnv }) diff --git a/packages/components/nodes/vectorstores/Pinecone_Existing/pinecone.png b/packages/components/nodes/vectorstores/Pinecone/pinecone.png similarity index 100% rename from packages/components/nodes/vectorstores/Pinecone_Existing/pinecone.png rename to packages/components/nodes/vectorstores/Pinecone/pinecone.png diff --git a/packages/components/nodes/vectorstores/Pinecone_Upsert/pinecone.png b/packages/components/nodes/vectorstores/Pinecone_Upsert/pinecone.png deleted file mode 100644 index 1ae189fdc..000000000 Binary files a/packages/components/nodes/vectorstores/Pinecone_Upsert/pinecone.png and /dev/null differ diff --git a/packages/components/package.json b/packages/components/package.json index 830e03816..262a49db9 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -1,6 +1,6 @@ { "name": "flowise-components", - "version": "1.3.8", + "version": "1.3.10", "description": "Flowiseai Components", "main": "dist/src/index", "types": "dist/src/index.d.ts", @@ -20,13 +20,14 @@ "@aws-sdk/client-dynamodb": "^3.360.0", "@aws-sdk/client-s3": "^3.427.0", "@dqbd/tiktoken": "^1.0.7", + "@elastic/elasticsearch": "^8.9.0", "@getzep/zep-js": "^0.6.3", "@gomomento/sdk": "^1.40.2", "@google-ai/generativelanguage": "^0.2.1", "@huggingface/inference": "^2.6.1", "@notionhq/client": "^2.2.8", "@opensearch-project/opensearch": "^1.2.0", - "@pinecone-database/pinecone": "^0.0.14", + "@pinecone-database/pinecone": "^1.1.1", "@qdrant/js-client-rest": "^1.2.2", "@supabase/supabase-js": "^2.29.0", "@types/js-yaml": "^4.0.5", diff --git a/packages/components/src/handler.ts b/packages/components/src/handler.ts index 3b1952d6d..236b423f6 100644 --- a/packages/components/src/handler.ts +++ b/packages/components/src/handler.ts @@ -238,9 +238,6 @@ export const additionalCallbacks = async (nodeData: INodeData, options: ICommonO }) callbacks.push(tracer) } else if (provider === 'langFuse') { - const flushAt = analytic[provider].flushAt as string - const flushInterval = analytic[provider].flushInterval as string - const requestTimeout = analytic[provider].requestTimeout as string const release = analytic[provider].release as string const langFuseSecretKey = getCredentialParam('langFuseSecretKey', credentialData, nodeData) @@ -252,9 +249,6 @@ export const additionalCallbacks = async (nodeData: INodeData, options: ICommonO publicKey: langFusePublicKey, baseUrl: langFuseEndpoint ?? 'https://cloud.langfuse.com' } - if (flushAt) langFuseOptions.flushAt = parseInt(flushAt, 10) - if (flushInterval) langFuseOptions.flushInterval = parseInt(flushInterval, 10) - if (requestTimeout) langFuseOptions.requestTimeout = parseInt(requestTimeout, 10) if (release) langFuseOptions.release = release const handler = new CallbackHandler(langFuseOptions) diff --git a/packages/server/package.json b/packages/server/package.json index a5fb994c7..eadedea07 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "flowise", - "version": "1.3.7", + "version": "1.3.8", "description": "Flowiseai Server", "main": "dist/index", "types": "dist/index.d.ts", diff --git a/packages/ui/package.json b/packages/ui/package.json index 239cc3ceb..41ad515f6 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -1,6 +1,6 @@ { "name": "flowise-ui", - "version": "1.3.5", + "version": "1.3.6", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://flowiseai.com", "author": { diff --git a/packages/ui/src/ui-component/dialog/AnalyseFlowDialog.js b/packages/ui/src/ui-component/dialog/AnalyseFlowDialog.js index 2d9a7d91f..dd6bb8ab9 100644 --- a/packages/ui/src/ui-component/dialog/AnalyseFlowDialog.js +++ b/packages/ui/src/ui-component/dialog/AnalyseFlowDialog.js @@ -81,27 +81,6 @@ const analyticProviders = [ type: 'credential', credentialNames: ['langfuseApi'] }, - { - label: 'Flush At', - name: 'flushAt', - type: 'number', - optional: true, - description: 'Number of queued requests' - }, - { - label: 'Flush Interval', - name: 'flushInterval', - type: 'number', - optional: true, - description: 'Interval in ms to flush requests' - }, - { - label: 'Request Timeout', - name: 'requestTimeout', - type: 'number', - optional: true, - description: 'Timeout in ms for requests' - }, { label: 'Release', name: 'release',