Merge branch 'main' into feature/ChatHistory2

CONTRIBUTING-ZH.md

@@ -119,7 +119,7 @@ Flowise 在一个单一的单体存储库中有 3 个不同的模块。
 Flowise 支持不同的环境变量来配置您的实例。您可以在 `packages/server` 文件夹中的 `.env` 文件中指定以下变量。阅读[更多信息](https://docs.flowiseai.com/environment-variables)
 
 | 变量名                      | 描述                                                   | 类型                                            | 默认值                              |
-| -------------------------- | ------------------------------------------------------ | ----------------------------------------------- | ----------------------------------- |
+| --------------------------- | ------------------------------------------------------ | ----------------------------------------------- | ----------------------------------- |
 | PORT                        | Flowise 运行的 HTTP 端口                               | 数字                                            | 3000                                |
 | FLOWISE_USERNAME            | 登录用户名                                             | 字符串                                          |                                     |
 | FLOWISE_PASSWORD            | 登录密码                                               | 字符串                                          |                                     |
@@ -129,7 +129,6 @@ Flowise 支持不同的环境变量来配置您的实例。您可以在 `package
 | APIKEY_PATH                 | 存储 API 密钥的位置                                    | 字符串                                          | `your-path/Flowise/packages/server` |
 | TOOL_FUNCTION_BUILTIN_DEP   | 用于工具函数的 NodeJS 内置模块                         | 字符串                                          |                                     |
 | TOOL_FUNCTION_EXTERNAL_DEP  | 用于工具函数的外部模块                                 | 字符串                                          |                                     |
-| OVERRIDE_DATABASE          | 是否使用默认值覆盖当前数据库                           | 枚举字符串: `true`, `false`                     | `true`                              |
 | DATABASE_TYPE               | 存储 flowise 数据的数据库类型                          | 枚举字符串: `sqlite`, `mysql`, `postgres`       | `sqlite`                            |
 | DATABASE_PATH               | 数据库保存的位置（当 DATABASE_TYPE 是 sqlite 时）      | 字符串                                          | `your-home-dir/.flowise`            |
 | DATABASE_HOST               | 主机 URL 或 IP 地址（当 DATABASE_TYPE 不是 sqlite 时） | 字符串                                          |                                     |
@@ -137,6 +136,8 @@ Flowise 支持不同的环境变量来配置您的实例。您可以在 `package
 | DATABASE_USERNAME           | 数据库用户名（当 DATABASE_TYPE 不是 sqlite 时）        | 字符串                                          |                                     |
 | DATABASE_PASSWORD           | 数据库密码（当 DATABASE_TYPE 不是 sqlite 时）          | 字符串                                          |                                     |
 | DATABASE_NAME               | 数据库名称（当 DATABASE_TYPE 不是 sqlite 时）          | 字符串                                          |                                     |
+| SECRETKEY_PATH              | 保存加密密钥（用于加密/解密凭据）的位置                | 字符串                                          | `your-path/Flowise/packages/server` |
+| FLOWISE_SECRETKEY_OVERWRITE | 加密密钥用于替代存储在 SECRETKEY_PATH 中的密钥         | 字符串                                          |
 
 您也可以在使用 `npx` 时指定环境变量。例如：
 

CONTRIBUTING.md

@@ -121,7 +121,7 @@ Flowise has 3 different modules in a single mono repository.
 Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. Read [more](https://docs.flowiseai.com/environment-variables)
 
 | Variable                    | Description                                                                  | Type                                             | Default                             |
-| -------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
+| --------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | --- |
 | PORT                        | The HTTP port Flowise runs on                                                | Number                                           | 3000                                |
 | FLOWISE_USERNAME            | Username to login                                                            | String                                           |                                     |
 | FLOWISE_PASSWORD            | Password to login                                                            | String                                           |                                     |
@@ -130,8 +130,7 @@ Flowise support different environment variables to configure your instance. You
 | LOG_LEVEL                   | Different levels of logs                                                     | Enum String: `error`, `info`, `verbose`, `debug` | `info`                              |
 | APIKEY_PATH                 | Location where api keys are saved                                            | String                                           | `your-path/Flowise/packages/server` |
 | TOOL_FUNCTION_BUILTIN_DEP   | NodeJS built-in modules to be used for Tool Function                         | String                                           |                                     |
-| TOOL_FUNCTION_EXTERNAL_DEP | External modules to be used for Tool Function                                | String                                           |                                     |
+| TOOL_FUNCTION_EXTERNAL_DEP  | External modules to be used for Tool Function                                | String                                           |                                     |     |
-| OVERRIDE_DATABASE          | Override current database with default                                       | Enum String: `true`, `false`                     | `true`                              |
 | DATABASE_TYPE               | Type of database to store the flowise data                                   | Enum String: `sqlite`, `mysql`, `postgres`       | `sqlite`                            |
 | DATABASE_PATH               | Location where database is saved (When DATABASE_TYPE is sqlite)              | String                                           | `your-home-dir/.flowise`            |
 | DATABASE_HOST               | Host URL or IP address (When DATABASE_TYPE is not sqlite)                    | String                                           |                                     |
@@ -139,8 +138,8 @@ Flowise support different environment variables to configure your instance. You
 | DATABASE_USER               | Database username (When DATABASE_TYPE is not sqlite)                         | String                                           |                                     |
 | DATABASE_PASSWORD           | Database password (When DATABASE_TYPE is not sqlite)                         | String                                           |                                     |
 | DATABASE_NAME               | Database name (When DATABASE_TYPE is not sqlite)                             | String                                           |                                     |
-| PASSPHRASE                 | Passphrase used to create encryption key                                     | String                                           | `MYPASSPHRASE`                      |
 | SECRETKEY_PATH              | Location where encryption key (used to encrypt/decrypt credentials) is saved | String                                           | `your-path/Flowise/packages/server` |
+| FLOWISE_SECRETKEY_OVERWRITE | Encryption key to be used instead of the key stored in SECRETKEY_PATH        | String                                           |
 
 You can also specify the env variables when using `npx`. For example:
 

docker/.env.example

@@ -1,5 +1,4 @@
 PORT=3000
-PASSPHRASE=MYPASSPHRASE # Passphrase used to create encryption key
 DATABASE_PATH=/root/.flowise
 APIKEY_PATH=/root/.flowise
 SECRETKEY_PATH=/root/.flowise
@@ -13,10 +12,10 @@ LOG_PATH=/root/.flowise/logs
 # DATABASE_NAME="flowise"
 # DATABASE_USER=""
 # DATABASE_PASSWORD=""
-# OVERRIDE_DATABASE=true
 
 # FLOWISE_USERNAME=user
 # FLOWISE_PASSWORD=1234
+# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
 # DEBUG=true
 # LOG_LEVEL=debug (error | warn | info | verbose | debug)
 # TOOL_FUNCTION_BUILTIN_DEP=crypto,fs

docker/docker-compose.yml

@@ -6,13 +6,13 @@ services:
         restart: always
         environment:
             - PORT=${PORT}
-            - PASSPHRASE=${PASSPHRASE}
             - FLOWISE_USERNAME=${FLOWISE_USERNAME}
             - FLOWISE_PASSWORD=${FLOWISE_PASSWORD}
             - DEBUG=${DEBUG}
             - DATABASE_PATH=${DATABASE_PATH}
             - APIKEY_PATH=${APIKEY_PATH}
             - SECRETKEY_PATH=${SECRETKEY_PATH}
+            - FLOWISE_SECRETKEY_OVERWRITE=${FLOWISE_SECRETKEY_OVERWRITE}
             - LOG_LEVEL=${LOG_LEVEL}
             - LOG_PATH=${LOG_PATH}
         ports:

package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise",
-    "version": "1.3.4",
+    "version": "1.3.5",
     "private": true,
     "homepage": "https://flowiseai.com",
     "workspaces": [

packages/components/nodes/agents/BabyAGI/BabyAGI.ts

@@ -1,7 +1,7 @@
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { BabyAGI } from './core'
 import { BaseChatModel } from 'langchain/chat_models/base'
-import { VectorStore } from 'langchain/vectorstores'
+import { VectorStore } from 'langchain/vectorstores/base'
 
 class BabyAGI_Agents implements INode {
     label: string

packages/components/nodes/chains/VectorDBQAChain/VectorDBQAChain.ts

@@ -2,7 +2,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter
 import { getBaseClasses } from '../../../src/utils'
 import { VectorDBQAChain } from 'langchain/chains'
 import { BaseLanguageModel } from 'langchain/base_language'
-import { VectorStore } from 'langchain/vectorstores'
+import { VectorStore } from 'langchain/vectorstores/base'
 import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
 
 class VectorDBQAChain_Chains implements INode {

packages/components/nodes/vectorstores/Milvus/Milvus_Existing.ts

@@ -21,7 +21,7 @@ class Milvus_Existing_VectorStores implements INode {
     constructor() {
         this.label = 'Milvus Load Existing collection'
         this.name = 'milvusExistingCollection'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Milvus'
         this.icon = 'milvus.svg'
         this.category = 'Vector Stores'
@@ -50,6 +50,25 @@ class Milvus_Existing_VectorStores implements INode {
                 label: 'Milvus Collection Name',
                 name: 'milvusCollection',
                 type: 'string'
+            },
+            {
+                label: 'Milvus Filter',
+                name: 'milvusFilter',
+                type: 'string',
+                optional: true,
+                description:
+                    'Filter data with a simple string query. Refer Milvus <a target="_blank" href="https://milvus.io/blog/2022-08-08-How-to-use-string-data-to-empower-your-similarity-search-applications.md#Hybrid-search">docs</a> for more details.',
+                placeholder: 'doc=="a"',
+                additionalParams: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
             }
         ]
         this.outputs = [
@@ -70,6 +89,7 @@ class Milvus_Existing_VectorStores implements INode {
         // server setup
         const address = nodeData.inputs?.milvusServerUrl as string
         const collectionName = nodeData.inputs?.milvusCollection as string
+        const milvusFilter = nodeData.inputs?.milvusFilter as string
 
         // embeddings
         const embeddings = nodeData.inputs?.embeddings as Embeddings
@@ -109,7 +129,7 @@ class Milvus_Existing_VectorStores implements INode {
                 throw new Error(`Collection not found: ${vectorStore.collectionName}, please create collection before search.`)
             }
 
-            const filterStr = filter ?? ''
+            const filterStr = milvusFilter ?? filter ?? ''
 
             await vectorStore.grabCollectionFields()
 

packages/components/nodes/vectorstores/Vectara/Vectara_Existing.ts

@@ -92,7 +92,7 @@ class VectaraExisting_VectorStores implements INode {
         const credentialData = await getCredentialData(nodeData.credential ?? '', options)
         const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
         const customerId = getCredentialParam('customerID', credentialData, nodeData)
-        const corpusId = getCredentialParam('corpusID', credentialData, nodeData)
+        const corpusId = getCredentialParam('corpusID', credentialData, nodeData).split(',')
 
         const vectaraMetadataFilter = nodeData.inputs?.filter as string
         const sentencesBefore = nodeData.inputs?.sentencesBefore as number

packages/components/nodes/vectorstores/Vectara/Vectara_Upload.ts

@@ -0,0 +1,176 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig, VectaraFile } from 'langchain/vectorstores/vectara'
+
+class VectaraUpload_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Vectara Upload File'
+        this.name = 'vectaraUpload'
+        this.version = 1.0
+        this.type = 'Vectara'
+        this.icon = 'vectara.png'
+        this.category = 'Vector Stores'
+        this.description = 'Upload files to Vectara'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['vectaraApi']
+        }
+        this.inputs = [
+            {
+                label: 'File',
+                name: 'file',
+                description:
+                    'File to upload to Vectara. Supported file types: https://docs.vectara.com/docs/api-reference/indexing-apis/file-upload/file-upload-filetypes',
+                type: 'file'
+            },
+            {
+                label: 'Vectara Metadata Filter',
+                name: 'filter',
+                description:
+                    'Filter to apply to Vectara metadata. Refer to the <a target="_blank" href="https://docs.flowiseai.com/vector-stores/vectara">documentation</a> on how to use Vectara filters with Flowise.',
+                type: 'string',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Sentences Before',
+                name: 'sentencesBefore',
+                description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Sentences After',
+                name: 'sentencesAfter',
+                description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Lambda',
+                name: 'lambda',
+                description:
+                    'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Defaults to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Vectara Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Vectara Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
+            }
+        ]
+    }
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+        const customerId = getCredentialParam('customerID', credentialData, nodeData)
+        const corpusId = getCredentialParam('corpusID', credentialData, nodeData).split(',')
+
+        const fileBase64 = nodeData.inputs?.file
+        const vectaraMetadataFilter = nodeData.inputs?.filter as string
+        const sentencesBefore = nodeData.inputs?.sentencesBefore as number
+        const sentencesAfter = nodeData.inputs?.sentencesAfter as number
+        const lambda = nodeData.inputs?.lambda as number
+        const output = nodeData.outputs?.output as string
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseInt(topK, 10) : 4
+
+        const vectaraArgs: VectaraLibArgs = {
+            apiKey: apiKey,
+            customerId: customerId,
+            corpusId: corpusId
+        }
+
+        const vectaraFilter: VectaraFilter = {}
+        if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
+        if (lambda) vectaraFilter.lambda = lambda
+
+        const vectaraContextConfig: VectaraContextConfig = {}
+        if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
+        if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
+        vectaraFilter.contextConfig = vectaraContextConfig
+
+        let files: string[] = []
+
+        if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+            files = JSON.parse(fileBase64)
+        } else {
+            files = [fileBase64]
+        }
+
+        const vectaraFiles: VectaraFile[] = []
+        for (const file of files) {
+            const splitDataURI = file.split(',')
+            splitDataURI.pop()
+            const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+            const blob = new Blob([bf])
+            vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
+        }
+
+        const vectorStore = new VectaraStore(vectaraArgs)
+        await vectorStore.addFiles(vectaraFiles)
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k, vectaraFilter)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+const getFileName = (fileBase64: string) => {
+    let fileNames = []
+    if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
+        const files = JSON.parse(fileBase64)
+        for (const file of files) {
+            const splitDataURI = file.split(',')
+            const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
+            fileNames.push(filename)
+        }
+        return fileNames.join(', ')
+    } else {
+        const splitDataURI = fileBase64.split(',')
+        const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
+        return filename
+    }
+}
+
+module.exports = { nodeClass: VectaraUpload_VectorStores }

packages/components/nodes/vectorstores/Vectara/Vectara_Upsert.ts

@@ -101,7 +101,7 @@ class VectaraUpsert_VectorStores implements INode {
         const credentialData = await getCredentialData(nodeData.credential ?? '', options)
         const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
         const customerId = getCredentialParam('customerID', credentialData, nodeData)
-        const corpusId = getCredentialParam('corpusID', credentialData, nodeData)
+        const corpusId = getCredentialParam('corpusID', credentialData, nodeData).split(',')
 
         const docs = nodeData.inputs?.document as Document[]
         const embeddings = {} as Embeddings

packages/components/package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise-components",
-    "version": "1.3.4",
+    "version": "1.3.5",
     "description": "Flowiseai Components",
     "main": "dist/src/index",
     "types": "dist/src/index.d.ts",
@@ -42,7 +42,7 @@
         "google-auth-library": "^9.0.0",
         "graphql": "^16.6.0",
         "html-to-text": "^9.0.5",
-        "langchain": "^0.0.145",
+        "langchain": "^0.0.147",
         "langfuse-langchain": "^1.0.14-alpha.0",
         "langsmith": "^0.0.32",
         "linkifyjs": "^4.1.1",

packages/components/src/utils.ts

@@ -396,6 +396,9 @@ const getEncryptionKeyPath = (): string => {
  * @returns {Promise<string>}
  */
 const getEncryptionKey = async (): Promise<string> => {
+    if (process.env.FLOWISE_SECRETKEY_OVERWRITE !== undefined && process.env.FLOWISE_SECRETKEY_OVERWRITE !== '') {
+        return process.env.FLOWISE_SECRETKEY_OVERWRITE
+    }
     try {
         return await fs.promises.readFile(getEncryptionKeyPath(), 'utf8')
     } catch (error) {

packages/server/.env.example

@@ -1,5 +1,4 @@
 PORT=3000
-PASSPHRASE=MYPASSPHRASE # Passphrase used to create encryption key
 # DATABASE_PATH=/your_database_path/.flowise
 # APIKEY_PATH=/your_api_key_path/.flowise
 # SECRETKEY_PATH=/your_api_key_path/.flowise
@@ -13,10 +12,10 @@ PASSPHRASE=MYPASSPHRASE # Passphrase used to create encryption key
 # DATABASE_NAME="flowise"
 # DATABASE_USER=""
 # DATABASE_PASSWORD=""
-# OVERRIDE_DATABASE=true
 
 # FLOWISE_USERNAME=user
 # FLOWISE_PASSWORD=1234
+# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey
 # DEBUG=true
 # LOG_LEVEL=debug (error | warn | info | verbose | debug)
 # TOOL_FUNCTION_BUILTIN_DEP=crypto,fs

packages/server/README-ZH.md

@@ -35,26 +35,6 @@ FLOWISE_PASSWORD=1234
 
 Flowise 支持不同的环境变量来配置您的实例。您可以在`packages/server`文件夹中的`.env`文件中指定以下变量。阅读[更多](https://docs.flowiseai.com/environment-variables)
 
-| 变量                       | 描述                                                   | 类型                                            | 默认值                              |
-| -------------------------- | ------------------------------------------------------ | ----------------------------------------------- | ----------------------------------- |
-| PORT                       | Flowise 运行的 HTTP 端口                               | 数字                                            | 3000                                |
-| FLOWISE_USERNAME           | 登录的用户名                                           | 字符串                                          |                                     |
-| FLOWISE_PASSWORD           | 登录的密码                                             | 字符串                                          |                                     |
-| DEBUG                      | 打印组件的日志                                         | 布尔值                                          |                                     |
-| LOG_PATH                   | 存储日志文件的位置                                     | 字符串                                          | `your-path/Flowise/logs`            |
-| LOG_LEVEL                  | 日志的不同级别                                         | 枚举字符串：`error`、`info`、`verbose`、`debug` | `info`                              |
-| APIKEY_PATH                | 存储 API 密钥的位置                                    | 字符串                                          | `your-path/Flowise/packages/server` |
-| TOOL_FUNCTION_BUILTIN_DEP  | 用于工具函数的 NodeJS 内置模块                         | 字符串                                          |                                     |
-| TOOL_FUNCTION_EXTERNAL_DEP | 用于工具函数的外部模块                                 | 字符串                                          |                                     |
-| OVERRIDE_DATABASE          | 使用默认值覆盖当前数据库                               | 枚举字符串：`true`、`false`                     | `true`                              |
-| DATABASE_TYPE              | 存储 flowise 数据的数据库类型                          | 枚举字符串：`sqlite`、`mysql`、`postgres`       | `sqlite`                            |
-| DATABASE_PATH              | 数据库的保存位置（当 DATABASE_TYPE 为 sqlite 时）      | 字符串                                          | `your-home-dir/.flowise`            |
-| DATABASE_HOST              | 主机 URL 或 IP 地址（当 DATABASE_TYPE 不为 sqlite 时） | 字符串                                          |                                     |
-| DATABASE_PORT              | 数据库端口（当 DATABASE_TYPE 不为 sqlite 时）          | 字符串                                          |                                     |
-| DATABASE_USERNAME          | 数据库用户名（当 DATABASE_TYPE 不为 sqlite 时）        | 字符串                                          |                                     |
-| DATABASE_PASSWORD          | 数据库密码（当 DATABASE_TYPE 不为 sqlite 时）          | 字符串                                          |                                     |
-| DATABASE_NAME              | 数据库名称（当 DATABASE_TYPE 不为 sqlite 时）          | 字符串                                          |                                     |
-
 您还可以在使用`npx`时指定环境变量。例如：
 
 ```

packages/server/marketplaces/chatflows/Vectara LLM Chain Upload.json

@@ -1,11 +1,125 @@
 {
-    "description": "A simple LLM chain that uses Vectara to enable conversations with uploaded documents",
+    "description": "A simple LLM chain that uses Vectara to enable conversations with uploaded files",
     "nodes": [
+        {
+            "width": 300,
+            "height": 524,
+            "id": "vectaraUpload_0",
+            "position": { "x": 219.0098475967174, "y": 189.74396248534583 },
+            "type": "customNode",
+            "data": {
+                "id": "vectaraUpload_0",
+                "label": "Vectara Upload File",
+                "version": 1,
+                "name": "vectaraUpload",
+                "type": "Vectara",
+                "baseClasses": ["Vectara", "VectorStoreRetriever", "BaseRetriever"],
+                "category": "Vector Stores",
+                "description": "Upload files to Vectara",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["vectaraApi"],
+                        "id": "vectaraUpload_0-input-credential-credential"
+                    },
+                    {
+                        "label": "File",
+                        "name": "file",
+                        "description": "File to upload to Vectara. Supported file types: https://docs.vectara.com/docs/api-reference/indexing-apis/file-upload/file-upload-filetypes",
+                        "type": "file",
+                        "id": "vectaraUpload_0-input-file-file"
+                    },
+                    {
+                        "label": "Vectara Metadata Filter",
+                        "name": "filter",
+                        "description": "Filter to apply to Vectara metadata. Refer to the <a target=\"_blank\" href=\"https://docs.flowiseai.com/vector-stores/vectara\">documentation</a> on how to use Vectara filters with Flowise.",
+                        "type": "string",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "vectaraUpload_0-input-filter-string"
+                    },
+                    {
+                        "label": "Sentences Before",
+                        "name": "sentencesBefore",
+                        "description": "Number of sentences to fetch before the matched sentence. Defaults to 2.",
+                        "type": "number",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "vectaraUpload_0-input-sentencesBefore-number"
+                    },
+                    {
+                        "label": "Sentences After",
+                        "name": "sentencesAfter",
+                        "description": "Number of sentences to fetch after the matched sentence. Defaults to 2.",
+                        "type": "number",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "vectaraUpload_0-input-sentencesAfter-number"
+                    },
+                    {
+                        "label": "Lambda",
+                        "name": "lambda",
+                        "description": "Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.",
+                        "type": "number",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "vectaraUpload_0-input-lambda-number"
+                    },
+                    {
+                        "label": "Top K",
+                        "name": "topK",
+                        "description": "Number of top results to fetch. Defaults to 4",
+                        "placeholder": "4",
+                        "type": "number",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "vectaraUpload_0-input-topK-number"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {
+                    "filter": "",
+                    "sentencesBefore": "",
+                    "sentencesAfter": "",
+                    "lambda": "",
+                    "topK": ""
+                },
+                "outputAnchors": [
+                    {
+                        "name": "output",
+                        "label": "Output",
+                        "type": "options",
+                        "options": [
+                            {
+                                "id": "vectaraUpload_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever",
+                                "name": "retriever",
+                                "label": "Vectara Retriever",
+                                "type": "Vectara | VectorStoreRetriever | BaseRetriever"
+                            },
+                            {
+                                "id": "vectaraUpload_0-output-vectorStore-Vectara|VectorStore",
+                                "name": "vectorStore",
+                                "label": "Vectara Vector Store",
+                                "type": "Vectara | VectorStore"
+                            }
+                        ],
+                        "default": "retriever"
+                    }
+                ],
+                "outputs": { "output": "retriever" },
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": { "x": 219.0098475967174, "y": 189.74396248534583 },
+            "dragging": false
+        },
         {
             "width": 300,
             "height": 525,
             "id": "chatOpenAI_0",
-            "position": { "x": 514.1088940275924, "y": 199.574479681537 },
+            "position": { "x": 669.6533996522251, "y": 177.86181519287192 },
             "type": "customNode",
             "data": {
                 "id": "chatOpenAI_0",
@@ -13,7 +127,7 @@
                 "version": 1,
                 "name": "chatOpenAI",
                 "type": "ChatOpenAI",
-                "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"],
+                "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
                 "category": "Chat Models",
                 "description": "Wrapper around OpenAI large language models that use the Chat endpoint",
                 "inputParams": [
@@ -36,7 +150,10 @@
                             { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" },
                             { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" },
                             { "label": "gpt-3.5-turbo-16k", "name": "gpt-3.5-turbo-16k" },
-                            { "label": "gpt-3.5-turbo-16k-0613", "name": "gpt-3.5-turbo-16k-0613" }
+                            {
+                                "label": "gpt-3.5-turbo-16k-0613",
+                                "name": "gpt-3.5-turbo-16k-0613"
+                            }
                         ],
                         "default": "gpt-3.5-turbo",
                         "optional": true,
@@ -103,6 +220,14 @@
                         "optional": true,
                         "additionalParams": true,
                         "id": "chatOpenAI_0-input-basepath-string"
+                    },
+                    {
+                        "label": "BaseOptions",
+                        "name": "baseOptions",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-baseOptions-json"
                     }
                 ],
                 "inputAnchors": [],
@@ -114,28 +239,29 @@
                     "frequencyPenalty": "",
                     "presencePenalty": "",
                     "timeout": "",
-                    "basepath": ""
+                    "basepath": "",
+                    "baseOptions": ""
                 },
                 "outputAnchors": [
                     {
-                        "id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel",
+                        "id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
                         "name": "chatOpenAI",
                         "label": "ChatOpenAI",
-                        "type": "ChatOpenAI | BaseChatModel | BaseLanguageModel"
+                        "type": "ChatOpenAI | BaseChatModel | BaseLanguageModel | Runnable"
                     }
                 ],
                 "outputs": {},
                 "selected": false
             },
             "selected": false,
-            "positionAbsolute": { "x": 514.1088940275924, "y": 199.574479681537 },
+            "positionAbsolute": { "x": 669.6533996522251, "y": 177.86181519287192 },
             "dragging": false
         },
         {
             "width": 300,
             "height": 481,
             "id": "conversationalRetrievalQAChain_0",
-            "position": { "x": 900.4793407261002, "y": 205.9476004518217 },
+            "position": { "x": 1135.5490908971935, "y": 201.62146241822506 },
             "type": "customNode",
             "data": {
                 "id": "conversationalRetrievalQAChain_0",
@@ -143,7 +269,7 @@
                 "version": 1,
                 "name": "conversationalRetrievalQAChain",
                 "type": "ConversationalRetrievalQAChain",
-                "baseClasses": ["ConversationalRetrievalQAChain", "BaseChain"],
+                "baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "Runnable"],
                 "category": "Chains",
                 "description": "Document QA - built on RetrievalQAChain to provide a chat history component",
                 "inputParams": [
@@ -214,234 +340,45 @@
                 ],
                 "inputs": {
                     "model": "{{chatOpenAI_0.data.instance}}",
-                    "vectorStoreRetriever": "{{vectaraUpsert_0.data.instance}}",
+                    "vectorStoreRetriever": "{{vectaraUpload_0.data.instance}}",
                     "memory": "",
-                    "returnSourceDocuments": "",
+                    "returnSourceDocuments": true,
                     "systemMessagePrompt": "",
                     "chainOption": ""
                 },
                 "outputAnchors": [
                     {
-                        "id": "conversationalRetrievalQAChain_0-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain",
+                        "id": "conversationalRetrievalQAChain_0-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|Runnable",
                         "name": "conversationalRetrievalQAChain",
                         "label": "ConversationalRetrievalQAChain",
-                        "type": "ConversationalRetrievalQAChain | BaseChain"
+                        "type": "ConversationalRetrievalQAChain | BaseChain | Runnable"
                     }
                 ],
                 "outputs": {},
                 "selected": false
             },
             "selected": false,
-            "positionAbsolute": { "x": 900.4793407261002, "y": 205.9476004518217 },
+            "dragging": false,
-            "dragging": false
+            "positionAbsolute": { "x": 1135.5490908971935, "y": 201.62146241822506 }
-        },
-        {
-            "width": 300,
-            "height": 509,
-            "id": "pdfFile_0",
-            "position": { "x": -210.44158723479913, "y": 236.6627524951051 },
-            "type": "customNode",
-            "data": {
-                "id": "pdfFile_0",
-                "label": "Pdf File",
-                "version": 1,
-                "name": "pdfFile",
-                "type": "Document",
-                "baseClasses": ["Document"],
-                "category": "Document Loaders",
-                "description": "Load data from PDF files",
-                "inputParams": [
-                    { "label": "Pdf File", "name": "pdfFile", "type": "file", "fileType": ".pdf", "id": "pdfFile_0-input-pdfFile-file" },
-                    {
-                        "label": "Usage",
-                        "name": "usage",
-                        "type": "options",
-                        "options": [
-                            { "label": "One document per page", "name": "perPage" },
-                            { "label": "One document per file", "name": "perFile" }
-                        ],
-                        "default": "perPage",
-                        "id": "pdfFile_0-input-usage-options"
-                    },
-                    {
-                        "label": "Use Legacy Build",
-                        "name": "legacyBuild",
-                        "type": "boolean",
-                        "optional": true,
-                        "additionalParams": true,
-                        "id": "pdfFile_0-input-legacyBuild-boolean"
-                    },
-                    {
-                        "label": "Metadata",
-                        "name": "metadata",
-                        "type": "json",
-                        "optional": true,
-                        "additionalParams": true,
-                        "id": "pdfFile_0-input-metadata-json"
-                    }
-                ],
-                "inputAnchors": [
-                    {
-                        "label": "Text Splitter",
-                        "name": "textSplitter",
-                        "type": "TextSplitter",
-                        "optional": true,
-                        "id": "pdfFile_0-input-textSplitter-TextSplitter"
-                    }
-                ],
-                "inputs": { "textSplitter": "", "usage": "perPage", "legacyBuild": "", "metadata": "" },
-                "outputAnchors": [
-                    { "id": "pdfFile_0-output-pdfFile-Document", "name": "pdfFile", "label": "Document", "type": "Document" }
-                ],
-                "outputs": {},
-                "selected": false
-            },
-            "selected": false,
-            "positionAbsolute": { "x": -210.44158723479913, "y": 236.6627524951051 },
-            "dragging": false
-        },
-        {
-            "width": 300,
-            "height": 408,
-            "id": "vectaraUpsert_0",
-            "position": { "x": 172.06946164914868, "y": 373.11406233089934 },
-            "type": "customNode",
-            "data": {
-                "id": "vectaraUpsert_0",
-                "label": "Vectara Upsert Document",
-                "version": 1,
-                "name": "vectaraUpsert",
-                "type": "Vectara",
-                "baseClasses": ["Vectara", "VectorStoreRetriever", "BaseRetriever"],
-                "category": "Vector Stores",
-                "description": "Upsert documents to Vectara",
-                "inputParams": [
-                    {
-                        "label": "Connect Credential",
-                        "name": "credential",
-                        "type": "credential",
-                        "credentialNames": ["vectaraApi"],
-                        "id": "vectaraUpsert_0-input-credential-credential"
-                    },
-                    {
-                        "label": "Vectara Metadata Filter",
-                        "name": "filter",
-                        "description": "Filter to apply to Vectara metadata. Refer to the <a target=\"_blank\" href=\"https://docs.flowiseai.com/vector-stores/vectara\">documentation</a> on how to use Vectara filters with Flowise.",
-                        "type": "string",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "vectaraUpsert_0-input-filter-string"
-                    },
-                    {
-                        "label": "Sentences Before",
-                        "name": "sentencesBefore",
-                        "description": "Number of sentences to fetch before the matched sentence. Defaults to 2.",
-                        "type": "number",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "vectaraUpsert_0-input-sentencesBefore-number"
-                    },
-                    {
-                        "label": "Sentences After",
-                        "name": "sentencesAfter",
-                        "description": "Number of sentences to fetch after the matched sentence. Defaults to 2.",
-                        "type": "number",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "vectaraUpsert_0-input-sentencesAfter-number"
-                    },
-                    {
-                        "label": "Lambda",
-                        "name": "lambda",
-                        "description": "Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.",
-                        "type": "number",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "vectaraUpsert_0-input-lambda-number"
-                    },
-                    {
-                        "label": "Top K",
-                        "name": "topK",
-                        "description": "Number of top results to fetch. Defaults to 4",
-                        "placeholder": "4",
-                        "type": "number",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "vectaraUpsert_0-input-topK-number"
-                    }
-                ],
-                "inputAnchors": [
-                    {
-                        "label": "Document",
-                        "name": "document",
-                        "type": "Document",
-                        "list": true,
-                        "id": "vectaraUpsert_0-input-document-Document"
-                    }
-                ],
-                "inputs": {
-                    "document": ["{{pdfFile_0.data.instance}}"],
-                    "filter": "",
-                    "sentencesBefore": "",
-                    "sentencesAfter": "",
-                    "lambda": "",
-                    "topK": ""
-                },
-                "outputAnchors": [
-                    {
-                        "name": "output",
-                        "label": "Output",
-                        "type": "options",
-                        "options": [
-                            {
-                                "id": "vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever",
-                                "name": "retriever",
-                                "label": "Vectara Retriever",
-                                "type": "Vectara | VectorStoreRetriever | BaseRetriever"
-                            },
-                            {
-                                "id": "vectaraUpsert_0-output-vectorStore-Vectara|VectorStore",
-                                "name": "vectorStore",
-                                "label": "Vectara Vector Store",
-                                "type": "Vectara | VectorStore"
-                            }
-                        ],
-                        "default": "retriever"
-                    }
-                ],
-                "outputs": { "output": "retriever" },
-                "selected": false
-            },
-            "positionAbsolute": { "x": 172.06946164914868, "y": 373.11406233089934 },
-            "selected": false
         }
     ],
     "edges": [
         {
-            "source": "chatOpenAI_0",
+            "source": "vectaraUpload_0",
-            "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel",
+            "sourceHandle": "vectaraUpload_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever",
-            "target": "conversationalRetrievalQAChain_0",
-            "targetHandle": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
-            "type": "buttonedge",
-            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
-            "data": { "label": "" }
-        },
-        {
-            "source": "pdfFile_0",
-            "sourceHandle": "pdfFile_0-output-pdfFile-Document",
-            "target": "vectaraUpsert_0",
-            "targetHandle": "vectaraUpsert_0-input-document-Document",
-            "type": "buttonedge",
-            "id": "pdfFile_0-pdfFile_0-output-pdfFile-Document-vectaraUpsert_0-vectaraUpsert_0-input-document-Document",
-            "data": { "label": "" }
-        },
-        {
-            "source": "vectaraUpsert_0",
-            "sourceHandle": "vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever",
             "target": "conversationalRetrievalQAChain_0",
             "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
             "type": "buttonedge",
-            "id": "vectaraUpsert_0-vectaraUpsert_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
+            "id": "vectaraUpload_0-vectaraUpload_0-output-retriever-Vectara|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
+            "data": { "label": "" }
+        },
+        {
+            "source": "chatOpenAI_0",
+            "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+            "target": "conversationalRetrievalQAChain_0",
+            "targetHandle": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
+            "type": "buttonedge",
+            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
             "data": { "label": "" }
         }
     ]

packages/server/package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise",
-    "version": "1.3.4",
+    "version": "1.3.5",
     "description": "Flowiseai Server",
     "main": "dist/index",
     "types": "dist/index.d.ts",

packages/server/src/commands/start.ts

@@ -19,15 +19,14 @@ export default class Start extends Command {
         FLOWISE_USERNAME: Flags.string(),
         FLOWISE_PASSWORD: Flags.string(),
         PORT: Flags.string(),
-        PASSPHRASE: Flags.string(),
         DEBUG: Flags.string(),
         APIKEY_PATH: Flags.string(),
         SECRETKEY_PATH: Flags.string(),
+        FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
         LOG_PATH: Flags.string(),
         LOG_LEVEL: Flags.string(),
         TOOL_FUNCTION_BUILTIN_DEP: Flags.string(),
         TOOL_FUNCTION_EXTERNAL_DEP: Flags.string(),
-        OVERRIDE_DATABASE: Flags.string(),
         DATABASE_TYPE: Flags.string(),
         DATABASE_PATH: Flags.string(),
         DATABASE_PORT: Flags.string(),
@@ -80,8 +79,8 @@ export default class Start extends Command {
         if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH
 
         // Credentials
-        if (flags.PASSPHRASE) process.env.PASSPHRASE = flags.PASSPHRASE
         if (flags.SECRETKEY_PATH) process.env.SECRETKEY_PATH = flags.SECRETKEY_PATH
+        if (flags.FLOWISE_SECRETKEY_OVERWRITE) process.env.FLOWISE_SECRETKEY_OVERWRITE = flags.FLOWISE_SECRETKEY_OVERWRITE
 
         // Logs
         if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
@@ -92,7 +91,6 @@ export default class Start extends Command {
         if (flags.TOOL_FUNCTION_EXTERNAL_DEP) process.env.TOOL_FUNCTION_EXTERNAL_DEP = flags.TOOL_FUNCTION_EXTERNAL_DEP
 
         // Database config
-        if (flags.OVERRIDE_DATABASE) process.env.OVERRIDE_DATABASE = flags.OVERRIDE_DATABASE
         if (flags.DATABASE_TYPE) process.env.DATABASE_TYPE = flags.DATABASE_TYPE
         if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH
         if (flags.DATABASE_PORT) process.env.DATABASE_PORT = flags.DATABASE_PORT

packages/server/src/utils/index.ts

@@ -28,7 +28,7 @@ import {
     convertChatHistoryToText
 } from 'flowise-components'
 import { scryptSync, randomBytes, timingSafeEqual } from 'crypto'
-import { lib, PBKDF2, AES, enc } from 'crypto-js'
+import { AES, enc } from 'crypto-js'
 
 import { ChatFlow } from '../database/entities/ChatFlow'
 import { ChatMessage } from '../database/entities/ChatMessage'
@@ -814,12 +814,7 @@ export const getEncryptionKeyPath = (): string => {
  * @returns {string}
  */
 export const generateEncryptKey = (): string => {
-    const salt = lib.WordArray.random(128 / 8)
+    return randomBytes(24).toString('base64')
-    const key256Bits = PBKDF2(process.env.PASSPHRASE || 'MYPASSPHRASE', salt, {
-        keySize: 256 / 32,
-        iterations: 1000
-    })
-    return key256Bits.toString()
 }
 
 /**
@@ -827,6 +822,9 @@ export const generateEncryptKey = (): string => {
  * @returns {Promise<string>}
  */
 export const getEncryptionKey = async (): Promise<string> => {
+    if (process.env.FLOWISE_SECRETKEY_OVERWRITE !== undefined && process.env.FLOWISE_SECRETKEY_OVERWRITE !== '') {
+        return process.env.FLOWISE_SECRETKEY_OVERWRITE
+    }
     try {
         return await fs.promises.readFile(getEncryptionKeyPath(), 'utf8')
     } catch (error) {
@@ -868,7 +866,7 @@ export const decryptCredentialData = async (
         return JSON.parse(decryptedData.toString(enc.Utf8))
     } catch (e) {
         console.error(e)
-        throw new Error('Credentials could not be decrypted.')
+        return {}
     }
 }
 

packages/ui/package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise-ui",
-    "version": "1.3.2",
+    "version": "1.3.3",
     "license": "SEE LICENSE IN LICENSE.md",
     "homepage": "https://flowiseai.com",
     "author": {