Feature/update upsert API (#3836)

* update upsert API

* add fix for override files in upsert vector

packages/api-documentation/src/yml/swagger.yml

@@ -628,8 +628,8 @@ paths:
                 - document-store
             security:
                 - bearerAuth: []
-            summary: Upsert new document to document store
-            description: Upsert new document to document store
+            summary: Upsert document to document store
+            description: Upsert document to document store
             operationId: upsertDocument
             parameters:
                 - in: path
@@ -2326,6 +2326,13 @@ components:
                     type: string
                     format: uuid
                     description: Document ID within the store. If provided, existing configuration from the document will be used for the new document
+                metadata:
+                    type: object
+                    description: Metadata associated with the document
+                    example: { 'foo': 'bar' }
+                replaceExisting:
+                    type: boolean
+                    description: Whether to replace existing document loader with the new upserted chunks. However this does not delete the existing embeddings in the vector store
                 loader:
                     type: object
                     properties:

packages/server/src/Interface.DocumentStore.ts

@@ -70,6 +70,8 @@ export interface IDocumentStoreLoaderForPreview extends IDocumentStoreLoader {
 
 export interface IDocumentStoreUpsertData {
     docId: string
+    metadata?: string | object
+    replaceExisting?: boolean
     loader?: {
         name: string
         config: ICommonObject

packages/server/src/controllers/documentstore/index.ts

@@ -428,6 +428,27 @@ const generateDocStoreToolDesc = async (req: Request, res: Response, next: NextF
     }
 }
 
+const getDocStoreConfigs = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.id === 'undefined' || req.params.id === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.getDocStoreConfigs - storeId not provided!`
+            )
+        }
+        if (typeof req.params.loaderId === 'undefined' || req.params.loaderId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.getDocStoreConfigs - doc loader Id not provided!`
+            )
+        }
+        const apiResponse = await documentStoreService.findDocStoreAvailableConfigs(req.params.id, req.params.loaderId)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
 export default {
     deleteDocumentStore,
     createDocumentStore,
@@ -453,5 +474,6 @@ export default {
     upsertDocStoreMiddleware,
     refreshDocStoreMiddleware,
     saveProcessingLoader,
-    generateDocStoreToolDesc
+    generateDocStoreToolDesc,
+    getDocStoreConfigs
 }

packages/server/src/index.ts

@@ -25,6 +25,7 @@ import { validateAPIKey } from './utils/validateKey'
 import { IMetricsProvider } from './Interface.Metrics'
 import { Prometheus } from './metrics/Prometheus'
 import { OpenTelemetry } from './metrics/OpenTelemetry'
+import { WHITELIST_URLS } from './utils/constants'
 import 'global-agent/bootstrap'
 
 declare global {
@@ -124,27 +125,7 @@ export class App {
             next()
         })
 
-        const whitelistURLs = [
-            '/api/v1/verify/apikey/',
-            '/api/v1/chatflows/apikey/',
-            '/api/v1/public-chatflows',
-            '/api/v1/public-chatbotConfig',
-            '/api/v1/prediction/',
-            '/api/v1/vector/upsert/',
-            '/api/v1/node-icon/',
-            '/api/v1/components-credentials-icon/',
-            '/api/v1/chatflows-streaming',
-            '/api/v1/chatflows-uploads',
-            '/api/v1/openai-assistants-file/download',
-            '/api/v1/feedback',
-            '/api/v1/leads',
-            '/api/v1/get-upload-file',
-            '/api/v1/ip',
-            '/api/v1/ping',
-            '/api/v1/version',
-            '/api/v1/attachments',
-            '/api/v1/metrics'
-        ]
+        const whitelistURLs = WHITELIST_URLS
         const URL_CASE_INSENSITIVE_REGEX: RegExp = /\/api\/v1\//i
         const URL_CASE_SENSITIVE_REGEX: RegExp = /\/api\/v1\//
 

packages/server/src/routes/documentstore/index.ts

@@ -21,6 +21,8 @@ router.get('/store/:id', documentStoreController.getDocumentStoreById)
 router.put('/store/:id', documentStoreController.updateDocumentStore)
 // Delete documentStore
 router.delete('/store/:id', documentStoreController.deleteDocumentStore)
+// Get document store configs
+router.get('/store-configs/:id/:loaderId', documentStoreController.getDocStoreConfigs)
 
 /** Component Nodes = Document Store - Loaders */
 // Get all loaders

packages/server/src/services/assistants/index.ts

@@ -15,6 +15,7 @@ import { DocumentStore } from '../../database/entities/DocumentStore'
 import { ICommonObject } from 'flowise-components'
 import logger from '../../utils/logger'
 import { ASSISTANT_PROMPT_GENERATOR } from '../../utils/prompt'
+import { INPUT_PARAMS_TYPE } from '../../utils/constants'
 
 const createAssistant = async (requestBody: any): Promise<Assistant> => {
     try {
@@ -425,26 +426,11 @@ const getDocumentStores = async (): Promise<any> => {
 const getTools = async (): Promise<any> => {
     try {
         const tools = await nodesService.getAllNodesForCategory('Tools')
-        const whitelistTypes = [
-            'asyncOptions',
-            'options',
-            'multiOptions',
-            'datagrid',
-            'string',
-            'number',
-            'boolean',
-            'password',
-            'json',
-            'code',
-            'date',
-            'file',
-            'folder',
-            'tabs'
-        ]
+
         // filter out those tools that input params type are not in the list
         const filteredTools = tools.filter((tool) => {
             const inputs = tool.inputs || []
-            return inputs.every((input) => whitelistTypes.includes(input.type))
+            return inputs.every((input) => INPUT_PARAMS_TYPE.includes(input.type))
         })
         return filteredTools
     } catch (error) {

packages/server/src/services/documentstore/index.ts

@@ -24,7 +24,8 @@ import {
     IDocumentStoreRefreshData,
     IDocumentStoreUpsertData,
     IDocumentStoreWhereUsed,
-    INodeData
+    INodeData,
+    IOverrideConfig
 } from '../../Interface'
 import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk'
 import { v4 as uuidv4 } from 'uuid'
@@ -41,6 +42,7 @@ import { UpsertHistory } from '../../database/entities/UpsertHistory'
 import { cloneDeep, omit } from 'lodash'
 import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../../Interface.Metrics'
 import { DOCUMENTSTORE_TOOL_DESCRIPTION_PROMPT_GENERATOR } from '../../utils/prompt'
+import { INPUT_PARAMS_TYPE } from '../../utils/constants'
 
 const DOCUMENT_STORE_BASE_FOLDER = 'docustore'
 
@@ -1323,6 +1325,15 @@ const upsertDocStoreMiddleware = async (
 ) => {
     const appServer = getRunningExpressApp()
     const docId = data.docId
+    let metadata = {}
+    if (data.metadata) {
+        try {
+            metadata = typeof data.metadata === 'string' ? JSON.parse(data.metadata) : data.metadata
+        } catch (error) {
+            throw new InternalFlowiseError(StatusCodes.BAD_REQUEST, `Error: Invalid metadata`)
+        }
+    }
+    const replaceExisting = data.replaceExisting ?? false
     const newLoader = typeof data.loader === 'string' ? JSON.parse(data.loader) : data.loader
     const newSplitter = typeof data.splitter === 'string' ? JSON.parse(data.splitter) : data.splitter
     const newVectorStore = typeof data.vectorStore === 'string' ? JSON.parse(data.vectorStore) : data.vectorStore
@@ -1479,6 +1490,13 @@ const upsertDocStoreMiddleware = async (
         }
     }
 
+    if (Object.keys(metadata).length > 0) {
+        loaderConfig = {
+            ...loaderConfig,
+            metadata
+        }
+    }
+
     // Step 4: Verification for must have components
     if (!loaderName || !loaderId || !loaderConfig) {
         throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Loader not configured`)
@@ -1503,7 +1521,7 @@ const upsertDocStoreMiddleware = async (
         splitterConfig
     }
 
-    if (isRefreshExisting) {
+    if (isRefreshExisting || replaceExisting) {
         processData.id = docId
     }
 
@@ -1629,6 +1647,146 @@ const generateDocStoreToolDesc = async (docStoreId: string, selectedChatModel: I
     }
 }
 
+export const findDocStoreAvailableConfigs = async (storeId: string, docId: string) => {
+    // find the document store
+    const appServer = getRunningExpressApp()
+    const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ id: storeId })
+
+    if (!entity) {
+        throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
+    }
+
+    const loaders = JSON.parse(entity.loaders)
+    const loader = loaders.find((ldr: IDocumentStoreLoader) => ldr.id === docId)
+    if (!loader) {
+        throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document loader ${docId} not found`)
+    }
+
+    const nodes = []
+    const componentCredentials = appServer.nodesPool.componentCredentials
+
+    const loaderName = loader.loaderId
+    const loaderLabel = appServer.nodesPool.componentNodes[loaderName].label
+
+    const loaderInputs =
+        appServer.nodesPool.componentNodes[loaderName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+    nodes.push({
+        label: loaderLabel,
+        nodeId: `${loaderName}_0`,
+        inputParams: loaderInputs
+    })
+
+    const splitterName = loader.splitterId
+    if (splitterName) {
+        const splitterLabel = appServer.nodesPool.componentNodes[splitterName].label
+        const splitterInputs =
+            appServer.nodesPool.componentNodes[splitterName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+        nodes.push({
+            label: splitterLabel,
+            nodeId: `${splitterName}_0`,
+            inputParams: splitterInputs
+        })
+    }
+
+    if (entity.vectorStoreConfig) {
+        const vectorStoreName = JSON.parse(entity.vectorStoreConfig || '{}').name
+        const vectorStoreLabel = appServer.nodesPool.componentNodes[vectorStoreName].label
+        const vectorStoreInputs =
+            appServer.nodesPool.componentNodes[vectorStoreName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+        nodes.push({
+            label: vectorStoreLabel,
+            nodeId: `${vectorStoreName}_0`,
+            inputParams: vectorStoreInputs
+        })
+    }
+
+    if (entity.embeddingConfig) {
+        const embeddingName = JSON.parse(entity.embeddingConfig || '{}').name
+        const embeddingLabel = appServer.nodesPool.componentNodes[embeddingName].label
+        const embeddingInputs =
+            appServer.nodesPool.componentNodes[embeddingName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+        nodes.push({
+            label: embeddingLabel,
+            nodeId: `${embeddingName}_0`,
+            inputParams: embeddingInputs
+        })
+    }
+
+    if (entity.recordManagerConfig) {
+        const recordManagerName = JSON.parse(entity.recordManagerConfig || '{}').name
+        const recordManagerLabel = appServer.nodesPool.componentNodes[recordManagerName].label
+        const recordManagerInputs =
+            appServer.nodesPool.componentNodes[recordManagerName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+        nodes.push({
+            label: recordManagerLabel,
+            nodeId: `${recordManagerName}_0`,
+            inputParams: recordManagerInputs
+        })
+    }
+
+    const configs: IOverrideConfig[] = []
+    for (const node of nodes) {
+        const inputParams = node.inputParams
+        for (const inputParam of inputParams) {
+            let obj: IOverrideConfig
+            if (inputParam.type === 'file') {
+                obj = {
+                    node: node.label,
+                    nodeId: node.nodeId,
+                    label: inputParam.label,
+                    name: 'files',
+                    type: inputParam.fileType ?? inputParam.type
+                }
+            } else if (inputParam.type === 'options') {
+                obj = {
+                    node: node.label,
+                    nodeId: node.nodeId,
+                    label: inputParam.label,
+                    name: inputParam.name,
+                    type: inputParam.options
+                        ? inputParam.options
+                              ?.map((option) => {
+                                  return option.name
+                              })
+                              .join(', ')
+                        : 'string'
+                }
+            } else if (inputParam.type === 'credential') {
+                // get component credential inputs
+                for (const name of inputParam.credentialNames ?? []) {
+                    if (Object.prototype.hasOwnProperty.call(componentCredentials, name)) {
+                        const inputs = componentCredentials[name]?.inputs ?? []
+                        for (const input of inputs) {
+                            obj = {
+                                node: node.label,
+                                nodeId: node.nodeId,
+                                label: input.label,
+                                name: input.name,
+                                type: input.type === 'password' ? 'string' : input.type
+                            }
+                            configs.push(obj)
+                        }
+                    }
+                }
+                continue
+            } else {
+                obj = {
+                    node: node.label,
+                    nodeId: node.nodeId,
+                    label: inputParam.label,
+                    name: inputParam.name,
+                    type: inputParam.type === 'password' ? 'string' : inputParam.type
+                }
+            }
+            if (!configs.some((config) => JSON.stringify(config) === JSON.stringify(obj))) {
+                configs.push(obj)
+            }
+        }
+    }
+
+    return configs
+}
+
 export default {
     updateDocumentStoreUsage,
     deleteDocumentStore,
@@ -1656,5 +1814,6 @@ export default {
     updateVectorStoreConfigOnly,
     upsertDocStoreMiddleware,
     refreshDocStoreMiddleware,
-    generateDocStoreToolDesc
+    generateDocStoreToolDesc,
+    findDocStoreAvailableConfigs
 }

packages/server/src/utils/constants.ts

@@ -0,0 +1,38 @@
+export const WHITELIST_URLS = [
+    '/api/v1/verify/apikey/',
+    '/api/v1/chatflows/apikey/',
+    '/api/v1/public-chatflows',
+    '/api/v1/public-chatbotConfig',
+    '/api/v1/prediction/',
+    '/api/v1/vector/upsert/',
+    '/api/v1/node-icon/',
+    '/api/v1/components-credentials-icon/',
+    '/api/v1/chatflows-streaming',
+    '/api/v1/chatflows-uploads',
+    '/api/v1/openai-assistants-file/download',
+    '/api/v1/feedback',
+    '/api/v1/leads',
+    '/api/v1/get-upload-file',
+    '/api/v1/ip',
+    '/api/v1/ping',
+    '/api/v1/version',
+    '/api/v1/attachments',
+    '/api/v1/metrics'
+]
+
+export const INPUT_PARAMS_TYPE = [
+    'asyncOptions',
+    'options',
+    'multiOptions',
+    'datagrid',
+    'string',
+    'number',
+    'boolean',
+    'password',
+    'json',
+    'code',
+    'date',
+    'file',
+    'folder',
+    'tabs'
+]

packages/server/src/utils/upsertVector.ts

@@ -161,16 +161,29 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
         const availableVariables = await appServer.AppDataSource.getRepository(Variable).find()
         const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
 
-        // For "files" input, add a new node override with the actual input name such as pdfFile, txtFile, etc.
+        // For "files" input, add a new node override with the actual input name such as pdfFile, txtFile, etc, to allow overriding the input
         for (const nodeLabel in nodeOverrides) {
             const params = nodeOverrides[nodeLabel]
             const enabledFileParam = params.find((param) => param.enabled && param.name === 'files')
             if (enabledFileParam) {
-                const fileInputFieldFromExt = mapExtToInputField(enabledFileParam.type)
-                nodeOverrides[nodeLabel].push({
-                    ...enabledFileParam,
-                    name: fileInputFieldFromExt
-                })
+                if (enabledFileParam.type.includes(',')) {
+                    const fileInputFieldsFromExt = enabledFileParam.type.split(',').map((fileType) => mapExtToInputField(fileType.trim()))
+                    for (const fileInputFieldFromExt of fileInputFieldsFromExt) {
+                        if (nodeOverrides[nodeLabel].some((param) => param.name === fileInputFieldFromExt)) {
+                            continue
+                        }
+                        nodeOverrides[nodeLabel].push({
+                            ...enabledFileParam,
+                            name: fileInputFieldFromExt
+                        })
+                    }
+                } else {
+                    const fileInputFieldFromExt = mapExtToInputField(enabledFileParam.type)
+                    nodeOverrides[nodeLabel].push({
+                        ...enabledFileParam,
+                        name: fileInputFieldFromExt
+                    })
+                }
             }
         }
 

packages/ui/src/api/documentstore.js

@@ -6,6 +6,7 @@ const getSpecificDocumentStore = (id) => client.get(`/document-store/store/${id}
 const createDocumentStore = (body) => client.post(`/document-store/store`, body)
 const updateDocumentStore = (id, body) => client.put(`/document-store/store/${id}`, body)
 const deleteDocumentStore = (id) => client.delete(`/document-store/store/${id}`)
+const getDocumentStoreConfig = (storeId, loaderId) => client.get(`/document-store/store-configs/${storeId}/${loaderId}`)
 
 const deleteLoaderFromStore = (id, fileId) => client.delete(`/document-store/loader/${id}/${fileId}`)
 const deleteChunkFromStore = (storeId, loaderId, chunkId) => client.delete(`/document-store/chunks/${storeId}/${loaderId}/${chunkId}`)
@@ -52,5 +53,6 @@ export default {
     updateVectorStoreConfig,
     saveProcessingLoader,
     refreshLoader,
-    generateDocStoreToolDesc
+    generateDocStoreToolDesc,
+    getDocumentStoreConfig
 }

packages/ui/src/views/chatflows/APICodeDialog.jsx

@@ -38,7 +38,7 @@ import cURLSVG from '@/assets/images/cURL.svg'
 import EmbedSVG from '@/assets/images/embed.svg'
 import ShareChatbotSVG from '@/assets/images/sharing.png'
 import settingsSVG from '@/assets/images/settings.svg'
-import { IconBulb, IconBox, IconVariable } from '@tabler/icons-react'
+import { IconBulb, IconBox, IconVariable, IconExclamationCircle } from '@tabler/icons-react'
 
 // API
 import apiKeyApi from '@/api/apikey'
@@ -726,9 +726,44 @@ formData.append("openAIApiKey[openAIEmbeddings_0]", "sk-my-openai-2nd-key")`
                                 <CheckboxInput label='Show Override Config' value={checkboxVal} onChange={onCheckBoxChanged} />
                                 {checkboxVal && getConfigApi.data && getConfigApi.data.length > 0 && (
                                     <>
-                                        <Typography sx={{ mt: 2, mb: 3 }}>
+                                        <Typography sx={{ mt: 2 }}>
                                             You can override existing input configuration of the chatflow with overrideConfig property.
                                         </Typography>
+                                        <div
+                                            style={{
+                                                display: 'flex',
+                                                flexDirection: 'column',
+                                                borderRadius: 10,
+                                                background: 'rgb(254,252,191)',
+                                                padding: 10,
+                                                marginTop: 10,
+                                                marginBottom: 10
+                                            }}
+                                        >
+                                            <div
+                                                style={{
+                                                    display: 'flex',
+                                                    flexDirection: 'row',
+                                                    alignItems: 'center'
+                                                }}
+                                            >
+                                                <IconExclamationCircle size={30} color='rgb(116,66,16)' />
+                                                <span style={{ color: 'rgb(116,66,16)', marginLeft: 10, fontWeight: 500 }}>
+                                                    {
+                                                        'For security reason, override config is disabled by default. You can change this by going into Chatflow Configuration -> Security tab, and enable the property you want to override.'
+                                                    }
+                                                    &nbsp;Refer{' '}
+                                                    <a
+                                                        rel='noreferrer'
+                                                        target='_blank'
+                                                        href='https://docs.flowiseai.com/using-flowise/api#override-config'
+                                                    >
+                                                        here
+                                                    </a>{' '}
+                                                    for more details
+                                                </span>
+                                            </div>
+                                        </div>
                                         <Stack direction='column' spacing={2} sx={{ width: '100%', my: 2 }}>
                                             <Card sx={{ borderColor: theme.palette.primary[200] + 75, p: 2 }} variant='outlined'>
                                                 <Stack sx={{ mt: 1, mb: 2, ml: 1, alignItems: 'center' }} direction='row' spacing={2}>

packages/ui/src/views/docstore/DocStoreAPIDialog.jsx

@@ -0,0 +1,396 @@
+import { createPortal } from 'react-dom'
+import { useState, useEffect } from 'react'
+import PropTypes from 'prop-types'
+import rehypeMathjax from 'rehype-mathjax'
+import rehypeRaw from 'rehype-raw'
+import remarkGfm from 'remark-gfm'
+import remarkMath from 'remark-math'
+import { MemoizedReactMarkdown } from '@/ui-component/markdown/MemoizedReactMarkdown'
+import { CodeBlock } from '@/ui-component/markdown/CodeBlock'
+import { Typography, Stack, Card, Accordion, AccordionSummary, AccordionDetails, Dialog, DialogContent, DialogTitle } from '@mui/material'
+import { TableViewOnly } from '@/ui-component/table/Table'
+import documentstoreApi from '@/api/documentstore'
+import useApi from '@/hooks/useApi'
+import { useTheme } from '@mui/material/styles'
+import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
+
+const DocStoreAPIDialog = ({ show, dialogProps, onCancel }) => {
+    const [nodeConfig, setNodeConfig] = useState({})
+    const [values, setValues] = useState('')
+    const theme = useTheme()
+    const [nodeConfigExpanded, setNodeConfigExpanded] = useState({})
+
+    const getConfigApi = useApi(documentstoreApi.getDocumentStoreConfig)
+
+    const formDataRequest = () => {
+        return `With the Upsert API, you can choose an existing document and reuse the same configuration for upserting.
+
+\`\`\`python
+import requests
+import json
+
+API_URL = "http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId}"
+API_KEY = "your_api_key_here"
+
+# use form data to upload files
+form_data = {
+    "files": ('my-another-file.pdf', open('my-another-file.pdf', 'rb'))
+}
+
+body_data = {
+    "docId": "${dialogProps.loaderId}",
+    "metadata": {}, # Add additional metadata to the document chunks
+    "replaceExisting": True, # Replace existing document with the new upserted chunks
+    "splitter": json.dumps({"config":{"chunkSize":20000}}) # Override existing configuration
+    # "loader": "",
+    # "vectorStore": "",
+    # "embedding": "",
+    # "recordManager": "",
+}
+
+headers = {
+    "Authorization": f"Bearer {BEARER_TOKEN}"
+}
+
+def query(form_data):
+    response = requests.post(API_URL, files=form_data, data=body_data, headers=headers)
+    print(response)
+    return response.json()
+
+output = query(form_data)
+print(output)
+\`\`\`
+
+\`\`\`javascript
+// use FormData to upload files
+let formData = new FormData();
+formData.append("files", input.files[0]);
+formData.append("docId", "${dialogProps.loaderId}");
+formData.append("splitter", JSON.stringify({"config":{"chunkSize":20000}}));
+// Add additional metadata to the document chunks
+formData.append("metadata", "{}");
+// Replace existing document with the new upserted chunks
+formData.append("replaceExisting", "true");
+// Override existing configuration
+// formData.append("loader", "");
+// formData.append("embedding", "");
+// formData.append("vectorStore", "");
+// formData.append("recordManager", "");
+
+async function query(formData) {
+    const response = await fetch(
+        "http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId}",
+        {
+            method: "POST",
+            headers: {
+                "Authorization": "Bearer <your_api_key_here>"
+            },
+            body: formData
+        }
+    );
+    const result = await response.json();
+    return result;
+}
+
+query(formData).then((response) => {
+    console.log(response);
+});
+\`\`\`
+
+\`\`\`bash
+curl -X POST http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId} \\
+  -H "Authorization: Bearer <your_api_key_here>" \\
+  -F "files=@<file-path>" \\
+  -F "docId=${dialogProps.loaderId}" \\
+  -F "splitter={"config":{"chunkSize":20000}}" \\
+  -F "metadata={}" \\
+  -F "replaceExisting=true" \\
+  # Override existing configuration:
+  # -F "loader=" \\
+  # -F "embedding=" \\
+  # -F "vectorStore=" \\
+  # -F "recordManager="
+\`\`\`
+`
+    }
+
+    const jsonDataRequest = () => {
+        return `With the Upsert API, you can choose an existing document and reuse the same configuration for upserting.
+ 
+\`\`\`python
+import requests
+
+API_URL = "http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId}"
+API_KEY = "your_api_key_here"
+
+headers = {
+    "Authorization": f"Bearer {BEARER_TOKEN}"
+}
+
+def query(payload):
+    response = requests.post(API_URL, json=payload, headers=headers)
+    return response.json()
+
+output = query({
+    "docId": "${dialogProps.loaderId}",
+    "metadata": "{}", # Add additional metadata to the document chunks
+    "replaceExisting": True, # Replace existing document with the new upserted chunks
+    # Override existing configuration
+    "loader": {
+        "config": {
+            "text": "This is a new text"
+        }
+    },
+    "splitter": {
+        "config": {
+            "chunkSize": 20000
+        }
+    },
+    # embedding: {},
+    # vectorStore: {},
+    # recordManager: {}
+})
+print(output)
+\`\`\`
+
+\`\`\`javascript
+async function query(data) {
+    const response = await fetch(
+        "http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId}",
+        {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": "Bearer <your_api_key_here>"
+            },
+            body: JSON.stringify(data)
+        }
+    );
+    const result = await response.json();
+    return result;
+}
+
+query({
+    "docId": "${dialogProps.loaderId},
+    "metadata": "{}", // Add additional metadata to the document chunks
+    "replaceExisting": true, // Replace existing document with the new upserted chunks
+    // Override existing configuration
+    "loader": {
+        "config": {
+            "text": "This is a new text"
+        }
+    },
+    "splitter": {
+        "config": {
+            "chunkSize": 20000
+        }
+    },
+    // embedding: {},
+    // vectorStore: {},
+    // recordManager: {}
+}).then((response) => {
+    console.log(response);
+});
+\`\`\`
+
+\`\`\`bash
+curl -X POST http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId} \\
+  -H "Content-Type: application/json" \\
+  -H "Authorization: Bearer <your_api_key_here>" \\
+  -d '{
+        "docId": "${dialogProps.loaderId}",
+        "metadata": "{}",
+        "replaceExisting": true,
+        "loader": {
+            "config": {
+                "text": "This is a new text"
+            }
+        },
+        "splitter": {
+            "config": {
+                "chunkSize": 20000
+            }
+        }
+        // Override existing configuration
+        // "embedding": {},
+        // "vectorStore": {},
+        // "recordManager": {}
+      }'
+
+\`\`\`
+`
+    }
+
+    const groupByNodeLabel = (nodes) => {
+        const result = {}
+        const seenNodes = new Set()
+        let isFormDataBody = false
+
+        nodes.forEach((item) => {
+            const { node, nodeId, label, name, type } = item
+            if (name === 'files') isFormDataBody = true
+            seenNodes.add(node)
+
+            if (!result[node]) {
+                result[node] = {
+                    nodeIds: [],
+                    params: []
+                }
+            }
+
+            if (!result[node].nodeIds.includes(nodeId)) result[node].nodeIds.push(nodeId)
+
+            const param = { label, name, type }
+
+            if (!result[node].params.some((existingParam) => JSON.stringify(existingParam) === JSON.stringify(param))) {
+                result[node].params.push(param)
+            }
+        })
+
+        // Sort the nodeIds array
+        for (const node in result) {
+            result[node].nodeIds.sort()
+        }
+        setNodeConfig(result)
+
+        if (isFormDataBody) {
+            setValues(formDataRequest())
+        } else {
+            setValues(jsonDataRequest())
+        }
+    }
+
+    const handleAccordionChange = (nodeLabel) => (event, isExpanded) => {
+        const accordianNodes = { ...nodeConfigExpanded }
+        accordianNodes[nodeLabel] = isExpanded
+        setNodeConfigExpanded(accordianNodes)
+    }
+
+    useEffect(() => {
+        if (getConfigApi.data) {
+            groupByNodeLabel(getConfigApi.data)
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getConfigApi.data])
+
+    useEffect(() => {
+        if (show && dialogProps) {
+            getConfigApi.request(dialogProps.storeId, dialogProps.loaderId)
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [show, dialogProps])
+
+    const portalElement = document.getElementById('portal')
+
+    const component = show ? (
+        <Dialog
+            onClose={onCancel}
+            open={show}
+            fullWidth
+            maxWidth='lg'
+            aria-labelledby='alert-dialog-title'
+            aria-describedby='alert-dialog-description'
+        >
+            <DialogTitle sx={{ fontSize: '1rem' }} id='alert-dialog-title'>
+                {dialogProps.title}
+            </DialogTitle>
+            <DialogContent>
+                <MemoizedReactMarkdown
+                    remarkPlugins={[remarkGfm, remarkMath]}
+                    rehypePlugins={[rehypeMathjax, rehypeRaw]}
+                    components={{
+                        code({ inline, className, children, ...props }) {
+                            const match = /language-(\w+)/.exec(className || '')
+                            return !inline ? (
+                                <CodeBlock
+                                    isDialog={true}
+                                    language={(match && match[1]) || ''}
+                                    value={String(children).replace(/\n$/, '')}
+                                    {...props}
+                                />
+                            ) : (
+                                <code className={className} {...props}>
+                                    {children}
+                                </code>
+                            )
+                        }
+                    }}
+                >
+                    {values}
+                </MemoizedReactMarkdown>
+
+                <Typography sx={{ mt: 3, mb: 1 }}>You can override existing configurations:</Typography>
+
+                <Stack direction='column' spacing={2} sx={{ width: '100%', my: 2 }}>
+                    <Card sx={{ borderColor: theme.palette.primary[200] + 75, p: 2 }} variant='outlined'>
+                        {Object.keys(nodeConfig)
+                            .sort()
+                            .map((nodeLabel) => (
+                                <Accordion
+                                    expanded={nodeConfigExpanded[nodeLabel] || false}
+                                    onChange={handleAccordionChange(nodeLabel)}
+                                    key={nodeLabel}
+                                    disableGutters
+                                >
+                                    <AccordionSummary
+                                        expandIcon={<ExpandMoreIcon />}
+                                        aria-controls={`nodes-accordian-${nodeLabel}`}
+                                        id={`nodes-accordian-header-${nodeLabel}`}
+                                    >
+                                        <Stack flexDirection='row' sx={{ gap: 2, alignItems: 'center', flexWrap: 'wrap' }}>
+                                            <Typography variant='h5'>{nodeLabel}</Typography>
+                                            {nodeConfig[nodeLabel].nodeIds.length > 0 &&
+                                                nodeConfig[nodeLabel].nodeIds.map((nodeId, index) => (
+                                                    <div
+                                                        key={index}
+                                                        style={{
+                                                            display: 'flex',
+                                                            flexDirection: 'row',
+                                                            width: 'max-content',
+                                                            borderRadius: 15,
+                                                            background: 'rgb(254,252,191)',
+                                                            padding: 5,
+                                                            paddingLeft: 10,
+                                                            paddingRight: 10
+                                                        }}
+                                                    >
+                                                        <span
+                                                            style={{
+                                                                color: 'rgb(116,66,16)',
+                                                                fontSize: '0.825rem'
+                                                            }}
+                                                        >
+                                                            {nodeId}
+                                                        </span>
+                                                    </div>
+                                                ))}
+                                        </Stack>
+                                    </AccordionSummary>
+                                    <AccordionDetails>
+                                        <TableViewOnly
+                                            rows={nodeConfig[nodeLabel].params.map((obj) => {
+                                                // eslint-disable-next-line
+                                                const { node, nodeId, ...rest } = obj
+                                                return rest
+                                            })}
+                                            columns={Object.keys(nodeConfig[nodeLabel].params[0]).slice(-3)}
+                                        />
+                                    </AccordionDetails>
+                                </Accordion>
+                            ))}
+                    </Card>
+                </Stack>
+            </DialogContent>
+        </Dialog>
+    ) : null
+
+    return createPortal(component, portalElement)
+}
+
+DocStoreAPIDialog.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onCancel: PropTypes.func
+}
+
+export default DocStoreAPIDialog

packages/ui/src/views/docstore/DocumentStoreDetail.jsx

@@ -37,6 +37,7 @@ import ViewHeader from '@/layout/MainLayout/ViewHeader'
 import DeleteDocStoreDialog from './DeleteDocStoreDialog'
 import DocumentStoreStatus from '@/views/docstore/DocumentStoreStatus'
 import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
+import DocStoreAPIDialog from './DocStoreAPIDialog'
 
 // API
 import documentsApi from '@/api/documentstore'
@@ -56,6 +57,7 @@ import FileChunksIcon from '@mui/icons-material/AppRegistration'
 import NoteAddIcon from '@mui/icons-material/NoteAdd'
 import SearchIcon from '@mui/icons-material/Search'
 import RefreshIcon from '@mui/icons-material/Refresh'
+import CodeIcon from '@mui/icons-material/Code'
 import doc_store_details_emptySVG from '@/assets/images/doc_store_details_empty.svg'
 
 // store
@@ -142,6 +144,8 @@ const DocumentStoreDetails = () => {
     const [documentLoaderListDialogProps, setDocumentLoaderListDialogProps] = useState({})
     const [showDeleteDocStoreDialog, setShowDeleteDocStoreDialog] = useState(false)
     const [deleteDocStoreDialogProps, setDeleteDocStoreDialogProps] = useState({})
+    const [showDocStoreAPIDialog, setShowDocStoreAPIDialog] = useState(false)
+    const [docStoreAPIDialogProps, setDocStoreAPIDialogProps] = useState({})
 
     const [anchorEl, setAnchorEl] = useState(null)
     const open = Boolean(anchorEl)
@@ -374,6 +378,16 @@ const DocumentStoreDetails = () => {
         setAnchorEl(event.currentTarget)
     }
 
+    const onViewUpsertAPI = (storeId, loaderId) => {
+        const props = {
+            title: `Upsert API`,
+            storeId,
+            loaderId
+        }
+        setDocStoreAPIDialogProps(props)
+        setShowDocStoreAPIDialog(true)
+    }
+
     const handleClose = () => {
         setAnchorEl(null)
     }
@@ -650,6 +664,7 @@ const DocumentStoreDetails = () => {
                                                             onChunkUpsert={() =>
                                                                 navigate(`/document-stores/vector/${documentStore.id}/${loader.id}`)
                                                             }
+                                                            onViewUpsertAPI={() => onViewUpsertAPI(documentStore.id, loader.id)}
                                                         />
                                                     ))}
                                             </>
@@ -695,6 +710,13 @@ const DocumentStoreDetails = () => {
                     onDelete={onDocStoreDelete}
                 />
             )}
+            {showDocStoreAPIDialog && (
+                <DocStoreAPIDialog
+                    show={showDocStoreAPIDialog}
+                    dialogProps={docStoreAPIDialogProps}
+                    onCancel={() => setShowDocStoreAPIDialog(false)}
+                />
+            )}
             {isBackdropLoading && <BackdropLoader open={isBackdropLoading} />}
             <ConfirmDialog />
         </>
@@ -784,6 +806,10 @@ function LoaderRow(props) {
                                 <NoteAddIcon />
                                 Upsert Chunks
                             </MenuItem>
+                            <MenuItem onClick={props.onViewUpsertAPI} disableRipple>
+                                <CodeIcon />
+                                View API
+                            </MenuItem>
                             <Divider sx={{ my: 0.5 }} />
                             <MenuItem onClick={props.onDeleteClick} disableRipple>
                                 <FileDeleteIcon />
@@ -805,6 +831,7 @@ LoaderRow.propTypes = {
     onViewChunksClick: PropTypes.func,
     onEditClick: PropTypes.func,
     onDeleteClick: PropTypes.func,
-    onChunkUpsert: PropTypes.func
+    onChunkUpsert: PropTypes.func,
+    onViewUpsertAPI: PropTypes.func
 }
 export default DocumentStoreDetails

packages/ui/src/views/vectorstore/VectorStoreDialog.jsx

@@ -23,7 +23,7 @@ import { CheckboxInput } from '@/ui-component/checkbox/Checkbox'
 import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
 import { TableViewOnly } from '@/ui-component/table/Table'
 
-import { IconX, IconBulb } from '@tabler/icons-react'
+import { IconX, IconBulb, IconExclamationCircle } from '@tabler/icons-react'
 import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
 import pythonSVG from '@/assets/images/python.svg'
 import javascriptSVG from '@/assets/images/javascript.svg'
@@ -545,6 +545,47 @@ formData.append("openAIApiKey[openAIEmbeddings_0]", "sk-my-openai-2nd-key")`
                                                                     showLineNumbers={false}
                                                                     wrapLines
                                                                 />
+                                                                <div
+                                                                    style={{
+                                                                        display: 'flex',
+                                                                        flexDirection: 'column',
+                                                                        borderRadius: 10,
+                                                                        background: 'rgb(254,252,191)',
+                                                                        padding: 10,
+                                                                        marginTop: 20,
+                                                                        marginBottom: 20
+                                                                    }}
+                                                                >
+                                                                    <div
+                                                                        style={{
+                                                                            display: 'flex',
+                                                                            flexDirection: 'row',
+                                                                            alignItems: 'center'
+                                                                        }}
+                                                                    >
+                                                                        <IconExclamationCircle size={30} color='rgb(116,66,16)' />
+                                                                        <span
+                                                                            style={{
+                                                                                color: 'rgb(116,66,16)',
+                                                                                marginLeft: 10,
+                                                                                fontWeight: 500
+                                                                            }}
+                                                                        >
+                                                                            {
+                                                                                'For security reason, override config is disabled by default. You can change this by going into Chatflow Configuration -> Security tab, and enable the property you want to override.'
+                                                                            }
+                                                                            &nbsp;Refer{' '}
+                                                                            <a
+                                                                                rel='noreferrer'
+                                                                                target='_blank'
+                                                                                href='https://docs.flowiseai.com/using-flowise/api#override-config'
+                                                                            >
+                                                                                here
+                                                                            </a>{' '}
+                                                                            for more details
+                                                                        </span>
+                                                                    </div>
+                                                                </div>
                                                                 <div
                                                                     style={{
                                                                         display: 'flex',