Feature/extract import all (#2796)

* use existing route to get all chatflows

* add export all chatflows functionality

* add read exported all chatflows json file functionality

* add save chatflows functionality in server

* chore rename saveChatflows to importChatflows and others

* chore rewrite snackbar message

* fix import chatflows when no data in chatflows db

* add handle when import file array length is 0

* chore update and add meaning comment in importChatflows

* update method of storing flowdata for importChatflows function

* Refresh/redirect to chatflows when import is successful

* fix lint

---------

Co-authored-by: Ilango <rajagopalilango@gmail.com>

packages/server/src/controllers/chatflows/index.ts

@@ -1,11 +1,11 @@
-import { Request, Response, NextFunction } from 'express'
+import { NextFunction, Request, Response } from 'express'
-import chatflowsService from '../../services/chatflows'
-import { ChatFlow } from '../../database/entities/ChatFlow'
-import { createRateLimiter } from '../../utils/rateLimit'
-import { getApiKey } from '../../utils/apiKey'
-import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
+import { ChatFlow } from '../../database/entities/ChatFlow'
+import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { ChatflowType } from '../../Interface'
+import chatflowsService from '../../services/chatflows'
+import { getApiKey } from '../../utils/apiKey'
+import { createRateLimiter } from '../../utils/rateLimit'
 
 const checkIfChatflowIsValidForStreaming = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -105,6 +105,16 @@ const saveChatflow = async (req: Request, res: Response, next: NextFunction) =>
     }
 }
 
+const importChatflows = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const chatflows: Partial<ChatFlow>[] = req.body.Chatflows
+        const apiResponse = await chatflowsService.importChatflows(chatflows)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
 const updateChatflow = async (req: Request, res: Response, next: NextFunction) => {
     try {
         if (typeof req.params === 'undefined' || !req.params.id) {
@@ -167,6 +177,7 @@ export default {
     getChatflowByApiKey,
     getChatflowById,
     saveChatflow,
+    importChatflows,
     updateChatflow,
     getSinglePublicChatflow,
     getSinglePublicChatbotConfig

packages/server/src/routes/chatflows/index.ts

@@ -4,6 +4,7 @@ const router = express.Router()
 
 // CREATE
 router.post('/', chatflowsController.saveChatflow)
+router.post('/importchatflows', chatflowsController.importChatflows)
 
 // READ
 router.get('/', chatflowsController.getAllChatflows)

packages/server/src/routes/index.ts

@@ -1,10 +1,10 @@
 import express from 'express'
 import apikeyRouter from './apikey'
 import assistantsRouter from './assistants'
+import chatMessageRouter from './chat-messages'
 import chatflowsRouter from './chatflows'
 import chatflowsStreamingRouter from './chatflows-streaming'
 import chatflowsUploadsRouter from './chatflows-uploads'
-import chatMessageRouter from './chat-messages'
 import componentsCredentialsRouter from './components-credentials'
 import componentsCredentialsIconRouter from './components-credentials-icon'
 import credentialsRouter from './credentials'
@@ -12,10 +12,10 @@ import documentStoreRouter from './documentstore'
 import feedbackRouter from './feedback'
 import fetchLinksRouter from './fetch-links'
 import flowConfigRouter from './flow-config'
-import internalChatmessagesRouter from './internal-chat-messages'
-import internalPredictionRouter from './internal-predictions'
 import getUploadFileRouter from './get-upload-file'
 import getUploadPathRouter from './get-upload-path'
+import internalChatmessagesRouter from './internal-chat-messages'
+import internalPredictionRouter from './internal-predictions'
 import leadsRouter from './leads'
 import loadPromptRouter from './load-prompts'
 import marketplacesRouter from './marketplaces'
@@ -27,18 +27,18 @@ import nodesRouter from './nodes'
 import openaiAssistantsRouter from './openai-assistants'
 import openaiAssistantsFileRouter from './openai-assistants-files'
 import openaiAssistantsVectorStoreRouter from './openai-assistants-vector-store'
+import pingRouter from './ping'
 import predictionRouter from './predictions'
 import promptListsRouter from './prompts-lists'
 import publicChatbotRouter from './public-chatbots'
 import publicChatflowsRouter from './public-chatflows'
 import statsRouter from './stats'
 import toolsRouter from './tools'
+import upsertHistoryRouter from './upsert-history'
 import variablesRouter from './variables'
 import vectorRouter from './vectors'
 import verifyRouter from './verify'
 import versionRouter from './versions'
-import upsertHistoryRouter from './upsert-history'
-import pingRouter from './ping'
 
 const router = express.Router()
 

packages/server/src/services/chatflows/index.ts

@@ -1,19 +1,18 @@
-import { StatusCodes } from 'http-status-codes'
-import { InternalFlowiseError } from '../../errors/internalFlowiseError'
-import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
-import { ChatflowType, IChatFlow } from '../../Interface'
-import { ChatFlow } from '../../database/entities/ChatFlow'
-import { getAppVersion, getTelemetryFlowObj, isFlowValidForStream, constructGraphs, getEndingNodes } from '../../utils'
-import logger from '../../utils/logger'
 import { removeFolderFromStorage } from 'flowise-components'
-import { IReactFlowObject } from '../../Interface'
+import { StatusCodes } from 'http-status-codes'
-import { utilGetUploadsConfig } from '../../utils/getUploadsConfig'
+import { ChatflowType, IChatFlow, IReactFlowObject } from '../../Interface'
+import { ChatFlow } from '../../database/entities/ChatFlow'
 import { ChatMessage } from '../../database/entities/ChatMessage'
 import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback'
 import { UpsertHistory } from '../../database/entities/UpsertHistory'
-import { containsBase64File, updateFlowDataWithFilePaths } from '../../utils/fileRepository'
+import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { getErrorMessage } from '../../errors/utils'
 import documentStoreService from '../../services/documentstore'
+import { constructGraphs, getAppVersion, getEndingNodes, getTelemetryFlowObj, isFlowValidForStream } from '../../utils'
+import { containsBase64File, updateFlowDataWithFilePaths } from '../../utils/fileRepository'
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
+import { utilGetUploadsConfig } from '../../utils/getUploadsConfig'
+import logger from '../../utils/logger'
 
 // Check if chatflow valid for streaming
 const checkIfChatflowIsValidForStreaming = async (chatflowId: string): Promise<any> => {
@@ -198,6 +197,60 @@ const saveChatflow = async (newChatFlow: ChatFlow): Promise<any> => {
     }
 }
 
+const importChatflows = async (newChatflows: Partial<ChatFlow>[]): Promise<any> => {
+    try {
+        const appServer = getRunningExpressApp()
+
+        // step 1 - check whether file chatflows array is zero
+        if (newChatflows.length == 0) throw new Error('No chatflows in this file.')
+
+        // step 2 - check whether ids are duplicate in database
+        let ids = '('
+        let count: number = 0
+        const lastCount = newChatflows.length - 1
+        newChatflows.forEach((newChatflow) => {
+            ids += `'${newChatflow.id}'`
+            if (lastCount != count) ids += ','
+            if (lastCount == count) ids += ')'
+            count += 1
+        })
+
+        const selectResponse = await appServer.AppDataSource.getRepository(ChatFlow)
+            .createQueryBuilder('cf')
+            .select('cf.id')
+            .where(`cf.id IN ${ids}`)
+            .getMany()
+        const foundIds = selectResponse.map((response) => {
+            return response.id
+        })
+
+        // step 3 - remove ids that are only duplicate
+        const prepChatflows: Partial<ChatFlow>[] = newChatflows.map((newChatflow) => {
+            let id: string = ''
+            if (newChatflow.id) id = newChatflow.id
+            let flowData: string = ''
+            if (newChatflow.flowData) flowData = newChatflow.flowData
+            if (foundIds.includes(id)) {
+                newChatflow.id = undefined
+                newChatflow.name += ' with new id'
+            }
+            newChatflow.type = 'CHATFLOW'
+            newChatflow.flowData = JSON.stringify(JSON.parse(flowData))
+            return newChatflow
+        })
+
+        // step 4 - transactional insert array of entities
+        const insertResponse = await appServer.AppDataSource.getRepository(ChatFlow).insert(prepChatflows)
+
+        return insertResponse
+    } catch (error) {
+        throw new InternalFlowiseError(
+            StatusCodes.INTERNAL_SERVER_ERROR,
+            `Error: chatflowsService.saveChatflows - ${getErrorMessage(error)}`
+        )
+    }
+}
+
 const updateChatflow = async (chatflow: ChatFlow, updateChatFlow: ChatFlow): Promise<any> => {
     try {
         const appServer = getRunningExpressApp()
@@ -299,6 +352,7 @@ export default {
     getChatflowByApiKey,
     getChatflowById,
     saveChatflow,
+    importChatflows,
     updateChatflow,
     getSinglePublicChatflow,
     getSinglePublicChatbotConfig

packages/ui/src/api/chatflows.js

@@ -10,6 +10,8 @@ const getSpecificChatflowFromPublicEndpoint = (id) => client.get(`/public-chatfl
 
 const createNewChatflow = (body) => client.post(`/chatflows`, body)
 
+const importChatflows = (body) => client.post(`/chatflows/importchatflows`, body)
+
 const updateChatflow = (id, body) => client.put(`/chatflows/${id}`, body)
 
 const deleteChatflow = (id) => client.delete(`/chatflows/${id}`)
@@ -24,6 +26,7 @@ export default {
     getSpecificChatflow,
     getSpecificChatflowFromPublicEndpoint,
     createNewChatflow,
+    importChatflows,
     updateChatflow,
     deleteChatflow,
     getIsChatflowStreaming,

packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx

@@ -1,13 +1,15 @@
-import { useState, useRef, useEffect } from 'react'
+import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction, MENU_OPEN, REMOVE_DIRTY } from '@/store/actions'
+import { sanitizeChatflows } from '@/utils/genericHelper'
+import useNotifier from '@/utils/useNotifier'
 import PropTypes from 'prop-types'
-import { useSelector } from 'react-redux'
+import { useEffect, useRef, useState } from 'react'
-
+import { useDispatch, useSelector } from 'react-redux'
 // material-ui
-import { useTheme } from '@mui/material/styles'
 import {
-    Box,
-    ButtonBase,
     Avatar,
+    Box,
+    Button,
+    ButtonBase,
     ClickAwayListener,
     Divider,
     List,
@@ -18,20 +20,27 @@ import {
     Popper,
     Typography
 } from '@mui/material'
+import { useTheme } from '@mui/material/styles'
 
 // third-party
 import PerfectScrollbar from 'react-perfect-scrollbar'
 
 // project imports
 import MainCard from '@/ui-component/cards/MainCard'
-import Transitions from '@/ui-component/extended/Transitions'
 import AboutDialog from '@/ui-component/dialog/AboutDialog'
+import Transitions from '@/ui-component/extended/Transitions'
 
 // assets
-import { IconLogout, IconSettings, IconInfoCircle } from '@tabler/icons-react'
+import { IconFileExport, IconFileUpload, IconInfoCircle, IconLogout, IconSettings, IconX } from '@tabler/icons-react'
-
 import './index.css'
 
+//API
+import chatFlowsApi from '@/api/chatflows'
+
+// Hooks
+import useApi from '@/hooks/useApi'
+import { useLocation, useNavigate } from 'react-router-dom'
+
 // ==============================|| PROFILE MENU ||============================== //
 
 const ProfileSection = ({ username, handleLogout }) => {
@@ -43,6 +52,17 @@ const ProfileSection = ({ username, handleLogout }) => {
     const [aboutDialogOpen, setAboutDialogOpen] = useState(false)
 
     const anchorRef = useRef(null)
+    const inputRef = useRef()
+
+    const navigate = useNavigate()
+    const location = useLocation()
+
+    // ==============================|| Snackbar ||============================== //
+
+    useNotifier()
+    const dispatch = useDispatch()
+    const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
+    const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
 
     const handleClose = (event) => {
         if (anchorRef.current && anchorRef.current.contains(event.target)) {
@@ -55,6 +75,106 @@ const ProfileSection = ({ username, handleLogout }) => {
         setOpen((prevOpen) => !prevOpen)
     }
 
+    const errorFailed = (message) => {
+        enqueueSnackbar({
+            message: message,
+            options: {
+                key: new Date().getTime() + Math.random(),
+                variant: 'error',
+                persist: true,
+                action: (key) => (
+                    <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                        <IconX />
+                    </Button>
+                )
+            }
+        })
+    }
+    const importChatflowsApi = useApi(chatFlowsApi.importChatflows)
+    const fileChange = (e) => {
+        if (!e.target.files) return
+
+        const file = e.target.files[0]
+
+        const reader = new FileReader()
+        reader.onload = (evt) => {
+            if (!evt?.target?.result) {
+                return
+            }
+            const chatflows = JSON.parse(evt.target.result)
+            importChatflowsApi.request(chatflows)
+        }
+        reader.readAsText(file)
+    }
+
+    const importChatflowsSuccess = () => {
+        dispatch({ type: REMOVE_DIRTY })
+        enqueueSnackbar({
+            message: `Import chatflows successful`,
+            options: {
+                key: new Date().getTime() + Math.random(),
+                variant: 'success',
+                action: (key) => (
+                    <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                        <IconX />
+                    </Button>
+                )
+            }
+        })
+    }
+    useEffect(() => {
+        if (importChatflowsApi.error) errorFailed(`Failed to import chatflows: ${importChatflowsApi.error.response.data.message}`)
+        if (importChatflowsApi.data) {
+            importChatflowsSuccess()
+            // if current location is /chatflows, refresh the page
+            if (location.pathname === '/chatflows') navigate(0)
+            else {
+                // if not redirect to /chatflows
+                dispatch({ type: MENU_OPEN, id: 'chatflows' })
+                navigate('/chatflows')
+            }
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [importChatflowsApi.error, importChatflowsApi.data])
+    const importAllChatflows = () => {
+        inputRef.current.click()
+    }
+    const getAllChatflowsApi = useApi(chatFlowsApi.getAllChatflows)
+
+    const exportChatflowsSuccess = () => {
+        dispatch({ type: REMOVE_DIRTY })
+        enqueueSnackbar({
+            message: `Export chatflows successful`,
+            options: {
+                key: new Date().getTime() + Math.random(),
+                variant: 'success',
+                action: (key) => (
+                    <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                        <IconX />
+                    </Button>
+                )
+            }
+        })
+    }
+
+    useEffect(() => {
+        if (getAllChatflowsApi.error) errorFailed(`Failed to export Chatflows: ${getAllChatflowsApi.error.response.data.message}`)
+        if (getAllChatflowsApi.data) {
+            const sanitizedChatflows = sanitizeChatflows(getAllChatflowsApi.data)
+            const dataStr = JSON.stringify({ Chatflows: sanitizedChatflows }, null, 2)
+            const dataUri = 'data:application/json;charset=utf-8,' + encodeURIComponent(dataStr)
+
+            const exportFileDefaultName = 'AllChatflows.json'
+
+            const linkElement = document.createElement('a')
+            linkElement.setAttribute('href', dataUri)
+            linkElement.setAttribute('download', exportFileDefaultName)
+            linkElement.click()
+            exportChatflowsSuccess()
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getAllChatflowsApi.error, getAllChatflowsApi.data])
+
     const prevOpen = useRef(open)
     useEffect(() => {
         if (prevOpen.current === true && open === false) {
@@ -135,6 +255,29 @@ const ProfileSection = ({ username, handleLogout }) => {
                                                     }
                                                 }}
                                             >
+                                                <ListItemButton
+                                                    sx={{ borderRadius: `${customization.borderRadius}px` }}
+                                                    onClick={() => {
+                                                        getAllChatflowsApi.request()
+                                                    }}
+                                                >
+                                                    <ListItemIcon>
+                                                        <IconFileExport stroke={1.5} size='1.3rem' />
+                                                    </ListItemIcon>
+                                                    <ListItemText primary={<Typography variant='body2'>Export Chatflows</Typography>} />
+                                                </ListItemButton>
+                                                <ListItemButton
+                                                    sx={{ borderRadius: `${customization.borderRadius}px` }}
+                                                    onClick={() => {
+                                                        importAllChatflows()
+                                                    }}
+                                                >
+                                                    <ListItemIcon>
+                                                        <IconFileUpload stroke={1.5} size='1.3rem' />
+                                                    </ListItemIcon>
+                                                    <ListItemText primary={<Typography variant='body2'>Import Chatflows</Typography>} />
+                                                </ListItemButton>
+                                                <input ref={inputRef} type='file' hidden onChange={fileChange} />
                                                 <ListItemButton
                                                     sx={{ borderRadius: `${customization.borderRadius}px` }}
                                                     onClick={() => {

packages/ui/src/utils/genericHelper.js

@@ -340,6 +340,18 @@ export const getFolderName = (base64ArrayStr) => {
     }
 }
 
+export const sanitizeChatflows = (arrayChatflows) => {
+    const sanitizedChatflows = arrayChatflows.map((chatFlow) => {
+        const sanitizeFlowData = generateExportFlowData(JSON.parse(chatFlow.flowData))
+        return {
+            id: chatFlow.id,
+            name: chatFlow.name,
+            flowData: JSON.stringify(sanitizeFlowData, null, 2)
+        }
+    })
+    return sanitizedChatflows
+}
+
 export const generateExportFlowData = (flowData) => {
     const nodes = flowData.nodes
     const edges = flowData.edges