From 95b2cf7b7f4ab1193b771e212cd91f57f76c2f11 Mon Sep 17 00:00:00 2001 From: Ong Chung Yau <33013947+chungyau97@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:47:41 +0800 Subject: [PATCH] Feature/extract import all (#2796) * use existing route to get all chatflows * add export all chatflows functionality * add read exported all chatflows json file functionality * add save chatflows functionality in server * chore rename saveChatflows to importChatflows and others * chore rewrite snackbar message * fix import chatflows when no data in chatflows db * add handle when import file array length is 0 * chore update and add meaning comment in importChatflows * update method of storing flowdata for importChatflows function * Refresh/redirect to chatflows when import is successful * fix lint --------- Co-authored-by: Ilango --- .../server/src/controllers/chatflows/index.ts | 23 ++- packages/server/src/routes/chatflows/index.ts | 1 + packages/server/src/routes/index.ts | 10 +- .../server/src/services/chatflows/index.ts | 74 ++++++-- packages/ui/src/api/chatflows.js | 3 + .../Header/ProfileSection/index.jsx | 161 +++++++++++++++++- packages/ui/src/utils/genericHelper.js | 12 ++ 7 files changed, 254 insertions(+), 30 deletions(-) diff --git a/packages/server/src/controllers/chatflows/index.ts b/packages/server/src/controllers/chatflows/index.ts index f6e903188..61ba0691d 100644 --- a/packages/server/src/controllers/chatflows/index.ts +++ b/packages/server/src/controllers/chatflows/index.ts @@ -1,11 +1,11 @@ -import { Request, Response, NextFunction } from 'express' -import chatflowsService from '../../services/chatflows' -import { ChatFlow } from '../../database/entities/ChatFlow' -import { createRateLimiter } from '../../utils/rateLimit' -import { getApiKey } from '../../utils/apiKey' -import { InternalFlowiseError } from '../../errors/internalFlowiseError' +import { NextFunction, Request, Response } from 'express' import { StatusCodes } from 'http-status-codes' +import { ChatFlow } from '../../database/entities/ChatFlow' +import { InternalFlowiseError } from '../../errors/internalFlowiseError' import { ChatflowType } from '../../Interface' +import chatflowsService from '../../services/chatflows' +import { getApiKey } from '../../utils/apiKey' +import { createRateLimiter } from '../../utils/rateLimit' const checkIfChatflowIsValidForStreaming = async (req: Request, res: Response, next: NextFunction) => { try { @@ -105,6 +105,16 @@ const saveChatflow = async (req: Request, res: Response, next: NextFunction) => } } +const importChatflows = async (req: Request, res: Response, next: NextFunction) => { + try { + const chatflows: Partial[] = req.body.Chatflows + const apiResponse = await chatflowsService.importChatflows(chatflows) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + const updateChatflow = async (req: Request, res: Response, next: NextFunction) => { try { if (typeof req.params === 'undefined' || !req.params.id) { @@ -167,6 +177,7 @@ export default { getChatflowByApiKey, getChatflowById, saveChatflow, + importChatflows, updateChatflow, getSinglePublicChatflow, getSinglePublicChatbotConfig diff --git a/packages/server/src/routes/chatflows/index.ts b/packages/server/src/routes/chatflows/index.ts index 8fb457fd3..b0c535067 100644 --- a/packages/server/src/routes/chatflows/index.ts +++ b/packages/server/src/routes/chatflows/index.ts @@ -4,6 +4,7 @@ const router = express.Router() // CREATE router.post('/', chatflowsController.saveChatflow) +router.post('/importchatflows', chatflowsController.importChatflows) // READ router.get('/', chatflowsController.getAllChatflows) diff --git a/packages/server/src/routes/index.ts b/packages/server/src/routes/index.ts index 9c6e65087..075b28fb9 100644 --- a/packages/server/src/routes/index.ts +++ b/packages/server/src/routes/index.ts @@ -1,10 +1,10 @@ import express from 'express' import apikeyRouter from './apikey' import assistantsRouter from './assistants' +import chatMessageRouter from './chat-messages' import chatflowsRouter from './chatflows' import chatflowsStreamingRouter from './chatflows-streaming' import chatflowsUploadsRouter from './chatflows-uploads' -import chatMessageRouter from './chat-messages' import componentsCredentialsRouter from './components-credentials' import componentsCredentialsIconRouter from './components-credentials-icon' import credentialsRouter from './credentials' @@ -12,10 +12,10 @@ import documentStoreRouter from './documentstore' import feedbackRouter from './feedback' import fetchLinksRouter from './fetch-links' import flowConfigRouter from './flow-config' -import internalChatmessagesRouter from './internal-chat-messages' -import internalPredictionRouter from './internal-predictions' import getUploadFileRouter from './get-upload-file' import getUploadPathRouter from './get-upload-path' +import internalChatmessagesRouter from './internal-chat-messages' +import internalPredictionRouter from './internal-predictions' import leadsRouter from './leads' import loadPromptRouter from './load-prompts' import marketplacesRouter from './marketplaces' @@ -27,18 +27,18 @@ import nodesRouter from './nodes' import openaiAssistantsRouter from './openai-assistants' import openaiAssistantsFileRouter from './openai-assistants-files' import openaiAssistantsVectorStoreRouter from './openai-assistants-vector-store' +import pingRouter from './ping' import predictionRouter from './predictions' import promptListsRouter from './prompts-lists' import publicChatbotRouter from './public-chatbots' import publicChatflowsRouter from './public-chatflows' import statsRouter from './stats' import toolsRouter from './tools' +import upsertHistoryRouter from './upsert-history' import variablesRouter from './variables' import vectorRouter from './vectors' import verifyRouter from './verify' import versionRouter from './versions' -import upsertHistoryRouter from './upsert-history' -import pingRouter from './ping' const router = express.Router() diff --git a/packages/server/src/services/chatflows/index.ts b/packages/server/src/services/chatflows/index.ts index e68e02b95..e4708aedd 100644 --- a/packages/server/src/services/chatflows/index.ts +++ b/packages/server/src/services/chatflows/index.ts @@ -1,19 +1,18 @@ -import { StatusCodes } from 'http-status-codes' -import { InternalFlowiseError } from '../../errors/internalFlowiseError' -import { getRunningExpressApp } from '../../utils/getRunningExpressApp' -import { ChatflowType, IChatFlow } from '../../Interface' -import { ChatFlow } from '../../database/entities/ChatFlow' -import { getAppVersion, getTelemetryFlowObj, isFlowValidForStream, constructGraphs, getEndingNodes } from '../../utils' -import logger from '../../utils/logger' import { removeFolderFromStorage } from 'flowise-components' -import { IReactFlowObject } from '../../Interface' -import { utilGetUploadsConfig } from '../../utils/getUploadsConfig' +import { StatusCodes } from 'http-status-codes' +import { ChatflowType, IChatFlow, IReactFlowObject } from '../../Interface' +import { ChatFlow } from '../../database/entities/ChatFlow' import { ChatMessage } from '../../database/entities/ChatMessage' import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback' import { UpsertHistory } from '../../database/entities/UpsertHistory' -import { containsBase64File, updateFlowDataWithFilePaths } from '../../utils/fileRepository' +import { InternalFlowiseError } from '../../errors/internalFlowiseError' import { getErrorMessage } from '../../errors/utils' import documentStoreService from '../../services/documentstore' +import { constructGraphs, getAppVersion, getEndingNodes, getTelemetryFlowObj, isFlowValidForStream } from '../../utils' +import { containsBase64File, updateFlowDataWithFilePaths } from '../../utils/fileRepository' +import { getRunningExpressApp } from '../../utils/getRunningExpressApp' +import { utilGetUploadsConfig } from '../../utils/getUploadsConfig' +import logger from '../../utils/logger' // Check if chatflow valid for streaming const checkIfChatflowIsValidForStreaming = async (chatflowId: string): Promise => { @@ -198,6 +197,60 @@ const saveChatflow = async (newChatFlow: ChatFlow): Promise => { } } +const importChatflows = async (newChatflows: Partial[]): Promise => { + try { + const appServer = getRunningExpressApp() + + // step 1 - check whether file chatflows array is zero + if (newChatflows.length == 0) throw new Error('No chatflows in this file.') + + // step 2 - check whether ids are duplicate in database + let ids = '(' + let count: number = 0 + const lastCount = newChatflows.length - 1 + newChatflows.forEach((newChatflow) => { + ids += `'${newChatflow.id}'` + if (lastCount != count) ids += ',' + if (lastCount == count) ids += ')' + count += 1 + }) + + const selectResponse = await appServer.AppDataSource.getRepository(ChatFlow) + .createQueryBuilder('cf') + .select('cf.id') + .where(`cf.id IN ${ids}`) + .getMany() + const foundIds = selectResponse.map((response) => { + return response.id + }) + + // step 3 - remove ids that are only duplicate + const prepChatflows: Partial[] = newChatflows.map((newChatflow) => { + let id: string = '' + if (newChatflow.id) id = newChatflow.id + let flowData: string = '' + if (newChatflow.flowData) flowData = newChatflow.flowData + if (foundIds.includes(id)) { + newChatflow.id = undefined + newChatflow.name += ' with new id' + } + newChatflow.type = 'CHATFLOW' + newChatflow.flowData = JSON.stringify(JSON.parse(flowData)) + return newChatflow + }) + + // step 4 - transactional insert array of entities + const insertResponse = await appServer.AppDataSource.getRepository(ChatFlow).insert(prepChatflows) + + return insertResponse + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: chatflowsService.saveChatflows - ${getErrorMessage(error)}` + ) + } +} + const updateChatflow = async (chatflow: ChatFlow, updateChatFlow: ChatFlow): Promise => { try { const appServer = getRunningExpressApp() @@ -299,6 +352,7 @@ export default { getChatflowByApiKey, getChatflowById, saveChatflow, + importChatflows, updateChatflow, getSinglePublicChatflow, getSinglePublicChatbotConfig diff --git a/packages/ui/src/api/chatflows.js b/packages/ui/src/api/chatflows.js index aa305d14f..fb76d5d39 100644 --- a/packages/ui/src/api/chatflows.js +++ b/packages/ui/src/api/chatflows.js @@ -10,6 +10,8 @@ const getSpecificChatflowFromPublicEndpoint = (id) => client.get(`/public-chatfl const createNewChatflow = (body) => client.post(`/chatflows`, body) +const importChatflows = (body) => client.post(`/chatflows/importchatflows`, body) + const updateChatflow = (id, body) => client.put(`/chatflows/${id}`, body) const deleteChatflow = (id) => client.delete(`/chatflows/${id}`) @@ -24,6 +26,7 @@ export default { getSpecificChatflow, getSpecificChatflowFromPublicEndpoint, createNewChatflow, + importChatflows, updateChatflow, deleteChatflow, getIsChatflowStreaming, diff --git a/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx b/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx index da8d2e1af..a742e312a 100644 --- a/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx +++ b/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx @@ -1,13 +1,15 @@ -import { useState, useRef, useEffect } from 'react' +import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction, MENU_OPEN, REMOVE_DIRTY } from '@/store/actions' +import { sanitizeChatflows } from '@/utils/genericHelper' +import useNotifier from '@/utils/useNotifier' import PropTypes from 'prop-types' -import { useSelector } from 'react-redux' - +import { useEffect, useRef, useState } from 'react' +import { useDispatch, useSelector } from 'react-redux' // material-ui -import { useTheme } from '@mui/material/styles' import { - Box, - ButtonBase, Avatar, + Box, + Button, + ButtonBase, ClickAwayListener, Divider, List, @@ -18,20 +20,27 @@ import { Popper, Typography } from '@mui/material' +import { useTheme } from '@mui/material/styles' // third-party import PerfectScrollbar from 'react-perfect-scrollbar' // project imports import MainCard from '@/ui-component/cards/MainCard' -import Transitions from '@/ui-component/extended/Transitions' import AboutDialog from '@/ui-component/dialog/AboutDialog' +import Transitions from '@/ui-component/extended/Transitions' // assets -import { IconLogout, IconSettings, IconInfoCircle } from '@tabler/icons-react' - +import { IconFileExport, IconFileUpload, IconInfoCircle, IconLogout, IconSettings, IconX } from '@tabler/icons-react' import './index.css' +//API +import chatFlowsApi from '@/api/chatflows' + +// Hooks +import useApi from '@/hooks/useApi' +import { useLocation, useNavigate } from 'react-router-dom' + // ==============================|| PROFILE MENU ||============================== // const ProfileSection = ({ username, handleLogout }) => { @@ -43,6 +52,17 @@ const ProfileSection = ({ username, handleLogout }) => { const [aboutDialogOpen, setAboutDialogOpen] = useState(false) const anchorRef = useRef(null) + const inputRef = useRef() + + const navigate = useNavigate() + const location = useLocation() + + // ==============================|| Snackbar ||============================== // + + useNotifier() + const dispatch = useDispatch() + const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) + const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args)) const handleClose = (event) => { if (anchorRef.current && anchorRef.current.contains(event.target)) { @@ -55,6 +75,106 @@ const ProfileSection = ({ username, handleLogout }) => { setOpen((prevOpen) => !prevOpen) } + const errorFailed = (message) => { + enqueueSnackbar({ + message: message, + options: { + key: new Date().getTime() + Math.random(), + variant: 'error', + persist: true, + action: (key) => ( + + ) + } + }) + } + const importChatflowsApi = useApi(chatFlowsApi.importChatflows) + const fileChange = (e) => { + if (!e.target.files) return + + const file = e.target.files[0] + + const reader = new FileReader() + reader.onload = (evt) => { + if (!evt?.target?.result) { + return + } + const chatflows = JSON.parse(evt.target.result) + importChatflowsApi.request(chatflows) + } + reader.readAsText(file) + } + + const importChatflowsSuccess = () => { + dispatch({ type: REMOVE_DIRTY }) + enqueueSnackbar({ + message: `Import chatflows successful`, + options: { + key: new Date().getTime() + Math.random(), + variant: 'success', + action: (key) => ( + + ) + } + }) + } + useEffect(() => { + if (importChatflowsApi.error) errorFailed(`Failed to import chatflows: ${importChatflowsApi.error.response.data.message}`) + if (importChatflowsApi.data) { + importChatflowsSuccess() + // if current location is /chatflows, refresh the page + if (location.pathname === '/chatflows') navigate(0) + else { + // if not redirect to /chatflows + dispatch({ type: MENU_OPEN, id: 'chatflows' }) + navigate('/chatflows') + } + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [importChatflowsApi.error, importChatflowsApi.data]) + const importAllChatflows = () => { + inputRef.current.click() + } + const getAllChatflowsApi = useApi(chatFlowsApi.getAllChatflows) + + const exportChatflowsSuccess = () => { + dispatch({ type: REMOVE_DIRTY }) + enqueueSnackbar({ + message: `Export chatflows successful`, + options: { + key: new Date().getTime() + Math.random(), + variant: 'success', + action: (key) => ( + + ) + } + }) + } + + useEffect(() => { + if (getAllChatflowsApi.error) errorFailed(`Failed to export Chatflows: ${getAllChatflowsApi.error.response.data.message}`) + if (getAllChatflowsApi.data) { + const sanitizedChatflows = sanitizeChatflows(getAllChatflowsApi.data) + const dataStr = JSON.stringify({ Chatflows: sanitizedChatflows }, null, 2) + const dataUri = 'data:application/json;charset=utf-8,' + encodeURIComponent(dataStr) + + const exportFileDefaultName = 'AllChatflows.json' + + const linkElement = document.createElement('a') + linkElement.setAttribute('href', dataUri) + linkElement.setAttribute('download', exportFileDefaultName) + linkElement.click() + exportChatflowsSuccess() + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getAllChatflowsApi.error, getAllChatflowsApi.data]) + const prevOpen = useRef(open) useEffect(() => { if (prevOpen.current === true && open === false) { @@ -135,6 +255,29 @@ const ProfileSection = ({ username, handleLogout }) => { } }} > + { + getAllChatflowsApi.request() + }} + > + + + + Export Chatflows} /> + + { + importAllChatflows() + }} + > + + + + Import Chatflows} /> + + { diff --git a/packages/ui/src/utils/genericHelper.js b/packages/ui/src/utils/genericHelper.js index 1d957fa38..85ba40d38 100644 --- a/packages/ui/src/utils/genericHelper.js +++ b/packages/ui/src/utils/genericHelper.js @@ -340,6 +340,18 @@ export const getFolderName = (base64ArrayStr) => { } } +export const sanitizeChatflows = (arrayChatflows) => { + const sanitizedChatflows = arrayChatflows.map((chatFlow) => { + const sanitizeFlowData = generateExportFlowData(JSON.parse(chatFlow.flowData)) + return { + id: chatFlow.id, + name: chatFlow.name, + flowData: JSON.stringify(sanitizeFlowData, null, 2) + } + }) + return sanitizedChatflows +} + export const generateExportFlowData = (flowData) => { const nodes = flowData.nodes const edges = flowData.edges