From 6ab1ff1062d8b45cb1e9c39ec96925d9d9cf06eb Mon Sep 17 00:00:00 2001 From: Henry Date: Sun, 14 May 2023 20:33:43 +0100 Subject: [PATCH] add functionality to export and load database --- packages/server/src/Interface.ts | 6 + packages/server/src/index.ts | 66 ++++++++++- packages/server/src/utils/index.ts | 13 +++ packages/ui/src/api/database.js | 9 ++ .../MainLayout/Header/ProfileSection/index.js | 110 ++++++++++++++++-- .../ui/src/layout/MainLayout/Header/index.js | 8 +- .../ui-component/loading/BackdropLoader.js | 16 +++ 7 files changed, 204 insertions(+), 24 deletions(-) create mode 100644 packages/ui/src/api/database.js create mode 100644 packages/ui/src/ui-component/loading/BackdropLoader.js diff --git a/packages/server/src/Interface.ts b/packages/server/src/Interface.ts index bba3ac8ac..db0116e71 100644 --- a/packages/server/src/Interface.ts +++ b/packages/server/src/Interface.ts @@ -132,3 +132,9 @@ export interface IOverrideConfig { name: string type: string } + +export interface IDatabaseExport { + chatmessages: IChatMessage[] + chatflows: IChatFlow[] + apikeys: ICommonObject[] +} diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 57601a16e..f1990630c 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -6,7 +6,7 @@ import http from 'http' import * as fs from 'fs' import basicAuth from 'express-basic-auth' -import { IChatFlow, IncomingInput, IReactFlowNode, IReactFlowObject, INodeData } from './Interface' +import { IChatFlow, IncomingInput, IReactFlowNode, IReactFlowObject, INodeData, IDatabaseExport } from './Interface' import { getNodeModulesPackagePath, getStartingNodes, @@ -22,7 +22,8 @@ import { compareKeys, mapMimeTypeToInputField, findAvailableConfigs, - isSameOverrideConfig + isSameOverrideConfig, + replaceAllAPIKeys } from './utils' import { cloneDeep } from 'lodash' import { getDataSource } from './DataSource' @@ -76,10 +77,12 @@ export class App { const basicAuthMiddleware = basicAuth({ users: { [username]: password } }) - const whitelistURLs = ['static', 'favicon', '/api/v1/prediction/', '/api/v1/node-icon/'] - this.app.use((req, res, next) => - whitelistURLs.some((url) => req.url.includes(url)) || req.url === '/' ? next() : basicAuthMiddleware(req, res, next) - ) + const whitelistURLs = ['/api/v1/prediction/', '/api/v1/node-icon/'] + this.app.use((req, res, next) => { + if (req.url.includes('/api/v1/')) { + whitelistURLs.some((url) => req.url.includes(url)) ? next() : basicAuthMiddleware(req, res, next) + } else next() + }) } const upload = multer({ dest: `${path.join(__dirname, '..', 'uploads')}/` }) @@ -233,6 +236,57 @@ export class App { return res.json(availableConfigs) }) + // ---------------------------------------- + // Export Load Chatflow & ChatMessage & Apikeys + // ---------------------------------------- + + this.app.get('/api/v1/database/export', async (req: Request, res: Response) => { + const chatmessages = await this.AppDataSource.getRepository(ChatMessage).find() + const chatflows = await this.AppDataSource.getRepository(ChatFlow).find() + const apikeys = await getAPIKeys() + const result: IDatabaseExport = { + chatmessages, + chatflows, + apikeys + } + return res.json(result) + }) + + this.app.post('/api/v1/database/load', async (req: Request, res: Response) => { + const databaseItems: IDatabaseExport = req.body + + await this.AppDataSource.getRepository(ChatFlow).delete({}) + await this.AppDataSource.getRepository(ChatMessage).delete({}) + + let error = '' + + // Get a new query runner instance + const queryRunner = this.AppDataSource.createQueryRunner() + + // Start a new transaction + await queryRunner.startTransaction() + + try { + const chatflows: ChatFlow[] = databaseItems.chatflows + const chatmessages: ChatMessage[] = databaseItems.chatmessages + + await queryRunner.manager.insert(ChatFlow, chatflows) + await queryRunner.manager.insert(ChatMessage, chatmessages) + + await queryRunner.commitTransaction() + } catch (err: any) { + error = err?.message ?? 'Error loading database' + await queryRunner.rollbackTransaction() + } finally { + await queryRunner.release() + } + + await replaceAllAPIKeys(databaseItems.apikeys) + + if (error) return res.status(500).send(error) + return res.status(201).send('OK') + }) + // ---------------------------------------- // Prediction // ---------------------------------------- diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 5f5a8d14c..787612844 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -539,6 +539,19 @@ export const deleteAPIKey = async (keyIdToDelete: string): Promise} + */ +export const replaceAllAPIKeys = async (content: ICommonObject[]): Promise => { + try { + await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8') + } catch (error) { + console.error(error) + } +} + /** * Map MimeType to InputField * @param {string} mimeType diff --git a/packages/ui/src/api/database.js b/packages/ui/src/api/database.js new file mode 100644 index 000000000..f36fb72c7 --- /dev/null +++ b/packages/ui/src/api/database.js @@ -0,0 +1,9 @@ +import client from './client' + +const getExportDatabase = () => client.get('/database/export') +const createLoadDatabase = (body) => client.post('/database/load', body) + +export default { + getExportDatabase, + createLoadDatabase +} diff --git a/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.js b/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.js index c0fa88078..f6f6a7307 100644 --- a/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.js +++ b/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.js @@ -1,8 +1,7 @@ import { useState, useRef, useEffect } from 'react' - import PropTypes from 'prop-types' - -import { useSelector } from 'react-redux' +import { useSelector, useDispatch } from 'react-redux' +import { useNavigate } from 'react-router-dom' // material-ui import { useTheme } from '@mui/material/styles' @@ -27,9 +26,15 @@ import PerfectScrollbar from 'react-perfect-scrollbar' // project imports import MainCard from 'ui-component/cards/MainCard' import Transitions from 'ui-component/extended/Transitions' +import { BackdropLoader } from 'ui-component/loading/BackdropLoader' // assets -import { IconLogout, IconSettings } from '@tabler/icons' +import { IconLogout, IconSettings, IconFileExport, IconFileDownload } from '@tabler/icons' + +// API +import databaseApi from 'api/database' + +import { SET_MENU } from 'store/actions' import './index.css' @@ -37,12 +42,16 @@ import './index.css' const ProfileSection = ({ username, handleLogout }) => { const theme = useTheme() + const dispatch = useDispatch() + const navigate = useNavigate() const customization = useSelector((state) => state.customization) const [open, setOpen] = useState(false) + const [loading, setLoading] = useState(false) const anchorRef = useRef(null) + const uploadRef = useRef(null) const handleClose = (event) => { if (anchorRef.current && anchorRef.current.contains(event.target)) { @@ -55,6 +64,56 @@ const ProfileSection = ({ username, handleLogout }) => { setOpen((prevOpen) => !prevOpen) } + const handleExportDB = async () => { + setOpen(false) + try { + const response = await databaseApi.getExportDatabase() + const exportItems = response.data + let dataStr = JSON.stringify(exportItems) + let dataUri = 'data:application/json;charset=utf-8,' + encodeURIComponent(dataStr) + + let exportFileDefaultName = `DB.json` + + let linkElement = document.createElement('a') + linkElement.setAttribute('href', dataUri) + linkElement.setAttribute('download', exportFileDefaultName) + linkElement.click() + } catch (e) { + console.error(e) + } + } + + const handleFileUpload = (e) => { + if (!e.target.files) return + + const file = e.target.files[0] + const reader = new FileReader() + reader.onload = async (evt) => { + if (!evt?.target?.result) { + return + } + const { result } = evt.target + + if (result.includes(`"chatmessages":[`) && result.includes(`"chatflows":[`) && result.includes(`"apikeys":[`)) { + dispatch({ type: SET_MENU, opened: false }) + setLoading(true) + + try { + await databaseApi.createLoadDatabase(JSON.parse(result)) + setLoading(false) + navigate('/', { replace: true }) + navigate(0) + } catch (e) { + console.error(e) + setLoading(false) + } + } else { + alert('Incorrect Flowise Database Format') + } + } + reader.readAsText(file) + } + const prevOpen = useRef(open) useEffect(() => { if (prevOpen.current === true && open === false) { @@ -109,11 +168,13 @@ const ProfileSection = ({ username, handleLogout }) => { - - - {username} - - + {username && ( + + + {username} + + + )} @@ -135,13 +196,36 @@ const ProfileSection = ({ username, handleLogout }) => { > { + setOpen(false) + uploadRef.current.click() + }} > - + - Logout} /> + Load Database} /> + + + + + Export Database} /> + + {localStorage.getItem('username') && localStorage.getItem('password') && ( + + + + + Logout} /> + + )} @@ -151,6 +235,8 @@ const ProfileSection = ({ username, handleLogout }) => { )} + handleFileUpload(e)} /> + ) } diff --git a/packages/ui/src/layout/MainLayout/Header/index.js b/packages/ui/src/layout/MainLayout/Header/index.js index 033eb3a6a..9630bf957 100644 --- a/packages/ui/src/layout/MainLayout/Header/index.js +++ b/packages/ui/src/layout/MainLayout/Header/index.js @@ -127,12 +127,8 @@ const Header = ({ handleLeftDrawerToggle }) => { - {localStorage.getItem('username') && localStorage.getItem('password') && ( - <> - - - - )} + + ) } diff --git a/packages/ui/src/ui-component/loading/BackdropLoader.js b/packages/ui/src/ui-component/loading/BackdropLoader.js new file mode 100644 index 000000000..d88f618f6 --- /dev/null +++ b/packages/ui/src/ui-component/loading/BackdropLoader.js @@ -0,0 +1,16 @@ +import PropTypes from 'prop-types' +import { Backdrop, CircularProgress } from '@mui/material' + +export const BackdropLoader = ({ open }) => { + return ( +
+ theme.zIndex.drawer + 1 }} open={open}> + + +
+ ) +} + +BackdropLoader.propTypes = { + open: PropTypes.bool +}