Merge pull request #469 from matthias/feature/winston-logging-clean

Added winston logging

.gitignore

@@ -8,6 +8,7 @@
 **/yarn.lock
 
 ## logs
+**/logs
 **/*.log
 
 ## build
@@ -42,4 +43,4 @@
 **/uploads
 
 ## compressed
-**/*.tgz
+**/*.tgz

docker/.env.example

@@ -4,4 +4,5 @@ PORT=3000
 # DEBUG=true
 # DATABASE_PATH=/your_database_path/.flowise
 # APIKEY_PATH=/your_api_key_path/.flowise
-# EXECUTION_MODE=child or main
+# LOG_PATH=/your_log_path/logs
+# EXECUTION_MODE=child or main

docker/docker-compose.yml

@@ -10,6 +10,7 @@ services:
             - FLOWISE_PASSWORD=${FLOWISE_PASSWORD}
             - DATABASE_PATH=${DATABASE_PATH}
             - APIKEY_PATH=${APIKEY_PATH}
+            - LOG_PATH=${LOG_PATH}
             - EXECUTION_MODE=${EXECUTION_MODE}
             - DEBUG=${DEBUG}
         ports:

packages/server/.env.example

@@ -4,4 +4,5 @@ PORT=3000
 # DEBUG=true
 # DATABASE_PATH=/your_database_path/.flowise
 # APIKEY_PATH=/your_api_key_path/.flowise
-# EXECUTION_MODE=child or main
+# LOG_PATH=/your_log_path/logs
+# EXECUTION_MODE=child or main

packages/server/package.json

@@ -58,7 +58,8 @@
         "reflect-metadata": "^0.1.13",
         "socket.io": "^4.6.1",
         "sqlite3": "^5.1.6",
-        "typeorm": "^0.3.6"
+        "typeorm": "^0.3.6",
+        "winston": "^3.9.0"
     },
     "devDependencies": {
         "@types/cors": "^2.8.12",

packages/server/src/commands/start.ts

@@ -3,6 +3,7 @@ import path from 'path'
 import * as Server from '../index'
 import * as DataSource from '../DataSource'
 import dotenv from 'dotenv'
+import logger from '../utils/logger'
 
 dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true })
 
@@ -21,15 +22,16 @@ export default class Start extends Command {
         DEBUG: Flags.string(),
         DATABASE_PATH: Flags.string(),
         APIKEY_PATH: Flags.string(),
+        LOG_PATH: Flags.string(),
         EXECUTION_MODE: Flags.string()
     }
 
     async stopProcess() {
-        console.info('Shutting down Flowise...')
+        logger.info('Shutting down Flowise...')
         try {
             // Shut down the app after timeout if it ever stuck removing pools
             setTimeout(() => {
-                console.info('Flowise was forced to shut down after 30 secs')
+                logger.info('Flowise was forced to shut down after 30 secs')
                 process.exit(processExitCode)
             }, 30000)
 
@@ -37,7 +39,7 @@ export default class Start extends Command {
             const serverApp = Server.getInstance()
             if (serverApp) await serverApp.stopApp()
         } catch (error) {
-            console.error('There was an error shutting down Flowise...', error)
+            logger.error('There was an error shutting down Flowise...', error)
         }
         process.exit(processExitCode)
     }
@@ -49,7 +51,7 @@ export default class Start extends Command {
         // Prevent throw new Error from crashing the app
         // TODO: Get rid of this and send proper error message to ui
         process.on('uncaughtException', (err) => {
-            console.error('uncaughtException: ', err)
+            logger.error('uncaughtException: ', err)
         })
 
         const { flags } = await this.parse(Start)
@@ -58,16 +60,17 @@ export default class Start extends Command {
         if (flags.PORT) process.env.PORT = flags.PORT
         if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH
         if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH
+        if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH
         if (flags.EXECUTION_MODE) process.env.EXECUTION_MODE = flags.EXECUTION_MODE
         if (flags.DEBUG) process.env.DEBUG = flags.DEBUG
 
         await (async () => {
             try {
-                this.log('Starting Flowise...')
+                logger.info('Starting Flowise...')
                 await DataSource.init()
                 await Server.start()
             } catch (error) {
-                console.error('There was an error starting Flowise...', error)
+                logger.error('There was an error starting Flowise...', error)
                 processExitCode = EXIT_CODE.FAILED
                 // @ts-ignore
                 process.emit('SIGINT')

packages/server/src/index.ts

@@ -6,6 +6,8 @@ import http from 'http'
 import * as fs from 'fs'
 import basicAuth from 'express-basic-auth'
 import { Server } from 'socket.io'
+import logger from './utils/logger'
+import { expressRequestLogger } from './utils/logger'
 
 import {
     IChatFlow,
@@ -57,13 +59,16 @@ export class App {
 
     constructor() {
         this.app = express()
+
+        // Add the expressRequestLogger middleware to log all requests
+        this.app.use(expressRequestLogger)
     }
 
     async initDatabase() {
         // Initialize database
         this.AppDataSource.initialize()
             .then(async () => {
-                console.info('📦[server]: Data Source has been initialized!')
+                logger.info('📦 [server]: Data Source has been initialized!')
 
                 // Initialize pools
                 this.nodesPool = new NodesPool()
@@ -75,7 +80,7 @@ export class App {
                 await getAPIKeys()
             })
             .catch((err) => {
-                console.error('❌[server]: Error during Data Source initialization:', err)
+                logger.error('❌ [server]: Error during Data Source initialization:', err)
             })
     }
 
@@ -633,7 +638,7 @@ export class App {
                 })
             })
         } catch (err) {
-            console.error(err)
+            logger.error(err)
         }
     }
 
@@ -811,7 +816,7 @@ export class App {
             const removePromises: any[] = []
             await Promise.all(removePromises)
         } catch (e) {
-            console.error(`❌[server]: Flowise Server shut down error: ${e}`)
+            logger.error(`❌[server]: Flowise Server shut down error: ${e}`)
         }
     }
 }
@@ -851,7 +856,7 @@ export async function start(): Promise<void> {
     await serverApp.config(io)
 
     server.listen(port, () => {
-        console.info(`⚡️[server]: Flowise Server is listening at ${port}`)
+        logger.info(`⚡️ [server]: Flowise Server is listening at ${port}`)
     })
 }
 

packages/server/src/utils/config.ts

@@ -0,0 +1,25 @@
+// BEWARE: This file is an intereem solution until we have a proper config strategy
+
+import path from 'path'
+import dotenv from 'dotenv'
+
+dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true })
+
+// default config
+const loggingConfig = {
+    dir: process.env.LOG_PATH ?? path.join(__dirname, '..', '..', '..', '..', 'logs'),
+    server: {
+        level: 'info',
+        filename: 'server.log',
+        errorFilename: 'server-error.log'
+    },
+    express: {
+        level: 'info',
+        format: 'jsonl', // can't be changed currently
+        filename: 'server-requests.log.jsonl' // should end with .jsonl
+    }
+}
+
+export default {
+    logging: loggingConfig
+}

packages/server/src/utils/index.ts

@@ -1,6 +1,7 @@
 import path from 'path'
 import fs from 'fs'
 import moment from 'moment'
+import logger from './logger'
 import {
     IComponentNodes,
     IDepthQueue,
@@ -227,7 +228,7 @@ export const buildLangchain = async (
                 databaseEntities
             })
         } catch (e: any) {
-            console.error(e)
+            logger.error(e)
             throw new Error(e)
         }
 
@@ -595,7 +596,7 @@ export const replaceAllAPIKeys = async (content: ICommonObject[]): Promise<void>
     try {
         await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
     } catch (error) {
-        console.error(error)
+        logger.error(error)
     }
 }
 

packages/server/src/utils/logger.ts

@@ -0,0 +1,100 @@
+import * as path from 'path'
+import * as fs from 'fs'
+import config from './config' // should be replaced by node-config or similar
+import { createLogger, transports, format } from 'winston'
+import { NextFunction, Request, Response } from 'express'
+
+const { combine, timestamp, printf } = format
+
+// expect the log dir be relative to the projects root
+const logDir = config.logging.dir
+
+// Create the log directory if it doesn't exist
+if (!fs.existsSync(logDir)) {
+    fs.mkdirSync(logDir)
+}
+
+const logger = createLogger({
+    format: combine(
+        timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+        format.json(),
+        printf(({ level, message, timestamp }) => {
+            return `${timestamp} [${level.toUpperCase()}]: ${message}`
+        })
+    ),
+    defaultMeta: {
+        package: 'server'
+    },
+    transports: [
+        new transports.Console(),
+        new transports.File({
+            filename: path.join(logDir, config.logging.server.filename ?? 'server.log'),
+            level: config.logging.server.level ?? 'info'
+        }),
+        new transports.File({
+            filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'),
+            level: 'error' // Log only errors to this file
+        })
+    ],
+    exceptionHandlers: [
+        new transports.File({
+            filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
+        })
+    ],
+    rejectionHandlers: [
+        new transports.File({
+            filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
+        })
+    ]
+})
+
+/**
+ * This function is used by express as a middleware.
+ * @example
+ *   this.app = express()
+ *   this.app.use(expressRequestLogger)
+ */
+export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void {
+    const fileLogger = createLogger({
+        format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json()),
+        defaultMeta: {
+            package: 'server',
+            request: {
+                method: req.method,
+                url: req.url,
+                body: req.body,
+                query: req.query,
+                params: req.params,
+                headers: req.headers
+            }
+        },
+        transports: [
+            new transports.File({
+                filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
+                level: 'debug'
+            })
+        ]
+    })
+
+    const getRequestEmoji = (method: string) => {
+        const requetsEmojis: Record<string, string> = {
+            GET: '⬇️',
+            POST: '⬆️',
+            PUT: '🖊',
+            DELETE: '❌'
+        }
+
+        return requetsEmojis[method] || '?'
+    }
+
+    if (req.method !== 'GET') {
+        fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
+        logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
+    } else {
+        fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
+    }
+
+    next()
+}
+
+export default logger