From a9e269b52c8578301a1f098f1d175e0148551cc4 Mon Sep 17 00:00:00 2001 From: Matthias Platzer Date: Mon, 3 Jul 2023 17:58:41 +0200 Subject: [PATCH] Added winston logging - use logger.xxx instead of console.xxx - added express middleware logging (using jsonl) - added LOG_PATH as environment variable - more configs postponed for later iteration --- .gitignore | 1 + docker/.env.example | 1 + packages/server/.env.example | 1 + packages/server/package.json | 3 +- packages/server/src/commands/start.ts | 13 ++-- packages/server/src/index.ts | 15 ++-- packages/server/src/utils/config.ts | 25 +++++++ packages/server/src/utils/index.ts | 5 +- packages/server/src/utils/logger.ts | 100 ++++++++++++++++++++++++++ 9 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 packages/server/src/utils/config.ts create mode 100644 packages/server/src/utils/logger.ts diff --git a/.gitignore b/.gitignore index 9f5ef2e56..3ae877768 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ **/yarn.lock ## logs +logs/**/* **/*.log ## build diff --git a/docker/.env.example b/docker/.env.example index 80fbc3be5..8e66d25eb 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -4,4 +4,5 @@ PORT=3000 # DEBUG=true # DATABASE_PATH=/your_database_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise +# LOG_PATH=/your_api_key_path/logs # EXECUTION_MODE=child or main \ No newline at end of file diff --git a/packages/server/.env.example b/packages/server/.env.example index 80fbc3be5..f1fbf9905 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -4,4 +4,5 @@ PORT=3000 # DEBUG=true # DATABASE_PATH=/your_database_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise +# LOG_PATH=./logs # EXECUTION_MODE=child or main \ No newline at end of file diff --git a/packages/server/package.json b/packages/server/package.json index eda693224..05abd6c93 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -58,7 +58,8 @@ "reflect-metadata": "^0.1.13", "socket.io": "^4.6.1", "sqlite3": "^5.1.6", - "typeorm": "^0.3.6" + "typeorm": "^0.3.6", + "winston": "^3.9.0" }, "devDependencies": { "@types/cors": "^2.8.12", diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts index d3efc1eb6..9bd1d64b3 100644 --- a/packages/server/src/commands/start.ts +++ b/packages/server/src/commands/start.ts @@ -3,6 +3,7 @@ import path from 'path' import * as Server from '../index' import * as DataSource from '../DataSource' import dotenv from 'dotenv' +import logger from '../utils/logger' dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true }) @@ -25,11 +26,11 @@ export default class Start extends Command { } async stopProcess() { - console.info('Shutting down Flowise...') + logger.info('Shutting down Flowise...') try { // Shut down the app after timeout if it ever stuck removing pools setTimeout(() => { - console.info('Flowise was forced to shut down after 30 secs') + logger.info('Flowise was forced to shut down after 30 secs') process.exit(processExitCode) }, 30000) @@ -37,7 +38,7 @@ export default class Start extends Command { const serverApp = Server.getInstance() if (serverApp) await serverApp.stopApp() } catch (error) { - console.error('There was an error shutting down Flowise...', error) + logger.error('There was an error shutting down Flowise...', error) } process.exit(processExitCode) } @@ -49,7 +50,7 @@ export default class Start extends Command { // Prevent throw new Error from crashing the app // TODO: Get rid of this and send proper error message to ui process.on('uncaughtException', (err) => { - console.error('uncaughtException: ', err) + logger.error('uncaughtException: ', err) }) const { flags } = await this.parse(Start) @@ -63,11 +64,11 @@ export default class Start extends Command { await (async () => { try { - this.log('Starting Flowise...') + logger.info('Starting Flowise...') await DataSource.init() await Server.start() } catch (error) { - console.error('There was an error starting Flowise...', error) + logger.error('There was an error starting Flowise...', error) processExitCode = EXIT_CODE.FAILED // @ts-ignore process.emit('SIGINT') diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 15762a23c..73dbada4f 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -6,6 +6,8 @@ import http from 'http' import * as fs from 'fs' import basicAuth from 'express-basic-auth' import { Server } from 'socket.io' +import logger from './utils/logger' +import { expressRequestLogger } from './utils/logger' import { IChatFlow, @@ -57,13 +59,16 @@ export class App { constructor() { this.app = express() + + // Add the expressRequestLogger middleware to log all requests + this.app.use(expressRequestLogger) } async initDatabase() { // Initialize database this.AppDataSource.initialize() .then(async () => { - console.info('📦[server]: Data Source has been initialized!') + logger.info('📦 [server]: Data Source has been initialized!') // Initialize pools this.nodesPool = new NodesPool() @@ -75,7 +80,7 @@ export class App { await getAPIKeys() }) .catch((err) => { - console.error('❌[server]: Error during Data Source initialization:', err) + logger.error('❌ [server]: Error during Data Source initialization:', err) }) } @@ -614,7 +619,7 @@ export class App { }) }) } catch (err) { - console.error(err) + logger.error(err) } } @@ -792,7 +797,7 @@ export class App { const removePromises: any[] = [] await Promise.all(removePromises) } catch (e) { - console.error(`❌[server]: Flowise Server shut down error: ${e}`) + logger.error(`❌[server]: Flowise Server shut down error: ${e}`) } } } @@ -832,7 +837,7 @@ export async function start(): Promise { await serverApp.config(io) server.listen(port, () => { - console.info(`⚡️[server]: Flowise Server is listening at ${port}`) + logger.info(`⚡️ [server]: Flowise Server is listening at ${port}`) }) } diff --git a/packages/server/src/utils/config.ts b/packages/server/src/utils/config.ts new file mode 100644 index 000000000..a4a339379 --- /dev/null +++ b/packages/server/src/utils/config.ts @@ -0,0 +1,25 @@ +// BEWARE: This file is an intereem solution until we have a proper config strategy + +import path from 'path' +import dotenv from 'dotenv' + +dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true }) + +// default config +const loggingConfig = { + dir: process.env.LOG_PATH ?? './logs', + server: { + level: 'info', + filename: 'server.log', + errorFilename: 'server-error.log' + }, + express: { + level: 'info', + format: 'jsonl', // can't be changed currently + filename: 'server-requests.log.jsonl' // should end with .jsonl + } +} + +export default { + logging: loggingConfig +} diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 005f4a4bb..55529afb0 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -1,6 +1,7 @@ import path from 'path' import fs from 'fs' import moment from 'moment' +import logger from './logger' import { IComponentNodes, IDepthQueue, @@ -227,7 +228,7 @@ export const buildLangchain = async ( databaseEntities }) } catch (e: any) { - console.error(e) + logger.error(e) throw new Error(e) } @@ -595,7 +596,7 @@ export const replaceAllAPIKeys = async (content: ICommonObject[]): Promise try { await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8') } catch (error) { - console.error(error) + logger.error(error) } } diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts new file mode 100644 index 000000000..277cf3ad5 --- /dev/null +++ b/packages/server/src/utils/logger.ts @@ -0,0 +1,100 @@ +import * as path from 'path' +import * as fs from 'fs' +import config from './config' // should be replaced by node-config or similar +import { createLogger, transports, format } from 'winston' +import { NextFunction, Request, Response } from 'express' + +const { combine, timestamp, printf } = format + +// expect the log dir be relative to the projects root +const logDir = path.join(__dirname, '../../../..', config.logging.dir ?? './logs') + +// Create the log directory if it doesn't exist +if (!fs.existsSync(logDir)) { + fs.mkdirSync(logDir) +} + +const logger = createLogger({ + format: combine( + timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + format.json(), + printf(({ level, message, timestamp }) => { + return `${timestamp} [${level.toUpperCase()}]: ${message}` + }) + ), + defaultMeta: { + package: 'server' + }, + transports: [ + new transports.Console(), + new transports.File({ + filename: path.join(logDir, config.logging.server.filename ?? 'server.log'), + level: config.logging.server.level ?? 'info' + }), + new transports.File({ + filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'), + level: 'error' // Log only errors to this file + }) + ], + exceptionHandlers: [ + new transports.File({ + filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log') + }) + ], + rejectionHandlers: [ + new transports.File({ + filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log') + }) + ] +}) + +/** + * This function is used by express as a middleware. + * @example + * this.app = express() + * this.app.use(expressRequestLogger) + */ +export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void { + const fileLogger = createLogger({ + format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json()), + defaultMeta: { + package: 'server', + request: { + method: req.method, + url: req.url, + body: req.body, + query: req.query, + params: req.params, + headers: req.headers + } + }, + transports: [ + new transports.File({ + filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'), + level: 'debug' + }) + ] + }) + + const getRequestEmoji = (method: string) => { + const requetsEmojis: Record = { + GET: '⬇️', + POST: '⬆️', + PUT: '🖊', + DELETE: '❌' + } + + return requetsEmojis[method] || '?' + } + + if (req.method !== 'GET') { + fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + } else { + fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + } + + next() +} + +export default logger