Fix: Add keep-alive options to Redis clients to prevent idle timeouts and socket closing. (#4377)

* redis keepalive mechanism for all redis

* removed offline queue commands

* Simplified changes for consistency. Added REDIS_KEEP_ALIVE env variable.

* update redis socket alive env variable

* lint fix

---------

Co-authored-by: Henry <hzj94@hotmail.com>
Co-authored-by: Henry Heng <henryheng@flowiseai.com>

docker/.env.example

@@ -101,4 +101,5 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
 # REDIS_CERT=
 # REDIS_KEY=
 # REDIS_CA=
-# ENABLE_BULLMQ_DASHBOARD=
+# REDIS_KEEP_ALIVE=
+# ENABLE_BULLMQ_DASHBOARD=

docker/docker-compose.yml

@@ -49,6 +49,7 @@ services:
             - REDIS_CERT=${REDIS_CERT}
             - REDIS_KEY=${REDIS_KEY}
             - REDIS_CA=${REDIS_CA}
+            - REDIS_KEEP_ALIVE=${REDIS_KEEP_ALIVE}
             - ENABLE_BULLMQ_DASHBOARD=${ENABLE_BULLMQ_DASHBOARD}
         ports:
             - '${PORT}:${PORT}'

docker/worker/docker-compose.yml

@@ -49,6 +49,7 @@ services:
             - REDIS_CERT=${REDIS_CERT}
             - REDIS_KEY=${REDIS_KEY}
             - REDIS_CA=${REDIS_CA}
+            - REDIS_KEEP_ALIVE=${REDIS_KEEP_ALIVE}
             - ENABLE_BULLMQ_DASHBOARD=${ENABLE_BULLMQ_DASHBOARD}
         ports:
             - '${PORT}:${PORT}'

packages/components/nodes/cache/RedisCache/RedisCache.ts

@@ -126,10 +126,19 @@ const getRedisClient = async (nodeData: INodeData, options: ICommonObject) => {
             host,
             username,
             password,
+            keepAlive:
+                process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                    ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                    : undefined,
             ...tlsOptions
         })
     } else {
-        client = new Redis(redisUrl)
+        client = new Redis(redisUrl, {
+            keepAlive:
+                process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                    ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                    : undefined
+        })
     }
 
     return client

packages/components/nodes/cache/RedisCache/RedisEmbeddingsCache.ts

@@ -83,10 +83,19 @@ class RedisEmbeddingsCache implements INode {
                 host,
                 username,
                 password,
+                keepAlive:
+                    process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                        ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                        : undefined,
                 ...tlsOptions
             })
         } else {
-            client = new Redis(redisUrl)
+            client = new Redis(redisUrl, {
+                keepAlive:
+                    process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                        ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                        : undefined
+            })
         }
 
         ttl ??= '3600'

packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts

@@ -132,7 +132,21 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
     }
 
     private async withRedisClient<T>(fn: (client: Redis) => Promise<T>): Promise<T> {
-        const client = typeof this.redisOptions === 'string' ? new Redis(this.redisOptions) : new Redis(this.redisOptions)
+        const client =
+            typeof this.redisOptions === 'string'
+                ? new Redis(this.redisOptions, {
+                      keepAlive:
+                          process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                              ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                              : undefined
+                  })
+                : new Redis({
+                      ...this.redisOptions,
+                      keepAlive:
+                          process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                              ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                              : undefined
+                  })
         try {
             return await fn(client)
         } finally {

packages/components/nodes/vectorstores/Redis/Redis.ts

@@ -147,7 +147,15 @@ class Redis_VectorStores implements INode {
             }
 
             try {
-                const redisClient = createClient({ url: redisUrl })
+                const redisClient = createClient({
+                    url: redisUrl,
+                    socket: {
+                        keepAlive:
+                            process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                                ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                                : undefined // milliseconds
+                    }
+                })
                 await redisClient.connect()
 
                 const storeConfig: RedisVectorStoreConfig = {
@@ -212,7 +220,15 @@ class Redis_VectorStores implements INode {
             redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr
         }
 
-        const redisClient = createClient({ url: redisUrl })
+        const redisClient = createClient({
+            url: redisUrl,
+            socket: {
+                keepAlive:
+                    process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                        ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                        : undefined // milliseconds
+            }
+        })
 
         const storeConfig: RedisVectorStoreConfig = {
             redisClient: redisClient,

packages/server/.env.example

@@ -99,4 +99,5 @@ PORT=3000
 # REDIS_CERT=
 # REDIS_KEY=
 # REDIS_CA=
-# ENABLE_BULLMQ_DASHBOARD=
+# REDIS_KEEP_ALIVE=
+# ENABLE_BULLMQ_DASHBOARD=

packages/server/src/CachePool.ts

@@ -12,7 +12,12 @@ export class CachePool {
     constructor() {
         if (process.env.MODE === MODE.QUEUE) {
             if (process.env.REDIS_URL) {
-                this.redisClient = new Redis(process.env.REDIS_URL)
+                this.redisClient = new Redis(process.env.REDIS_URL, {
+                    keepAlive:
+                        process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                            ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                            : undefined
+                })
             } else {
                 this.redisClient = new Redis({
                     host: process.env.REDIS_HOST || 'localhost',
@@ -26,6 +31,10 @@ export class CachePool {
                                   key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
                                   ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
                               }
+                            : undefined,
+                    keepAlive:
+                        process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                            ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
                             : undefined
                 })
             }

packages/server/src/commands/base.ts

@@ -76,6 +76,7 @@ export abstract class BaseCommand extends Command {
         REDIS_CERT: Flags.string(),
         REDIS_KEY: Flags.string(),
         REDIS_CA: Flags.string(),
+        REDIS_KEEP_ALIVE: Flags.string(),
         ENABLE_BULLMQ_DASHBOARD: Flags.string()
     }
 
@@ -211,6 +212,7 @@ export abstract class BaseCommand extends Command {
         if (flags.QUEUE_REDIS_EVENT_STREAM_MAX_LEN) process.env.QUEUE_REDIS_EVENT_STREAM_MAX_LEN = flags.QUEUE_REDIS_EVENT_STREAM_MAX_LEN
         if (flags.REMOVE_ON_AGE) process.env.REMOVE_ON_AGE = flags.REMOVE_ON_AGE
         if (flags.REMOVE_ON_COUNT) process.env.REMOVE_ON_COUNT = flags.REMOVE_ON_COUNT
+        if (flags.REDIS_KEEP_ALIVE) process.env.REDIS_KEEP_ALIVE = flags.REDIS_KEEP_ALIVE
         if (flags.ENABLE_BULLMQ_DASHBOARD) process.env.ENABLE_BULLMQ_DASHBOARD = flags.ENABLE_BULLMQ_DASHBOARD
     }
 }

packages/server/src/queue/QueueManager.ts

@@ -41,7 +41,12 @@ export class QueueManager {
             port: parseInt(process.env.REDIS_PORT || '6379'),
             username: process.env.REDIS_USERNAME || undefined,
             password: process.env.REDIS_PASSWORD || undefined,
-            tls: tlsOpts
+            tls: tlsOpts,
+            enableReadyCheck: true,
+            keepAlive:
+                process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                    ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                    : undefined
         }
     }
 

packages/server/src/queue/RedisEventPublisher.ts

@@ -7,7 +7,13 @@ export class RedisEventPublisher implements IServerSideEventStreamer {
     constructor() {
         if (process.env.REDIS_URL) {
             this.redisPublisher = createClient({
-                url: process.env.REDIS_URL
+                url: process.env.REDIS_URL,
+                socket: {
+                    keepAlive:
+                        process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                            ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                            : undefined
+                }
             })
         } else {
             this.redisPublisher = createClient({
@@ -19,7 +25,11 @@ export class RedisEventPublisher implements IServerSideEventStreamer {
                     tls: process.env.REDIS_TLS === 'true',
                     cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
                     key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
-                    ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+                    ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined,
+                    keepAlive:
+                        process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                            ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                            : undefined
                 }
             })
         }

packages/server/src/queue/RedisEventSubscriber.ts

@@ -9,7 +9,13 @@ export class RedisEventSubscriber {
     constructor(sseStreamer: SSEStreamer) {
         if (process.env.REDIS_URL) {
             this.redisSubscriber = createClient({
-                url: process.env.REDIS_URL
+                url: process.env.REDIS_URL,
+                socket: {
+                    keepAlive:
+                        process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                            ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                            : undefined
+                }
             })
         } else {
             this.redisSubscriber = createClient({
@@ -21,7 +27,11 @@ export class RedisEventSubscriber {
                     tls: process.env.REDIS_TLS === 'true',
                     cert: process.env.REDIS_CERT ? Buffer.from(process.env.REDIS_CERT, 'base64') : undefined,
                     key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
-                    ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
+                    ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined,
+                    keepAlive:
+                        process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                            ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                            : undefined
                 }
             })
         }

packages/server/src/utils/rateLimit.ts

@@ -24,7 +24,12 @@ export class RateLimiterManager {
     constructor() {
         if (process.env.MODE === MODE.QUEUE) {
             if (process.env.REDIS_URL) {
-                this.redisClient = new Redis(process.env.REDIS_URL)
+                this.redisClient = new Redis(process.env.REDIS_URL, {
+                    keepAlive:
+                        process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                            ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                            : undefined
+                })
             } else {
                 this.redisClient = new Redis({
                     host: process.env.REDIS_HOST || 'localhost',
@@ -38,6 +43,10 @@ export class RateLimiterManager {
                                   key: process.env.REDIS_KEY ? Buffer.from(process.env.REDIS_KEY, 'base64') : undefined,
                                   ca: process.env.REDIS_CA ? Buffer.from(process.env.REDIS_CA, 'base64') : undefined
                               }
+                            : undefined,
+                    keepAlive:
+                        process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                            ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
                             : undefined
                 })
             }
@@ -65,7 +74,13 @@ export class RateLimiterManager {
             port: parseInt(process.env.REDIS_PORT || '6379'),
             username: process.env.REDIS_USERNAME || undefined,
             password: process.env.REDIS_PASSWORD || undefined,
-            tls: tlsOpts
+            tls: tlsOpts,
+            maxRetriesPerRequest: null,
+            enableReadyCheck: true,
+            keepAlive:
+                process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10))
+                    ? parseInt(process.env.REDIS_KEEP_ALIVE, 10)
+                    : undefined
         }
     }