From 8ba1a0907721d7b52f35930ba5b8e1afa6e6c2ee Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Thu, 12 Jun 2025 17:25:43 +0100 Subject: [PATCH] Bugfix/Update worker docker (#4643) update worker docker --- .github/workflows/docker-image.yml | 27 ++- docker/README.md | 32 ++- docker/docker-compose-queue-prebuilt.yml | 296 +++++++++++++++++++++++ docker/docker-compose-queue-source.yml | 71 ++++++ docker/docker-compose.yml | 2 +- docker/worker/.env.example | 165 +++++++++++++ docker/worker/Dockerfile | 49 ++++ docker/worker/README.md | 10 +- docker/worker/docker-compose.yml | 14 +- docker/worker/healthcheck/healthcheck.js | 13 + docker/worker/healthcheck/package.json | 13 + packages/server/src/utils/logger.ts | 7 +- 12 files changed, 687 insertions(+), 12 deletions(-) create mode 100644 docker/docker-compose-queue-prebuilt.yml create mode 100644 docker/docker-compose-queue-source.yml create mode 100644 docker/worker/.env.example create mode 100644 docker/worker/Dockerfile create mode 100644 docker/worker/healthcheck/healthcheck.js create mode 100644 docker/worker/healthcheck/package.json diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 767fbd836..7faf472f5 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -19,6 +19,14 @@ on: options: - dev - prod + image_type: + description: 'Type of image to build and push.' + type: choice + required: true + default: 'main' + options: + - 'main' + - 'worker' node_version: description: 'Node.js version to build this image with.' type: choice @@ -41,6 +49,7 @@ jobs: id: defaults run: | echo "registry=${{ github.event.inputs.registry || 'aws_ecr' }}" >> $GITHUB_OUTPUT + echo "image_type=${{ github.event.inputs.image_type || 'main' }}" >> $GITHUB_OUTPUT echo "node_version=${{ github.event.inputs.node_version || '20' }}" >> $GITHUB_OUTPUT echo "tag_version=${{ github.event.inputs.tag_version || 'latest' }}" >> $GITHUB_OUTPUT @@ -82,10 +91,24 @@ jobs: uses: docker/build-push-action@v5.3.0 with: context: . - file: ${{ steps.defaults.outputs.registry == 'docker_hub' && './docker/Dockerfile' || 'Dockerfile' }} + file: | + ${{ + steps.defaults.outputs.image_type == 'worker' && 'docker/worker/Dockerfile' || + (steps.defaults.outputs.registry == 'docker_hub' && './docker/Dockerfile' || 'Dockerfile') + }} build-args: | NODE_VERSION=${{ steps.defaults.outputs.node_version }} platforms: linux/amd64,linux/arm64 push: true tags: | - ${{ steps.defaults.outputs.registry == 'docker_hub' && format('flowiseai/flowise:{0}', steps.defaults.outputs.tag_version) || format('{0}.dkr.ecr.{1}.amazonaws.com/flowise:{2}', secrets.AWS_ACCOUNT_ID, secrets.AWS_REGION, steps.defaults.outputs.tag_version) }} + ${{ + steps.defaults.outputs.registry == 'docker_hub' && + format('flowiseai/flowise{0}:{1}', + steps.defaults.outputs.image_type == 'worker' && '-worker' || '', + steps.defaults.outputs.tag_version) || + format('{0}.dkr.ecr.{1}.amazonaws.com/flowise{2}:{3}', + secrets.AWS_ACCOUNT_ID, + secrets.AWS_REGION, + steps.defaults.outputs.image_type == 'worker' && '-worker' || '', + steps.defaults.outputs.tag_version) + }} diff --git a/docker/README.md b/docker/README.md index d4dc5ae9a..29d802281 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,7 +11,7 @@ Starts Flowise from [DockerHub Image](https://hub.docker.com/r/flowiseai/flowise ## 🌱 Env Variables -If you like to persist your data (flows, logs, apikeys, credentials), set these variables in the `.env` file inside `docker` folder: +If you like to persist your data (flows, logs, credentials, storage), set these variables in the `.env` file inside `docker` folder: - DATABASE_PATH=/root/.flowise - LOG_PATH=/root/.flowise/logs @@ -19,3 +19,33 @@ If you like to persist your data (flows, logs, apikeys, credentials), set these - BLOB_STORAGE_PATH=/root/.flowise/storage Flowise also support different environment variables to configure your instance. Read [more](https://docs.flowiseai.com/environment-variables) + +## Queue Mode: + +### Building from source: + +You can build the images for worker and main from scratch with: + +``` +docker compose -f docker-compose-queue-source.yml up -d +``` + +Monitor Health: + +``` +docker compose -f docker-compose-queue-source.yml ps +``` + +### From pre-built images: + +You can also use the pre-built images: + +``` +docker compose -f docker-compose-queue-prebuilt.yml up -d +``` + +Monitor Health: + +``` +docker compose -f docker-compose-queue-prebuilt.yml ps +``` diff --git a/docker/docker-compose-queue-prebuilt.yml b/docker/docker-compose-queue-prebuilt.yml new file mode 100644 index 000000000..3777cd9d1 --- /dev/null +++ b/docker/docker-compose-queue-prebuilt.yml @@ -0,0 +1,296 @@ +version: '3.1' + +services: + redis: + image: redis:alpine + container_name: flowise-redis + ports: + - '6379:6379' + volumes: + - redis_data:/data + networks: + - flowise-net + restart: always + + flowise: + image: flowiseai/flowise:latest + container_name: flowise-main + restart: always + ports: + - '${PORT:-3000}:${PORT:-3000}' + volumes: + - ~/.flowise:/root/.flowise + environment: + # --- Essential Flowise Vars --- + - PORT=${PORT:-3000} + - DATABASE_PATH=${DATABASE_PATH:-/root/.flowise} + - DATABASE_TYPE=${DATABASE_TYPE} + - DATABASE_PORT=${DATABASE_PORT} + - DATABASE_HOST=${DATABASE_HOST} + - DATABASE_NAME=${DATABASE_NAME} + - DATABASE_USER=${DATABASE_USER} + - DATABASE_PASSWORD=${DATABASE_PASSWORD} + - DATABASE_SSL=${DATABASE_SSL} + - DATABASE_SSL_KEY_BASE64=${DATABASE_SSL_KEY_BASE64} + + # SECRET KEYS + - SECRETKEY_STORAGE_TYPE=${SECRETKEY_STORAGE_TYPE} + - SECRETKEY_PATH=${SECRETKEY_PATH} + - FLOWISE_SECRETKEY_OVERWRITE=${FLOWISE_SECRETKEY_OVERWRITE} + - SECRETKEY_AWS_ACCESS_KEY=${SECRETKEY_AWS_ACCESS_KEY} + - SECRETKEY_AWS_SECRET_KEY=${SECRETKEY_AWS_SECRET_KEY} + - SECRETKEY_AWS_REGION=${SECRETKEY_AWS_REGION} + - SECRETKEY_AWS_NAME=${SECRETKEY_AWS_NAME} + + # LOGGING + - DEBUG=${DEBUG} + - LOG_PATH=${LOG_PATH} + - LOG_LEVEL=${LOG_LEVEL} + + # CUSTOM TOOL DEPENDENCIES + - TOOL_FUNCTION_BUILTIN_DEP=${TOOL_FUNCTION_BUILTIN_DEP} + - TOOL_FUNCTION_EXTERNAL_DEP=${TOOL_FUNCTION_EXTERNAL_DEP} + + # STORAGE + - STORAGE_TYPE=${STORAGE_TYPE} + - BLOB_STORAGE_PATH=${BLOB_STORAGE_PATH} + - S3_STORAGE_BUCKET_NAME=${S3_STORAGE_BUCKET_NAME} + - S3_STORAGE_ACCESS_KEY_ID=${S3_STORAGE_ACCESS_KEY_ID} + - S3_STORAGE_SECRET_ACCESS_KEY=${S3_STORAGE_SECRET_ACCESS_KEY} + - S3_STORAGE_REGION=${S3_STORAGE_REGION} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL} + - S3_FORCE_PATH_STYLE=${S3_FORCE_PATH_STYLE} + - GOOGLE_CLOUD_STORAGE_CREDENTIAL=${GOOGLE_CLOUD_STORAGE_CREDENTIAL} + - GOOGLE_CLOUD_STORAGE_PROJ_ID=${GOOGLE_CLOUD_STORAGE_PROJ_ID} + - GOOGLE_CLOUD_STORAGE_BUCKET_NAME=${GOOGLE_CLOUD_STORAGE_BUCKET_NAME} + - GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS=${GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS} + + # SETTINGS + - NUMBER_OF_PROXIES=${NUMBER_OF_PROXIES} + - CORS_ORIGINS=${CORS_ORIGINS} + - IFRAME_ORIGINS=${IFRAME_ORIGINS} + - FLOWISE_FILE_SIZE_LIMIT=${FLOWISE_FILE_SIZE_LIMIT} + - SHOW_COMMUNITY_NODES=${SHOW_COMMUNITY_NODES} + - DISABLE_FLOWISE_TELEMETRY=${DISABLE_FLOWISE_TELEMETRY} + - DISABLED_NODES=${DISABLED_NODES} + - MODEL_LIST_CONFIG_JSON=${MODEL_LIST_CONFIG_JSON} + + # AUTH PARAMETERS + - APP_URL=${APP_URL} + - JWT_AUTH_TOKEN_SECRET=${JWT_AUTH_TOKEN_SECRET} + - JWT_REFRESH_TOKEN_SECRET=${JWT_REFRESH_TOKEN_SECRET} + - JWT_ISSUER=${JWT_ISSUER} + - JWT_AUDIENCE=${JWT_AUDIENCE} + - JWT_TOKEN_EXPIRY_IN_MINUTES=${JWT_TOKEN_EXPIRY_IN_MINUTES} + - JWT_REFRESH_TOKEN_EXPIRY_IN_MINUTES=${JWT_REFRESH_TOKEN_EXPIRY_IN_MINUTES} + - EXPIRE_AUTH_TOKENS_ON_RESTART=${EXPIRE_AUTH_TOKENS_ON_RESTART} + - EXPRESS_SESSION_SECRET=${EXPRESS_SESSION_SECRET} + - PASSWORD_RESET_TOKEN_EXPIRY_IN_MINS=${PASSWORD_RESET_TOKEN_EXPIRY_IN_MINS} + - PASSWORD_SALT_HASH_ROUNDS=${PASSWORD_SALT_HASH_ROUNDS} + - TOKEN_HASH_SECRET=${TOKEN_HASH_SECRET} + + # EMAIL + - SMTP_HOST=${SMTP_HOST} + - SMTP_PORT=${SMTP_PORT} + - SMTP_USER=${SMTP_USER} + - SMTP_PASSWORD=${SMTP_PASSWORD} + - SMTP_SECURE=${SMTP_SECURE} + - ALLOW_UNAUTHORIZED_CERTS=${ALLOW_UNAUTHORIZED_CERTS} + - SENDER_EMAIL=${SENDER_EMAIL} + + # ENTERPRISE + - LICENSE_URL=${LICENSE_URL} + - FLOWISE_EE_LICENSE_KEY=${FLOWISE_EE_LICENSE_KEY} + - OFFLINE=${OFFLINE} + - INVITE_TOKEN_EXPIRY_IN_HOURS=${INVITE_TOKEN_EXPIRY_IN_HOURS} + - WORKSPACE_INVITE_TEMPLATE_PATH=${WORKSPACE_INVITE_TEMPLATE_PATH} + + # METRICS COLLECTION + - POSTHOG_PUBLIC_API_KEY=${POSTHOG_PUBLIC_API_KEY} + - ENABLE_METRICS=${ENABLE_METRICS} + - METRICS_PROVIDER=${METRICS_PROVIDER} + - METRICS_INCLUDE_NODE_METRICS=${METRICS_INCLUDE_NODE_METRICS} + - METRICS_SERVICE_NAME=${METRICS_SERVICE_NAME} + - METRICS_OPEN_TELEMETRY_METRIC_ENDPOINT=${METRICS_OPEN_TELEMETRY_METRIC_ENDPOINT} + - METRICS_OPEN_TELEMETRY_PROTOCOL=${METRICS_OPEN_TELEMETRY_PROTOCOL} + - METRICS_OPEN_TELEMETRY_DEBUG=${METRICS_OPEN_TELEMETRY_DEBUG} + + # PROXY + - GLOBAL_AGENT_HTTP_PROXY=${GLOBAL_AGENT_HTTP_PROXY} + - GLOBAL_AGENT_HTTPS_PROXY=${GLOBAL_AGENT_HTTPS_PROXY} + - GLOBAL_AGENT_NO_PROXY=${GLOBAL_AGENT_NO_PROXY} + + # --- Queue Configuration (Main Instance) --- + - MODE=${MODE:-queue} + - QUEUE_NAME=${QUEUE_NAME:-flowise-queue} + - QUEUE_REDIS_EVENT_STREAM_MAX_LEN=${QUEUE_REDIS_EVENT_STREAM_MAX_LEN} + - WORKER_CONCURRENCY=${WORKER_CONCURRENCY} + - REMOVE_ON_AGE=${REMOVE_ON_AGE} + - REMOVE_ON_COUNT=${REMOVE_ON_COUNT} + - REDIS_URL=${REDIS_URL:-redis://redis:6379} + - REDIS_HOST=${REDIS_HOST} + - REDIS_PORT=${REDIS_PORT} + - REDIS_USERNAME=${REDIS_USERNAME} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_TLS=${REDIS_TLS} + - REDIS_CERT=${REDIS_CERT} + - REDIS_KEY=${REDIS_KEY} + - REDIS_CA=${REDIS_CA} + - REDIS_KEEP_ALIVE=${REDIS_KEEP_ALIVE} + - ENABLE_BULLMQ_DASHBOARD=${ENABLE_BULLMQ_DASHBOARD} + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:${PORT:-3000}/api/v1/ping'] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + entrypoint: /bin/sh -c "sleep 3; flowise start" + depends_on: + - redis + networks: + - flowise-net + + flowise-worker: + image: flowiseai/flowise-worker:latest + container_name: flowise-worker + restart: always + volumes: + - ~/.flowise:/root/.flowise + environment: + # --- Essential Flowise Vars --- + - WORKER_PORT=${WORKER_PORT:-5566} + - DATABASE_PATH=${DATABASE_PATH:-/root/.flowise} + - DATABASE_TYPE=${DATABASE_TYPE} + - DATABASE_PORT=${DATABASE_PORT} + - DATABASE_HOST=${DATABASE_HOST} + - DATABASE_NAME=${DATABASE_NAME} + - DATABASE_USER=${DATABASE_USER} + - DATABASE_PASSWORD=${DATABASE_PASSWORD} + - DATABASE_SSL=${DATABASE_SSL} + - DATABASE_SSL_KEY_BASE64=${DATABASE_SSL_KEY_BASE64} + + # SECRET KEYS + - SECRETKEY_STORAGE_TYPE=${SECRETKEY_STORAGE_TYPE} + - SECRETKEY_PATH=${SECRETKEY_PATH} + - FLOWISE_SECRETKEY_OVERWRITE=${FLOWISE_SECRETKEY_OVERWRITE} + - SECRETKEY_AWS_ACCESS_KEY=${SECRETKEY_AWS_ACCESS_KEY} + - SECRETKEY_AWS_SECRET_KEY=${SECRETKEY_AWS_SECRET_KEY} + - SECRETKEY_AWS_REGION=${SECRETKEY_AWS_REGION} + - SECRETKEY_AWS_NAME=${SECRETKEY_AWS_NAME} + + # LOGGING + - DEBUG=${DEBUG} + - LOG_PATH=${LOG_PATH} + - LOG_LEVEL=${LOG_LEVEL} + + # CUSTOM TOOL DEPENDENCIES + - TOOL_FUNCTION_BUILTIN_DEP=${TOOL_FUNCTION_BUILTIN_DEP} + - TOOL_FUNCTION_EXTERNAL_DEP=${TOOL_FUNCTION_EXTERNAL_DEP} + + # STORAGE + - STORAGE_TYPE=${STORAGE_TYPE} + - BLOB_STORAGE_PATH=${BLOB_STORAGE_PATH} + - S3_STORAGE_BUCKET_NAME=${S3_STORAGE_BUCKET_NAME} + - S3_STORAGE_ACCESS_KEY_ID=${S3_STORAGE_ACCESS_KEY_ID} + - S3_STORAGE_SECRET_ACCESS_KEY=${S3_STORAGE_SECRET_ACCESS_KEY} + - S3_STORAGE_REGION=${S3_STORAGE_REGION} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL} + - S3_FORCE_PATH_STYLE=${S3_FORCE_PATH_STYLE} + - GOOGLE_CLOUD_STORAGE_CREDENTIAL=${GOOGLE_CLOUD_STORAGE_CREDENTIAL} + - GOOGLE_CLOUD_STORAGE_PROJ_ID=${GOOGLE_CLOUD_STORAGE_PROJ_ID} + - GOOGLE_CLOUD_STORAGE_BUCKET_NAME=${GOOGLE_CLOUD_STORAGE_BUCKET_NAME} + - GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS=${GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS} + + # SETTINGS + - NUMBER_OF_PROXIES=${NUMBER_OF_PROXIES} + - CORS_ORIGINS=${CORS_ORIGINS} + - IFRAME_ORIGINS=${IFRAME_ORIGINS} + - FLOWISE_FILE_SIZE_LIMIT=${FLOWISE_FILE_SIZE_LIMIT} + - SHOW_COMMUNITY_NODES=${SHOW_COMMUNITY_NODES} + - DISABLE_FLOWISE_TELEMETRY=${DISABLE_FLOWISE_TELEMETRY} + - DISABLED_NODES=${DISABLED_NODES} + - MODEL_LIST_CONFIG_JSON=${MODEL_LIST_CONFIG_JSON} + + # AUTH PARAMETERS + - APP_URL=${APP_URL} + - JWT_AUTH_TOKEN_SECRET=${JWT_AUTH_TOKEN_SECRET} + - JWT_REFRESH_TOKEN_SECRET=${JWT_REFRESH_TOKEN_SECRET} + - JWT_ISSUER=${JWT_ISSUER} + - JWT_AUDIENCE=${JWT_AUDIENCE} + - JWT_TOKEN_EXPIRY_IN_MINUTES=${JWT_TOKEN_EXPIRY_IN_MINUTES} + - JWT_REFRESH_TOKEN_EXPIRY_IN_MINUTES=${JWT_REFRESH_TOKEN_EXPIRY_IN_MINUTES} + - EXPIRE_AUTH_TOKENS_ON_RESTART=${EXPIRE_AUTH_TOKENS_ON_RESTART} + - EXPRESS_SESSION_SECRET=${EXPRESS_SESSION_SECRET} + - PASSWORD_RESET_TOKEN_EXPIRY_IN_MINS=${PASSWORD_RESET_TOKEN_EXPIRY_IN_MINS} + - PASSWORD_SALT_HASH_ROUNDS=${PASSWORD_SALT_HASH_ROUNDS} + - TOKEN_HASH_SECRET=${TOKEN_HASH_SECRET} + + # EMAIL + - SMTP_HOST=${SMTP_HOST} + - SMTP_PORT=${SMTP_PORT} + - SMTP_USER=${SMTP_USER} + - SMTP_PASSWORD=${SMTP_PASSWORD} + - SMTP_SECURE=${SMTP_SECURE} + - ALLOW_UNAUTHORIZED_CERTS=${ALLOW_UNAUTHORIZED_CERTS} + - SENDER_EMAIL=${SENDER_EMAIL} + + # ENTERPRISE + - LICENSE_URL=${LICENSE_URL} + - FLOWISE_EE_LICENSE_KEY=${FLOWISE_EE_LICENSE_KEY} + - OFFLINE=${OFFLINE} + - INVITE_TOKEN_EXPIRY_IN_HOURS=${INVITE_TOKEN_EXPIRY_IN_HOURS} + - WORKSPACE_INVITE_TEMPLATE_PATH=${WORKSPACE_INVITE_TEMPLATE_PATH} + + # METRICS COLLECTION + - POSTHOG_PUBLIC_API_KEY=${POSTHOG_PUBLIC_API_KEY} + - ENABLE_METRICS=${ENABLE_METRICS} + - METRICS_PROVIDER=${METRICS_PROVIDER} + - METRICS_INCLUDE_NODE_METRICS=${METRICS_INCLUDE_NODE_METRICS} + - METRICS_SERVICE_NAME=${METRICS_SERVICE_NAME} + - METRICS_OPEN_TELEMETRY_METRIC_ENDPOINT=${METRICS_OPEN_TELEMETRY_METRIC_ENDPOINT} + - METRICS_OPEN_TELEMETRY_PROTOCOL=${METRICS_OPEN_TELEMETRY_PROTOCOL} + - METRICS_OPEN_TELEMETRY_DEBUG=${METRICS_OPEN_TELEMETRY_DEBUG} + + # PROXY + - GLOBAL_AGENT_HTTP_PROXY=${GLOBAL_AGENT_HTTP_PROXY} + - GLOBAL_AGENT_HTTPS_PROXY=${GLOBAL_AGENT_HTTPS_PROXY} + - GLOBAL_AGENT_NO_PROXY=${GLOBAL_AGENT_NO_PROXY} + + # --- Queue Configuration (Worker Instance) --- + - MODE=${MODE:-queue} + - QUEUE_NAME=${QUEUE_NAME:-flowise-queue} + - QUEUE_REDIS_EVENT_STREAM_MAX_LEN=${QUEUE_REDIS_EVENT_STREAM_MAX_LEN} + - WORKER_CONCURRENCY=${WORKER_CONCURRENCY} + - REMOVE_ON_AGE=${REMOVE_ON_AGE} + - REMOVE_ON_COUNT=${REMOVE_ON_COUNT} + - REDIS_URL=${REDIS_URL:-redis://redis:6379} + - REDIS_HOST=${REDIS_HOST} + - REDIS_PORT=${REDIS_PORT} + - REDIS_USERNAME=${REDIS_USERNAME} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_TLS=${REDIS_TLS} + - REDIS_CERT=${REDIS_CERT} + - REDIS_KEY=${REDIS_KEY} + - REDIS_CA=${REDIS_CA} + - REDIS_KEEP_ALIVE=${REDIS_KEEP_ALIVE} + - ENABLE_BULLMQ_DASHBOARD=${ENABLE_BULLMQ_DASHBOARD} + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:${WORKER_PORT:-5566}/healthz'] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + entrypoint: /bin/sh -c "node /app/healthcheck/healthcheck.js & sleep 5 && pnpm run start-worker" + depends_on: + - redis + - flowise + networks: + - flowise-net + +volumes: + redis_data: + driver: local + +networks: + flowise-net: + driver: bridge diff --git a/docker/docker-compose-queue-source.yml b/docker/docker-compose-queue-source.yml new file mode 100644 index 000000000..a95608e5c --- /dev/null +++ b/docker/docker-compose-queue-source.yml @@ -0,0 +1,71 @@ +version: '3.1' + +services: + redis: + image: redis:alpine + container_name: flowise-redis + ports: + - '6379:6379' + volumes: + - redis_data:/data + networks: + - flowise-net + + flowise: + container_name: flowise-main + build: + context: .. # Build using the Dockerfile in the root directory + dockerfile: docker/Dockerfile + ports: + - '${PORT}:${PORT}' + volumes: + # Mount local .flowise to container's default location + - ../.flowise:/root/.flowise + environment: + # --- Essential Flowise Vars --- + - PORT=${PORT:-3000} + - DATABASE_PATH=/root/.flowise + - SECRETKEY_PATH=/root/.flowise + - LOG_PATH=/root/.flowise/logs + - BLOB_STORAGE_PATH=/root/.flowise/storage + # --- Queue Vars (Main Instance) --- + - MODE=queue + - QUEUE_NAME=flowise-queue # Ensure this matches worker + - REDIS_URL=redis://redis:6379 # Use service name 'redis' + depends_on: + - redis + networks: + - flowise-net + + flowise-worker: + container_name: flowise-worker + build: + context: .. # Build context is still the root + dockerfile: docker/worker/Dockerfile # Ensure this path is correct + volumes: + # Mount same local .flowise to worker + - ../.flowise:/root/.flowise + environment: + # --- Essential Flowise Vars --- + - WORKER_PORT=${WORKER_PORT:-5566} # Port for worker healthcheck + - DATABASE_PATH=/root/.flowise + - SECRETKEY_PATH=/root/.flowise + - LOG_PATH=/root/.flowise/logs + - BLOB_STORAGE_PATH=/root/.flowise/storage + # --- Queue Vars (Main Instance) --- + - MODE=queue + - QUEUE_NAME=flowise-queue # Ensure this matches worker + - REDIS_URL=redis://redis:6379 # Use service name 'redis' + depends_on: + - redis + - flowise + networks: + - flowise-net + +volumes: + redis_data: + driver: local + +networks: + flowise-net: + driver: bridge diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5cc563fc3..3f7529983 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.1' services: flowise: - image: flowiseai/flowise + image: flowiseai/flowise:latest restart: always environment: - PORT=${PORT} diff --git a/docker/worker/.env.example b/docker/worker/.env.example new file mode 100644 index 000000000..0540cf768 --- /dev/null +++ b/docker/worker/.env.example @@ -0,0 +1,165 @@ +WORKER_PORT=5566 + +# APIKEY_PATH=/your_apikey_path/.flowise # (will be deprecated by end of 2025) + +############################################################################################################ +############################################## DATABASE #################################################### +############################################################################################################ + +DATABASE_PATH=/root/.flowise +# DATABASE_TYPE=postgres +# DATABASE_PORT=5432 +# DATABASE_HOST="" +# DATABASE_NAME=flowise +# DATABASE_USER=root +# DATABASE_PASSWORD=mypassword +# DATABASE_SSL=true +# DATABASE_SSL_KEY_BASE64= + + +############################################################################################################ +############################################## SECRET KEYS ################################################# +############################################################################################################ + +# SECRETKEY_STORAGE_TYPE=local #(local | aws) +SECRETKEY_PATH=/root/.flowise +# FLOWISE_SECRETKEY_OVERWRITE=myencryptionkey # (if you want to overwrite the secret key) +# SECRETKEY_AWS_ACCESS_KEY= +# SECRETKEY_AWS_SECRET_KEY= +# SECRETKEY_AWS_REGION=us-west-2 +# SECRETKEY_AWS_NAME=FlowiseEncryptionKey + + +############################################################################################################ +############################################## LOGGING ##################################################### +############################################################################################################ + +# DEBUG=true +LOG_PATH=/root/.flowise/logs +# LOG_LEVEL=info #(error | warn | info | verbose | debug) +# TOOL_FUNCTION_BUILTIN_DEP=crypto,fs +# TOOL_FUNCTION_EXTERNAL_DEP=moment,lodash + + +############################################################################################################ +############################################## STORAGE ##################################################### +############################################################################################################ + +# STORAGE_TYPE=local (local | s3 | gcs) +BLOB_STORAGE_PATH=/root/.flowise/storage +# S3_STORAGE_BUCKET_NAME=flowise +# S3_STORAGE_ACCESS_KEY_ID= +# S3_STORAGE_SECRET_ACCESS_KEY= +# S3_STORAGE_REGION=us-west-2 +# S3_ENDPOINT_URL= +# S3_FORCE_PATH_STYLE=false +# GOOGLE_CLOUD_STORAGE_CREDENTIAL=/the/keyfilename/path +# GOOGLE_CLOUD_STORAGE_PROJ_ID= +# GOOGLE_CLOUD_STORAGE_BUCKET_NAME= +# GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS=true + + +############################################################################################################ +############################################## SETTINGS #################################################### +############################################################################################################ + +# NUMBER_OF_PROXIES= 1 +# CORS_ORIGINS=* +# IFRAME_ORIGINS=* +# FLOWISE_FILE_SIZE_LIMIT=50mb +# SHOW_COMMUNITY_NODES=true +# DISABLE_FLOWISE_TELEMETRY=true +# DISABLED_NODES=bufferMemory,chatOpenAI (comma separated list of node names to disable) +# Uncomment the following line to enable model list config, load the list of models from your local config file +# see https://raw.githubusercontent.com/FlowiseAI/Flowise/main/packages/components/models.json for the format +# MODEL_LIST_CONFIG_JSON=/your_model_list_config_file_path + + +############################################################################################################ +############################################ AUTH PARAMETERS ############################################### +############################################################################################################ + +# APP_URL=http://localhost:3000 + +# SMTP_HOST=smtp.host.com +# SMTP_PORT=465 +# SMTP_USER=smtp_user +# SMTP_PASSWORD=smtp_password +# SMTP_SECURE=true +# ALLOW_UNAUTHORIZED_CERTS=false +# SENDER_EMAIL=team@example.com + +JWT_AUTH_TOKEN_SECRET='AABBCCDDAABBCCDDAABBCCDDAABBCCDDAABBCCDD' +JWT_REFRESH_TOKEN_SECRET='AABBCCDDAABBCCDDAABBCCDDAABBCCDDAABBCCDD' +JWT_ISSUER='ISSUER' +JWT_AUDIENCE='AUDIENCE' +JWT_TOKEN_EXPIRY_IN_MINUTES=360 +JWT_REFRESH_TOKEN_EXPIRY_IN_MINUTES=43200 +# EXPIRE_AUTH_TOKENS_ON_RESTART=true # (if you need to expire all tokens on app restart) +# EXPRESS_SESSION_SECRET=flowise + +# INVITE_TOKEN_EXPIRY_IN_HOURS=24 +# PASSWORD_RESET_TOKEN_EXPIRY_IN_MINS=15 +# PASSWORD_SALT_HASH_ROUNDS=10 +# TOKEN_HASH_SECRET='popcorn' + +# WORKSPACE_INVITE_TEMPLATE_PATH=/path/to/custom/workspace_invite.hbs + + +############################################################################################################ +############################################# ENTERPRISE ################################################### +############################################################################################################ + +# LICENSE_URL= +# FLOWISE_EE_LICENSE_KEY= +# OFFLINE= + + +############################################################################################################ +########################################### METRICS COLLECTION ############################################# +############################################################################################################ + +# POSTHOG_PUBLIC_API_KEY=your_posthog_public_api_key + +# ENABLE_METRICS=false +# METRICS_PROVIDER=prometheus # prometheus | open_telemetry +# METRICS_INCLUDE_NODE_METRICS=true # default is true +# METRICS_SERVICE_NAME=FlowiseAI + +# ONLY NEEDED if METRICS_PROVIDER=open_telemetry +# METRICS_OPEN_TELEMETRY_METRIC_ENDPOINT=http://localhost:4318/v1/metrics +# METRICS_OPEN_TELEMETRY_PROTOCOL=http # http | grpc | proto (default is http) +# METRICS_OPEN_TELEMETRY_DEBUG=true # default is false + + +############################################################################################################ +############################################### PROXY ###################################################### +############################################################################################################ + +# Uncomment the following lines to enable global agent proxy, see https://www.npmjs.com/package/global-agent for more details +# GLOBAL_AGENT_HTTP_PROXY=CorporateHttpProxyUrl +# GLOBAL_AGENT_HTTPS_PROXY=CorporateHttpsProxyUrl +# GLOBAL_AGENT_NO_PROXY=ExceptionHostsToBypassProxyIfNeeded + + +############################################################################################################ +########################################### QUEUE CONFIGURATION ############################################ +############################################################################################################ + +# MODE=queue #(queue | main) +# QUEUE_NAME=flowise-queue +# QUEUE_REDIS_EVENT_STREAM_MAX_LEN=100000 +# WORKER_CONCURRENCY=100000 +# REMOVE_ON_AGE=86400 +# REMOVE_ON_COUNT=10000 +# REDIS_URL= +# REDIS_HOST=localhost +# REDIS_PORT=6379 +# REDIS_USERNAME= +# REDIS_PASSWORD= +# REDIS_TLS= +# REDIS_CERT= +# REDIS_KEY= +# REDIS_CA= +# REDIS_KEEP_ALIVE= +# ENABLE_BULLMQ_DASHBOARD= \ No newline at end of file diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile new file mode 100644 index 000000000..655b3de47 --- /dev/null +++ b/docker/worker/Dockerfile @@ -0,0 +1,49 @@ +FROM node:20-alpine + +RUN apk add --update libc6-compat python3 make g++ +# needed for pdfjs-dist +RUN apk add --no-cache build-base cairo-dev pango-dev + +# Install Chromium and curl for container-level health checks +RUN apk add --no-cache chromium curl + +#install PNPM globaly +RUN npm install -g pnpm + +ENV PUPPETEER_SKIP_DOWNLOAD=true +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + +ENV NODE_OPTIONS=--max-old-space-size=8192 + +WORKDIR /usr/src + +# Copy app source +COPY . . + +RUN pnpm install + +RUN pnpm build + +# --- Healthcheck Setup --- + +WORKDIR /app/healthcheck + +COPY docker/worker/healthcheck/package.json . + +RUN npm install --omit=dev + +COPY docker/worker/healthcheck/healthcheck.js . + +# --- End Healthcheck Setup --- + +# Set the main working directory back +WORKDIR /usr/src + +# Environment variables for port configuration +ENV WORKER_PORT=5566 + +# Expose port (can be overridden by env var) +EXPOSE ${WORKER_PORT} + +# Start healthcheck in background and flowise worker in foreground +CMD ["/bin/sh", "-c", "node /app/healthcheck/healthcheck.js & sleep 5 && pnpm run start-worker"] diff --git a/docker/worker/README.md b/docker/worker/README.md index 82769c1e2..b2299cd03 100644 --- a/docker/worker/README.md +++ b/docker/worker/README.md @@ -18,7 +18,11 @@ Here’s an overview of the process: ## Setting up Worker: -1. Copy paste the same `.env` file used to setup main server. Change the `PORT` to other available port numbers. Ex: 5566 -2. `docker compose up -d` -3. Open [http://localhost:5566](http://localhost:5566) +1. Navigate to `docker/worker` folder +2. In the `.env.example`, setup all the necessary env variables for `QUEUE CONFIGURATION`. Env variables for worker must match the one for main server. Change the `WORKER_PORT` to other available port numbers to listen for healthcheck. Ex: 5566 +3. `docker compose up -d` 4. You can bring the worker container down by `docker compose stop` + +## Entrypoint: + +Different from main server image which is using `flowise start`, entrypoint for worker is `pnpm run start-worker`. This is because the worker's [Dockerfile](./Dockerfile) build the image from source files via `pnpm build` instead of npm registry via `RUN npm install -g flowise`. diff --git a/docker/worker/docker-compose.yml b/docker/worker/docker-compose.yml index a327d9ac4..4a8924dd2 100644 --- a/docker/worker/docker-compose.yml +++ b/docker/worker/docker-compose.yml @@ -2,10 +2,10 @@ version: '3.1' services: flowise: - image: flowiseai/flowise + image: flowiseai/flowise-worker:latest restart: always environment: - - PORT=${PORT} + - WORKER_PORT=${WORKER_PORT:-5566} # DATABASE - DATABASE_PATH=${DATABASE_PATH} @@ -124,7 +124,13 @@ services: - REDIS_KEEP_ALIVE=${REDIS_KEEP_ALIVE} - ENABLE_BULLMQ_DASHBOARD=${ENABLE_BULLMQ_DASHBOARD} ports: - - '${PORT}:${PORT}' + - '${WORKER_PORT}:${WORKER_PORT}' + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:${WORKER_PORT}/healthz'] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s volumes: - ~/.flowise:/root/.flowise - entrypoint: /bin/sh -c "sleep 3; flowise worker" + entrypoint: /bin/sh -c "node /app/healthcheck/healthcheck.js & sleep 5 && pnpm run start-worker" diff --git a/docker/worker/healthcheck/healthcheck.js b/docker/worker/healthcheck/healthcheck.js new file mode 100644 index 000000000..fcc204f7d --- /dev/null +++ b/docker/worker/healthcheck/healthcheck.js @@ -0,0 +1,13 @@ +const express = require('express') +const app = express() + +const port = process.env.WORKER_PORT || 5566 + +app.get('/healthz', (req, res) => { + res.status(200).send('OK') +}) + +app.listen(port, () => { + // eslint-disable-next-line no-console + console.log(`Healthcheck server listening on port ${port}`) +}) diff --git a/docker/worker/healthcheck/package.json b/docker/worker/healthcheck/package.json new file mode 100644 index 000000000..aa7bfd6be --- /dev/null +++ b/docker/worker/healthcheck/package.json @@ -0,0 +1,13 @@ +{ + "name": "flowise-worker-healthcheck", + "version": "1.0.0", + "description": "Simple healthcheck server for Flowise worker", + "main": "healthcheck.js", + "private": true, + "scripts": { + "start": "node healthcheck.js" + }, + "dependencies": { + "express": "^4.19.2" + } +} diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts index 64d016aae..43a75c925 100644 --- a/packages/server/src/utils/logger.ts +++ b/packages/server/src/utils/logger.ts @@ -111,6 +111,7 @@ const logger = createLogger({ defaultMeta: { package: 'server' }, + exitOnError: false, transports: [ new transports.Console(), ...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local' @@ -152,7 +153,11 @@ const logger = createLogger({ }) ] : []), - ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsErrorStream] : []) + ...(process.env.STORAGE_TYPE === 'gcs' ? [gcsErrorStream] : []), + // Always provide a fallback rejection handler when no other handlers are configured + ...((!process.env.DEBUG || process.env.DEBUG !== 'true') && process.env.STORAGE_TYPE !== 's3' && process.env.STORAGE_TYPE !== 'gcs' + ? [new transports.Console()] + : []) ] })