From 842bfc66fe012f268e98418cabf815da47d1986e Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Wed, 19 Jun 2024 23:26:56 +0100 Subject: [PATCH] Feature/Add ability to upload image url when calling API (#2683) add ability to upload image url when calling API --- packages/components/src/multiModalUtils.ts | 8 ++++++++ packages/server/src/utils/buildChatflow.ts | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/packages/components/src/multiModalUtils.ts b/packages/components/src/multiModalUtils.ts index fbb13ffb5..412361aa6 100644 --- a/packages/components/src/multiModalUtils.ts +++ b/packages/components/src/multiModalUtils.ts @@ -20,6 +20,14 @@ export const addImagesToMessages = async ( // as the image is stored in the server, read the file and convert it to base64 bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64') + imageContent.push({ + type: 'image_url', + image_url: { + url: bf, + detail: multiModalOption.image.imageResolution ?? 'low' + } + }) + } else if (upload.type == 'url' && bf) { imageContent.push({ type: 'image_url', image_url: { diff --git a/packages/server/src/utils/buildChatflow.ts b/packages/server/src/utils/buildChatflow.ts index b1669d050..9b363858c 100644 --- a/packages/server/src/utils/buildChatflow.ts +++ b/packages/server/src/utils/buildChatflow.ts @@ -94,6 +94,12 @@ export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInter fileUploads[i] = omit(upload, ['data']) } + if (upload.type === 'url' && upload.data) { + const filename = upload.name + const urlData = upload.data + fileUploads[i] = { data: urlData, name: filename, type: 'url', mime: upload.mime ?? 'image/png' } + } + // Run Speech to Text conversion if (upload.mime === 'audio/webm' || upload.mime === 'audio/mp4' || upload.mime === 'audio/ogg') { logger.debug(`Attempting a speech to text conversion...`)