Feature/Append uploaded images to message history (#3105)

append uploaded images to message history

packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts

@@ -424,7 +424,7 @@ class BufferMemory extends FlowiseMemory implements MemoryMethods {
         }
 
         if (returnBaseMessages) {
-            return mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/AgentMemory/sqliteSaver.ts

@@ -219,7 +219,7 @@ CREATE TABLE IF NOT EXISTS ${this.tableName} (
         }
 
         if (returnBaseMessages) {
-            return mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/BufferMemory/BufferMemory.ts

@@ -117,7 +117,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         }
 
         if (returnBaseMessages) {
-            return mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts

@@ -132,7 +132,7 @@ class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMe
         }
 
         if (returnBaseMessages) {
-            return mapChatMessageToBaseMessage(chatMessage)
+            return await mapChatMessageToBaseMessage(chatMessage)
         }
 
         let returnIMessages: IMessage[] = []

packages/components/nodes/memory/ConversationSummaryBufferMemory/ConversationSummaryBufferMemory.ts

@@ -136,7 +136,7 @@ class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory
             chatMessage.unshift(...prependMessages)
         }
 
-        let baseMessages = mapChatMessageToBaseMessage(chatMessage)
+        let baseMessages = await mapChatMessageToBaseMessage(chatMessage)
 
         // Prune baseMessages if it exceeds max token limit
         if (this.movingSummaryBuffer) {

packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts

@@ -127,7 +127,7 @@ class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements
             chatMessage.unshift(...prependMessages)
         }
 
-        const baseMessages = mapChatMessageToBaseMessage(chatMessage)
+        const baseMessages = await mapChatMessageToBaseMessage(chatMessage)
 
         // Get summary
         if (this.llm && typeof this.llm !== 'string') {

packages/components/nodes/memory/DynamoDb/DynamoDb.ts

@@ -260,7 +260,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
             .filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
         const baseMessages = messages.map(mapStoredMessageToChatMessage)
         if (prependMessages?.length) {
-            baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts

@@ -172,7 +172,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         const messages = document?.messages || []
         const baseMessages = messages.map(mapStoredMessageToChatMessage)
         if (prependMessages?.length) {
-            baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts

@@ -208,7 +208,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
         const baseMessages = orderedMessages.map(mapStoredMessageToChatMessage)
         if (prependMessages?.length) {
-            baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts

@@ -162,7 +162,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
         const previousMessages = orderedMessages.filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
         const baseMessages = previousMessages.map(mapStoredMessageToChatMessage)
         if (prependMessages?.length) {
-            baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/memory/ZepMemory/ZepMemory.ts

@@ -176,7 +176,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
         const memoryVariables = await this.loadMemoryVariables({}, id)
         const baseMessages = memoryVariables[this.memoryKey]
         if (prependMessages?.length) {
-            baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts

@@ -169,7 +169,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
         const memoryVariables = await this.loadMemoryVariables({}, id)
         const baseMessages = memoryVariables[this.memoryKey]
         if (prependMessages?.length) {
-            baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
+            baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
         }
         return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
     }

packages/components/src/utils.ts

@@ -5,9 +5,10 @@ import * as path from 'path'
 import { JSDOM } from 'jsdom'
 import { z } from 'zod'
 import { DataSource } from 'typeorm'
-import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable } from './Interface'
+import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable, MessageContentImageUrl } from './Interface'
 import { AES, enc } from 'crypto-js'
 import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages'
+import { getFileFromStorage } from './storageUtils'
 
 export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
 export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
@@ -601,14 +602,58 @@ export const getUserHome = (): string => {
  * @param {IChatMessage[]} chatmessages
  * @returns {BaseMessage[]}
  */
-export const mapChatMessageToBaseMessage = (chatmessages: any[] = []): BaseMessage[] => {
+export const mapChatMessageToBaseMessage = async (chatmessages: any[] = []): Promise<BaseMessage[]> => {
     const chatHistory = []
 
     for (const message of chatmessages) {
         if (message.role === 'apiMessage' || message.type === 'apiMessage') {
             chatHistory.push(new AIMessage(message.content || ''))
         } else if (message.role === 'userMessage' || message.role === 'userMessage') {
-            chatHistory.push(new HumanMessage(message.content || ''))
+            // check for image uploads
+            if (message.fileUploads) {
+                // example: [{"type":"stored-file","name":"0_DiXc4ZklSTo3M8J4.jpg","mime":"image/jpeg"}]
+                try {
+                    const uploads = JSON.parse(message.fileUploads)
+                    const imageContents: MessageContentImageUrl[] = []
+                    for (const upload of uploads) {
+                        if (upload.type === 'stored-file') {
+                            const fileData = await getFileFromStorage(upload.name, message.chatflowid, message.chatId)
+                            // as the image is stored in the server, read the file and convert it to base64
+                            const bf = 'data:' + upload.mime + ';base64,' + fileData.toString('base64')
+
+                            imageContents.push({
+                                type: 'image_url',
+                                image_url: {
+                                    url: bf
+                                }
+                            })
+                        } else if (upload.type === 'url') {
+                            imageContents.push({
+                                type: 'image_url',
+                                image_url: {
+                                    url: upload.data
+                                }
+                            })
+                        }
+                    }
+                    chatHistory.push(
+                        new HumanMessage({
+                            content: [
+                                {
+                                    type: 'text',
+                                    text: message.content
+                                },
+                                ...imageContents
+                            ]
+                        })
+                    )
+                } catch (e) {
+                    // failed to parse fileUploads, continue with text only
+                    chatHistory.push(new HumanMessage(message.content || ''))
+                }
+            } else {
+                chatHistory.push(new HumanMessage(message.content || ''))
+            }
         }
     }
     return chatHistory