From 4c1378c670d8da8d17631e781eb41c46fa6dcde2 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 2 Sep 2024 11:27:45 +0100 Subject: [PATCH] Feature/Append uploaded images to message history (#3105) append uploaded images to message history --- .../ConversationalRetrievalQAChain.ts | 2 +- .../nodes/memory/AgentMemory/sqliteSaver.ts | 2 +- .../nodes/memory/BufferMemory/BufferMemory.ts | 2 +- .../BufferWindowMemory/BufferWindowMemory.ts | 2 +- .../ConversationSummaryBufferMemory.ts | 2 +- .../ConversationSummaryMemory.ts | 2 +- .../nodes/memory/DynamoDb/DynamoDb.ts | 2 +- .../memory/MongoDBMemory/MongoDBMemory.ts | 2 +- .../RedisBackedChatMemory.ts | 2 +- .../UpstashRedisBackedChatMemory.ts | 2 +- .../nodes/memory/ZepMemory/ZepMemory.ts | 2 +- .../memory/ZepMemoryCloud/ZepMemoryCloud.ts | 2 +- packages/components/src/utils.ts | 51 +++++++++++++++++-- 13 files changed, 60 insertions(+), 15 deletions(-) diff --git a/packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts b/packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts index 5a0e40685..a65775e9b 100644 --- a/packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts +++ b/packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts @@ -424,7 +424,7 @@ class BufferMemory extends FlowiseMemory implements MemoryMethods { } if (returnBaseMessages) { - return mapChatMessageToBaseMessage(chatMessage) + return await mapChatMessageToBaseMessage(chatMessage) } let returnIMessages: IMessage[] = [] diff --git a/packages/components/nodes/memory/AgentMemory/sqliteSaver.ts b/packages/components/nodes/memory/AgentMemory/sqliteSaver.ts index a8eace00d..4e5c5b80f 100644 --- a/packages/components/nodes/memory/AgentMemory/sqliteSaver.ts +++ b/packages/components/nodes/memory/AgentMemory/sqliteSaver.ts @@ -219,7 +219,7 @@ CREATE TABLE IF NOT EXISTS ${this.tableName} ( } if (returnBaseMessages) { - return mapChatMessageToBaseMessage(chatMessage) + return await mapChatMessageToBaseMessage(chatMessage) } let returnIMessages: IMessage[] = [] diff --git a/packages/components/nodes/memory/BufferMemory/BufferMemory.ts b/packages/components/nodes/memory/BufferMemory/BufferMemory.ts index 63056007c..972301daf 100644 --- a/packages/components/nodes/memory/BufferMemory/BufferMemory.ts +++ b/packages/components/nodes/memory/BufferMemory/BufferMemory.ts @@ -117,7 +117,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods { } if (returnBaseMessages) { - return mapChatMessageToBaseMessage(chatMessage) + return await mapChatMessageToBaseMessage(chatMessage) } let returnIMessages: IMessage[] = [] diff --git a/packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts b/packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts index 422cac55f..5cf717a21 100644 --- a/packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts +++ b/packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts @@ -132,7 +132,7 @@ class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMe } if (returnBaseMessages) { - return mapChatMessageToBaseMessage(chatMessage) + return await mapChatMessageToBaseMessage(chatMessage) } let returnIMessages: IMessage[] = [] diff --git a/packages/components/nodes/memory/ConversationSummaryBufferMemory/ConversationSummaryBufferMemory.ts b/packages/components/nodes/memory/ConversationSummaryBufferMemory/ConversationSummaryBufferMemory.ts index 5f58b0ca8..f9b0c14f0 100644 --- a/packages/components/nodes/memory/ConversationSummaryBufferMemory/ConversationSummaryBufferMemory.ts +++ b/packages/components/nodes/memory/ConversationSummaryBufferMemory/ConversationSummaryBufferMemory.ts @@ -136,7 +136,7 @@ class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory chatMessage.unshift(...prependMessages) } - let baseMessages = mapChatMessageToBaseMessage(chatMessage) + let baseMessages = await mapChatMessageToBaseMessage(chatMessage) // Prune baseMessages if it exceeds max token limit if (this.movingSummaryBuffer) { diff --git a/packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts b/packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts index bd1c5617d..a1219c1ff 100644 --- a/packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts +++ b/packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts @@ -127,7 +127,7 @@ class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements chatMessage.unshift(...prependMessages) } - const baseMessages = mapChatMessageToBaseMessage(chatMessage) + const baseMessages = await mapChatMessageToBaseMessage(chatMessage) // Get summary if (this.llm && typeof this.llm !== 'string') { diff --git a/packages/components/nodes/memory/DynamoDb/DynamoDb.ts b/packages/components/nodes/memory/DynamoDb/DynamoDb.ts index 94525e967..0ad0354f4 100644 --- a/packages/components/nodes/memory/DynamoDb/DynamoDb.ts +++ b/packages/components/nodes/memory/DynamoDb/DynamoDb.ts @@ -260,7 +260,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods { .filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined) const baseMessages = messages.map(mapStoredMessageToChatMessage) if (prependMessages?.length) { - baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages)) + baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages))) } return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages) } diff --git a/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts b/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts index 98979d9fa..9bb6bbf82 100644 --- a/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts +++ b/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts @@ -172,7 +172,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods { const messages = document?.messages || [] const baseMessages = messages.map(mapStoredMessageToChatMessage) if (prependMessages?.length) { - baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages)) + baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages))) } return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages) } diff --git a/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts b/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts index 9900b8cd0..4d6ac3c72 100644 --- a/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts +++ b/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts @@ -208,7 +208,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods { const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message)) const baseMessages = orderedMessages.map(mapStoredMessageToChatMessage) if (prependMessages?.length) { - baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages)) + baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages))) } return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages) } diff --git a/packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts b/packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts index 7eb9b3906..595803c52 100644 --- a/packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts +++ b/packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts @@ -162,7 +162,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods { const previousMessages = orderedMessages.filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined) const baseMessages = previousMessages.map(mapStoredMessageToChatMessage) if (prependMessages?.length) { - baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages)) + baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages))) } return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages) } diff --git a/packages/components/nodes/memory/ZepMemory/ZepMemory.ts b/packages/components/nodes/memory/ZepMemory/ZepMemory.ts index 3f654146d..7832d0ff5 100644 --- a/packages/components/nodes/memory/ZepMemory/ZepMemory.ts +++ b/packages/components/nodes/memory/ZepMemory/ZepMemory.ts @@ -176,7 +176,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods { const memoryVariables = await this.loadMemoryVariables({}, id) const baseMessages = memoryVariables[this.memoryKey] if (prependMessages?.length) { - baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages)) + baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages))) } return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages) } diff --git a/packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts b/packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts index b524bfe54..237e026c7 100644 --- a/packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts +++ b/packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts @@ -169,7 +169,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods { const memoryVariables = await this.loadMemoryVariables({}, id) const baseMessages = memoryVariables[this.memoryKey] if (prependMessages?.length) { - baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages)) + baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages))) } return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages) } diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index 317103c8c..f3b8edbec 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -5,9 +5,10 @@ import * as path from 'path' import { JSDOM } from 'jsdom' import { z } from 'zod' import { DataSource } from 'typeorm' -import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable } from './Interface' +import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable, MessageContentImageUrl } from './Interface' import { AES, enc } from 'crypto-js' import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages' +import { getFileFromStorage } from './storageUtils' export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}} export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank @@ -601,14 +602,58 @@ export const getUserHome = (): string => { * @param {IChatMessage[]} chatmessages * @returns {BaseMessage[]} */ -export const mapChatMessageToBaseMessage = (chatmessages: any[] = []): BaseMessage[] => { +export const mapChatMessageToBaseMessage = async (chatmessages: any[] = []): Promise => { const chatHistory = [] for (const message of chatmessages) { if (message.role === 'apiMessage' || message.type === 'apiMessage') { chatHistory.push(new AIMessage(message.content || '')) } else if (message.role === 'userMessage' || message.role === 'userMessage') { - chatHistory.push(new HumanMessage(message.content || '')) + // check for image uploads + if (message.fileUploads) { + // example: [{"type":"stored-file","name":"0_DiXc4ZklSTo3M8J4.jpg","mime":"image/jpeg"}] + try { + const uploads = JSON.parse(message.fileUploads) + const imageContents: MessageContentImageUrl[] = [] + for (const upload of uploads) { + if (upload.type === 'stored-file') { + const fileData = await getFileFromStorage(upload.name, message.chatflowid, message.chatId) + // as the image is stored in the server, read the file and convert it to base64 + const bf = 'data:' + upload.mime + ';base64,' + fileData.toString('base64') + + imageContents.push({ + type: 'image_url', + image_url: { + url: bf + } + }) + } else if (upload.type === 'url') { + imageContents.push({ + type: 'image_url', + image_url: { + url: upload.data + } + }) + } + } + chatHistory.push( + new HumanMessage({ + content: [ + { + type: 'text', + text: message.content + }, + ...imageContents + ] + }) + ) + } catch (e) { + // failed to parse fileUploads, continue with text only + chatHistory.push(new HumanMessage(message.content || '')) + } + } else { + chatHistory.push(new HumanMessage(message.content || '')) + } } } return chatHistory