Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature/Append uploaded images to message history (#3105) 

append uploaded images to message history
This commit is contained in:
Henry Heng 2024-09-02 11:27:45 +01:00 committed by GitHub
parent b28885240d
commit 4c1378c670
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 60 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts
View File

@ -424,7 +424,7 @@ class BufferMemory extends FlowiseMemory implements MemoryMethods {
}
if (returnBaseMessages) {
return mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage)
}
let returnIMessages: IMessage[] = []

2
packages/components/nodes/memory/AgentMemory/sqliteSaver.ts
View File

@ -219,7 +219,7 @@ CREATE TABLE IF NOT EXISTS ${this.tableName} (
}
if (returnBaseMessages) {
return mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage)
}
let returnIMessages: IMessage[] = []

2
packages/components/nodes/memory/BufferMemory/BufferMemory.ts
View File

@ -117,7 +117,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
}
if (returnBaseMessages) {
return mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage)
}
let returnIMessages: IMessage[] = []

2
packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts
View File

@ -132,7 +132,7 @@ class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMe
}
if (returnBaseMessages) {
return mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage)
}
let returnIMessages: IMessage[] = []

2
packages/components/nodes/memory/ConversationSummaryBufferMemory/ConversationSummaryBufferMemory.ts
View File

@ -136,7 +136,7 @@ class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory
chatMessage.unshift(...prependMessages)
}
let baseMessages = mapChatMessageToBaseMessage(chatMessage)
let baseMessages = await mapChatMessageToBaseMessage(chatMessage)
// Prune baseMessages if it exceeds max token limit
if (this.movingSummaryBuffer) {

2
packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts
View File

@ -127,7 +127,7 @@ class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements
chatMessage.unshift(...prependMessages)
}
const baseMessages = mapChatMessageToBaseMessage(chatMessage)
const baseMessages = await mapChatMessageToBaseMessage(chatMessage)
// Get summary
if (this.llm && typeof this.llm !== 'string') {

2
packages/components/nodes/memory/DynamoDb/DynamoDb.ts
View File

@ -260,7 +260,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
.filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
const baseMessages = messages.map(mapStoredMessageToChatMessage)
if (prependMessages?.length) {
baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}

2
packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts
View File

@ -172,7 +172,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
const messages = document?.messages || []
const baseMessages = messages.map(mapStoredMessageToChatMessage)
if (prependMessages?.length) {
baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}

2
packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts
View File

@ -208,7 +208,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
const baseMessages = orderedMessages.map(mapStoredMessageToChatMessage)
if (prependMessages?.length) {
baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}

2
packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts
View File

@ -162,7 +162,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
const previousMessages = orderedMessages.filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
const baseMessages = previousMessages.map(mapStoredMessageToChatMessage)
if (prependMessages?.length) {
baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}

2
packages/components/nodes/memory/ZepMemory/ZepMemory.ts
View File

@ -176,7 +176,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
const memoryVariables = await this.loadMemoryVariables({}, id)
const baseMessages = memoryVariables[this.memoryKey]
if (prependMessages?.length) {
baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}

2
packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts
View File

@ -169,7 +169,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
const memoryVariables = await this.loadMemoryVariables({}, id)
const baseMessages = memoryVariables[this.memoryKey]
if (prependMessages?.length) {
baseMessages.unshift(...mapChatMessageToBaseMessage(prependMessages))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}

51
packages/components/src/utils.ts
View File

@ -5,9 +5,10 @@ import * as path from 'path'
import { JSDOM } from 'jsdom'
import { z } from 'zod'
import { DataSource } from 'typeorm'
import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable } from './Interface'
import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable, MessageContentImageUrl } from './Interface'
import { AES, enc } from 'crypto-js'
import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages'
import { getFileFromStorage } from './storageUtils'
export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
@ -601,14 +602,58 @@ export const getUserHome = (): string => {
* @param {IChatMessage[]} chatmessages
* @returns {BaseMessage[]}
*/
export const mapChatMessageToBaseMessage = (chatmessages: any[] = []): BaseMessage[] => {
export const mapChatMessageToBaseMessage = async (chatmessages: any[] = []): Promise<BaseMessage[]> => {
const chatHistory = []
for (const message of chatmessages) {
if (message.role === 'apiMessage' || message.type === 'apiMessage') {
chatHistory.push(new AIMessage(message.content || ''))
} else if (message.role === 'userMessage' || message.role === 'userMessage') {
chatHistory.push(new HumanMessage(message.content || ''))
// check for image uploads
if (message.fileUploads) {
// example: [{"type":"stored-file","name":"0_DiXc4ZklSTo3M8J4.jpg","mime":"image/jpeg"}]
try {
const uploads = JSON.parse(message.fileUploads)
const imageContents: MessageContentImageUrl[] = []
for (const upload of uploads) {
if (upload.type === 'stored-file') {
const fileData = await getFileFromStorage(upload.name, message.chatflowid, message.chatId)
// as the image is stored in the server, read the file and convert it to base64
const bf = 'data:' + upload.mime + ';base64,' + fileData.toString('base64')
imageContents.push({
type: 'image_url',
image_url: {
url: bf
}
})
} else if (upload.type === 'url') {
imageContents.push({
type: 'image_url',
image_url: {
url: upload.data
}
})
}
}
chatHistory.push(
new HumanMessage({
content: [
{
type: 'text',
text: message.content
},
...imageContents
]
})
)
} catch (e) {
// failed to parse fileUploads, continue with text only
chatHistory.push(new HumanMessage(message.content || ''))
}
} else {
chatHistory.push(new HumanMessage(message.content || ''))
}
}
}
return chatHistory