diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts index 14361e43f..404f4db47 100644 --- a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts +++ b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts @@ -151,7 +151,6 @@ const prepareAgent = async ( if (llmSupportsVision(model)) { const visionChatModel = model as IVisionChatModal - // let humanImageMessages: HumanMessage[] = [] const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) if (messageContent?.length) { diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index ffea16f5b..16493adce 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -1,7 +1,16 @@ import { ConversationChain } from 'langchain/chains' -import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts' +import { + ChatPromptTemplate, + HumanMessagePromptTemplate, + MessagesPlaceholder, + SystemMessagePromptTemplate, + BaseMessagePromptTemplateLike, + PromptTemplate +} from '@langchain/core/prompts' import { RunnableSequence } from '@langchain/core/runnables' import { StringOutputParser } from '@langchain/core/output_parsers' +import { BaseChatModel } from '@langchain/core/language_models/chat_models' +import { HumanMessage } from '@langchain/core/messages' import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse } from '../../outputparsers/OutputParserHelpers' @@ -156,12 +165,29 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte const memory = nodeData.inputs?.memory as FlowiseMemory const prompt = nodeData.inputs?.systemMessagePrompt as string const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate + let model = nodeData.inputs?.model as BaseChatModel if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) { const sysPrompt = chatPromptTemplate.promptMessages[0] const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1] const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt] + // OpenAI works better when separate images into standalone human messages + if (model instanceof ChatOpenAI && humanImageMessages.length) { + messages.push(new HumanMessage({ content: [...humanImageMessages] })) + } else if (humanImageMessages.length) { + const lastMessage = messages.pop() as HumanMessagePromptTemplate + const template = (lastMessage.prompt as PromptTemplate).template as string + const msg = HumanMessagePromptTemplate.fromTemplate([ + ...humanImageMessages, + { + text: template + } + ]) + msg.inputVariables = lastMessage.inputVariables + messages.push(msg) + } + const chatPrompt = ChatPromptTemplate.fromMessages(messages) if ((chatPromptTemplate as any).promptValues) { // @ts-ignore @@ -171,12 +197,19 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte return chatPrompt } - const messages = [ + const messages: BaseMessagePromptTemplateLike[] = [ SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage), - new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), - HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages]) + new MessagesPlaceholder(memory.memoryKey ?? 'chat_history') ] + // OpenAI works better when separate images into standalone human messages + if (model instanceof ChatOpenAI && humanImageMessages.length) { + messages.push(HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)) + messages.push(new HumanMessage({ content: [...humanImageMessages] })) + } else if (humanImageMessages.length) { + messages.push(HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages])) + } + const chatPrompt = ChatPromptTemplate.fromMessages(messages) return chatPrompt @@ -184,7 +217,7 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => { const chatHistory = options.chatHistory - let model = nodeData.inputs?.model as ChatOpenAI + let model = nodeData.inputs?.model as BaseChatModel const memory = nodeData.inputs?.memory as FlowiseMemory const memoryKey = memory.memoryKey ?? 'chat_history'