add separate logic for conversation chain for openai vision

packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts

@@ -151,7 +151,6 @@ const prepareAgent = async (
 
     if (llmSupportsVision(model)) {
         const visionChatModel = model as IVisionChatModal
-        // let humanImageMessages: HumanMessage[] = []
         const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
 
         if (messageContent?.length) {

packages/components/nodes/chains/ConversationChain/ConversationChain.ts

@@ -1,7 +1,16 @@
 import { ConversationChain } from 'langchain/chains'
-import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts'
+import {
+    ChatPromptTemplate,
+    HumanMessagePromptTemplate,
+    MessagesPlaceholder,
+    SystemMessagePromptTemplate,
+    BaseMessagePromptTemplateLike,
+    PromptTemplate
+} from '@langchain/core/prompts'
 import { RunnableSequence } from '@langchain/core/runnables'
 import { StringOutputParser } from '@langchain/core/output_parsers'
+import { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { HumanMessage } from '@langchain/core/messages'
 import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
 import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
@@ -156,12 +165,29 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const prompt = nodeData.inputs?.systemMessagePrompt as string
     const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate
+    let model = nodeData.inputs?.model as BaseChatModel
 
     if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) {
         const sysPrompt = chatPromptTemplate.promptMessages[0]
         const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1]
         const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt]
 
+        // OpenAI works better when separate images into standalone human messages
+        if (model instanceof ChatOpenAI && humanImageMessages.length) {
+            messages.push(new HumanMessage({ content: [...humanImageMessages] }))
+        } else if (humanImageMessages.length) {
+            const lastMessage = messages.pop() as HumanMessagePromptTemplate
+            const template = (lastMessage.prompt as PromptTemplate).template as string
+            const msg = HumanMessagePromptTemplate.fromTemplate([
+                ...humanImageMessages,
+                {
+                    text: template
+                }
+            ])
+            msg.inputVariables = lastMessage.inputVariables
+            messages.push(msg)
+        }
+
         const chatPrompt = ChatPromptTemplate.fromMessages(messages)
         if ((chatPromptTemplate as any).promptValues) {
             // @ts-ignore
@@ -171,12 +197,19 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte
         return chatPrompt
     }
 
-    const messages = [
+    const messages: BaseMessagePromptTemplateLike[] = [
         SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage),
-        new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
+        new MessagesPlaceholder(memory.memoryKey ?? 'chat_history')
-        HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages])
     ]
 
+    // OpenAI works better when separate images into standalone human messages
+    if (model instanceof ChatOpenAI && humanImageMessages.length) {
+        messages.push(HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`))
+        messages.push(new HumanMessage({ content: [...humanImageMessages] }))
+    } else if (humanImageMessages.length) {
+        messages.push(HumanMessagePromptTemplate.fromTemplate([`{${inputKey}}`, ...humanImageMessages]))
+    }
+
     const chatPrompt = ChatPromptTemplate.fromMessages(messages)
 
     return chatPrompt
@@ -184,7 +217,7 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte
 
 const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
     const chatHistory = options.chatHistory
-    let model = nodeData.inputs?.model as ChatOpenAI
+    let model = nodeData.inputs?.model as BaseChatModel
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const memoryKey = memory.memoryKey ?? 'chat_history'