From bce7ff9ada6db56bfda729fdbc446d335d6d8537 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Fri, 8 Mar 2024 17:59:54 +0530 Subject: [PATCH] refactoring of interface into the common interface.ts file and misc changes --- .../ConversationalAgent/ConversationalAgent.ts | 9 +-------- .../nodes/agents/MRKLAgentChat/MRKLAgentChat.ts | 3 +-- .../chains/ConversationChain/ConversationChain.ts | 11 +++++++++-- .../components/nodes/chains/LLMChain/LLMChain.ts | 3 +-- .../chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts | 3 +-- .../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts | 3 +-- packages/components/src/IVisionChatModal.ts | 12 ------------ packages/components/src/Interface.ts | 11 +++++++++++ packages/components/src/multiModalUtils.ts | 3 +-- 9 files changed, 26 insertions(+), 32 deletions(-) delete mode 100644 packages/components/src/IVisionChatModal.ts diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts index 6bb042b62..14361e43f 100644 --- a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts +++ b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts @@ -9,10 +9,9 @@ import { RunnableSequence } from '@langchain/core/runnables' import { ChatConversationalAgent } from 'langchain/agents' import { getBaseClasses } from '../../../src/utils' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' -import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' +import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { AgentExecutor } from '../../../src/agents' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' -import { IVisionChatModal } from '../../../src/IVisionChatModal' const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI. @@ -158,10 +157,6 @@ const prepareAgent = async ( if (messageContent?.length) { visionChatModel.setVisionModel() - // for (const msg of messageContent) { - // humanImageMessages.push(new HumanMessage({ content: [msg] })) - // } - // Pop the `agent_scratchpad` MessagePlaceHolder let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder if (prompt.promptMessages.at(-1) instanceof HumanMessagePromptTemplate) { @@ -176,8 +171,6 @@ const prepareAgent = async ( msg.inputVariables = lastMessage.inputVariables prompt.promptMessages.push(msg) } - // Add the HumanMessage for images - //prompt.promptMessages.push(...humanImageMessages) // Add the `agent_scratchpad` MessagePlaceHolder back prompt.promptMessages.push(messagePlaceholder) diff --git a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts index e1f16fd8a..f0466e08c 100644 --- a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts +++ b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts @@ -6,11 +6,10 @@ import type { PromptTemplate } from '@langchain/core/prompts' import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { pull } from 'langchain/hub' import { additionalCallbacks } from '../../../src/handler' -import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' +import { IVisionChatModal, FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { createReactAgent } from '../../../src/agents' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' -import { IVisionChatModal } from '../../../src/IVisionChatModal' class MRKLAgentChat_Agents implements INode { label: string diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index 814fc1cc1..ffea16f5b 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -7,10 +7,17 @@ import { checkInputs, Moderation, streamResponse } from '../../moderation/Modera import { formatResponse } from '../../outputparsers/OutputParserHelpers' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' -import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams, MessageContentImageUrl } from '../../../src/Interface' +import { + IVisionChatModal, + FlowiseMemory, + ICommonObject, + INode, + INodeData, + INodeParams, + MessageContentImageUrl +} from '../../../src/Interface' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' -import { IVisionChatModal } from '../../../src/IVisionChatModal' let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.` const inputKey = 'input' diff --git a/packages/components/nodes/chains/LLMChain/LLMChain.ts b/packages/components/nodes/chains/LLMChain/LLMChain.ts index 738a8c4b5..dc59760c0 100644 --- a/packages/components/nodes/chains/LLMChain/LLMChain.ts +++ b/packages/components/nodes/chains/LLMChain/LLMChain.ts @@ -4,14 +4,13 @@ import { HumanMessage } from '@langchain/core/messages' import { ChatPromptTemplate, FewShotPromptTemplate, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts' import { OutputFixingParser } from 'langchain/output_parsers' import { LLMChain } from 'langchain/chains' -import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { IVisionChatModal, ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { additionalCallbacks, ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler' import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers' import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils' -import { IVisionChatModal } from '../../../src/IVisionChatModal' class LLMChain_Chains implements INode { label: string diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts index d52e9900a..05665d1f6 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts @@ -1,6 +1,5 @@ import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic' -import { IMultiModalOption } from '../../../src' -import { IVisionChatModal } from '../../../src/IVisionChatModal' +import { IVisionChatModal, IMultiModalOption } from '../../../src' import { BaseLLMParams } from '@langchain/core/language_models/llms' export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChatModal { diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts index b00811d55..0227362cc 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts @@ -1,8 +1,7 @@ import type { ClientOptions } from 'openai' import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, LegacyOpenAIInput, AzureOpenAIInput } from '@langchain/openai' import { BaseChatModelParams } from '@langchain/core/language_models/chat_models' -import { IMultiModalOption } from '../../../src' -import { IVisionChatModal } from '../../../src/IVisionChatModal' +import { IMultiModalOption, IVisionChatModal } from '../../../src' export class ChatOpenAI extends LangchainChatOpenAI implements IVisionChatModal { configuredModel: string diff --git a/packages/components/src/IVisionChatModal.ts b/packages/components/src/IVisionChatModal.ts deleted file mode 100644 index 482ff70b2..000000000 --- a/packages/components/src/IVisionChatModal.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { IMultiModalOption } from './Interface' - -export interface IVisionChatModal { - id: string - configuredModel: string - configuredMaxToken: number - multiModalOption: IMultiModalOption - - setVisionModel(): void - revertToOriginalModel(): void - setMultiModalOption(multiModalOption: IMultiModalOption): void -} diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 0e280deae..798e91a2d 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -270,3 +270,14 @@ export abstract class FlowiseSummaryMemory extends ConversationSummaryMemory imp abstract addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId?: string): Promise abstract clearChatMessages(overrideSessionId?: string): Promise } + +export interface IVisionChatModal { + id: string + configuredModel: string + configuredMaxToken: number + multiModalOption: IMultiModalOption + + setVisionModel(): void + revertToOriginalModel(): void + setMultiModalOption(multiModalOption: IMultiModalOption): void +} diff --git a/packages/components/src/multiModalUtils.ts b/packages/components/src/multiModalUtils.ts index 186e85cae..b3a512102 100644 --- a/packages/components/src/multiModalUtils.ts +++ b/packages/components/src/multiModalUtils.ts @@ -1,8 +1,7 @@ -import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' +import { IVisionChatModal, ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' import path from 'path' import { getStoragePath } from './utils' import fs from 'fs' -import { IVisionChatModal } from './IVisionChatModal' export const addImagesToMessages = ( nodeData: INodeData,