Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Multimodal Fixes for cyclic (circular) dependencies during langsmith analysis...

This commit is contained in:
vinodkiran 2024-02-16 08:18:58 -05:00
parent 52ffa1772b
commit 10fc1bf08d
2 changed files with 35 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts
View File

@ -39,10 +39,9 @@ export class ChatOpenAI extends LangchainChatOpenAI {
return super.generate(messages, options, callbacks) return super.generate(messages, options, callbacks)
} }
private async injectMultiModalMessages(messages: BaseMessageLike[][], nodeOptions: MultiModalOptions) { private async injectMultiModalMessages(messages: BaseMessageLike[][], options: MultiModalOptions) {
const nodeData = nodeOptions.nodeData const optionsData = options.nodeOptions
const optionsData = nodeOptions.nodeOptions const messageContent = addImagesToMessages(optionsData, this.multiModalOption)
const messageContent = addImagesToMessages(nodeData, optionsData, this.multiModalOption)
if (messageContent?.length) { if (messageContent?.length) {
if (messages[0].length > 0 && messages[0][messages[0].length - 1] instanceof HumanMessage) { if (messages[0].length > 0 && messages[0][messages[0].length - 1] instanceof HumanMessage) {
// Change model to gpt-4-vision // Change model to gpt-4-vision

40
packages/components/src/multiModalUtils.ts
View File

@ -1,5 +1,4 @@
import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface'
import { ChatOpenAI as LangchainChatOpenAI } from 'langchain/chat_models/openai'
import path from 'path' import path from 'path'
import { getStoragePath } from './utils' import { getStoragePath } from './utils'
import fs from 'fs' import fs from 'fs'
@ -10,13 +9,12 @@ import { AgentExecutor as LcAgentExecutor, ChatAgent, RunnableAgent } from 'lang
import { AgentExecutor } from './agents' import { AgentExecutor } from './agents'
export interface MultiModalOptions { export interface MultiModalOptions {
nodeData: INodeData
nodeOptions: ICommonObject nodeOptions: ICommonObject
} }
export const injectLLMChainNodeData = (nodeData: INodeData, options: ICommonObject) => { export const injectLLMChainNodeData = (nodeData: INodeData, options: ICommonObject) => {
let llmChain = nodeData.instance as LLMChain let llmChain = nodeData.instance as LLMChain
;(llmChain.llm as ChatOpenAI).lc_kwargs.chainData = { nodeData: nodeData, nodeOptions: options } ;(llmChain.llm as ChatOpenAI).lc_kwargs.chainData = { nodeOptions: getUploadsFromOptions(options) }
} }
export const injectAgentExecutorNodeData = (agentExecutor: AgentExecutor, nodeData: INodeData, options: ICommonObject) => { export const injectAgentExecutorNodeData = (agentExecutor: AgentExecutor, nodeData: INodeData, options: ICommonObject) => {
@ -29,59 +27,61 @@ export const injectAgentExecutorNodeData = (agentExecutor: AgentExecutor, nodeDa
export const injectLcAgentExecutorNodeData = (agentExecutor: LcAgentExecutor, nodeData: INodeData, options: ICommonObject) => { export const injectLcAgentExecutorNodeData = (agentExecutor: LcAgentExecutor, nodeData: INodeData, options: ICommonObject) => {
if (agentExecutor.agent instanceof ChatAgent) { if (agentExecutor.agent instanceof ChatAgent) {
let llmChain = agentExecutor.agent.llmChain as LLMChain let llmChain = agentExecutor.agent.llmChain as LLMChain
;(llmChain.llm as ChatOpenAI).lc_kwargs.chainData = { nodeData: nodeData, nodeOptions: options } ;(llmChain.llm as ChatOpenAI).lc_kwargs.chainData = { nodeOptions: getUploadsFromOptions(options) }
} }
} }
export const injectRunnableNodeData = (runnableSequence: RunnableSequence, nodeData: INodeData, options: ICommonObject) => { export const injectRunnableNodeData = (runnableSequence: RunnableSequence, nodeData: INodeData, options: ICommonObject) => {
runnableSequence.steps.forEach((step) => { runnableSequence.steps.forEach((step) => {
if (step instanceof ChatOpenAI) { if (step instanceof ChatOpenAI) {
;(step as ChatOpenAI).lc_kwargs.chainData = { nodeData: nodeData, nodeOptions: options } ;(step as ChatOpenAI).lc_kwargs.chainData = { nodeOptions: getUploadsFromOptions(options) }
} }
if (step instanceof RunnableBinding) { if (step instanceof RunnableBinding) {
if ((step as RunnableBinding<any, any>).bound instanceof ChatOpenAI) { if ((step as RunnableBinding<any, any>).bound instanceof ChatOpenAI) {
;((step as RunnableBinding<any, any>).bound as ChatOpenAI).lc_kwargs.chainData = { ;((step as RunnableBinding<any, any>).bound as ChatOpenAI).lc_kwargs.chainData = {
nodeData: nodeData, nodeOptions: getUploadsFromOptions(options)
nodeOptions: options
} }
} }
} }
}) })
} }
export const addImagesToMessages = ( const getUploadsFromOptions = (options: ICommonObject): ICommonObject => {
nodeData: INodeData, if (options?.uploads) {
options: ICommonObject, return {
multiModalOption?: IMultiModalOption uploads: options.uploads,
): MessageContentImageUrl[] => { chatflowid: options.chatflowid,
const imageContent: MessageContentImageUrl[] = [] chatId: options.chatId
let model = nodeData.inputs?.model }
}
return {}
}
export const addImagesToMessages = (options: ICommonObject, multiModalOption?: IMultiModalOption): MessageContentImageUrl[] => {
const imageContent: MessageContentImageUrl[] = []
if (model instanceof LangchainChatOpenAI && multiModalOption) {
// Image Uploaded // Image Uploaded
if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) { if (multiModalOption?.image && multiModalOption?.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) {
const imageUploads = getImageUploads(options.uploads) const imageUploads = getImageUploads(options.uploads)
for (const upload of imageUploads) { for (const upload of imageUploads) {
let bf = upload.data
if (upload.type == 'stored-file') { if (upload.type == 'stored-file') {
const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name) const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name)
// as the image is stored in the server, read the file and convert it to base64 // as the image is stored in the server, read the file and convert it to base64
const contents = fs.readFileSync(filePath) const contents = fs.readFileSync(filePath)
bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64') let bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64')
imageContent.push({ imageContent.push({
type: 'image_url', type: 'image_url',
image_url: { image_url: {
url: bf, url: bf,
detail: multiModalOption.image.imageResolution ?? 'low' detail: multiModalOption?.image.imageResolution ?? 'low'
} }
}) })
} }
} }
} }
}
return imageContent return imageContent
} }