diff --git a/packages/components/src/IVisionChatModal.ts b/packages/components/src/IVisionChatModal.ts new file mode 100644 index 000000000..482ff70b2 --- /dev/null +++ b/packages/components/src/IVisionChatModal.ts @@ -0,0 +1,12 @@ +import { IMultiModalOption } from './Interface' + +export interface IVisionChatModal { + id: string + configuredModel: string + configuredMaxToken: number + multiModalOption: IMultiModalOption + + setVisionModel(): void + revertToOriginalModel(): void + setMultiModalOption(multiModalOption: IMultiModalOption): void +} diff --git a/packages/components/src/multiModalUtils.ts b/packages/components/src/multiModalUtils.ts index 00cc5bf33..186e85cae 100644 --- a/packages/components/src/multiModalUtils.ts +++ b/packages/components/src/multiModalUtils.ts @@ -1,8 +1,8 @@ import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' -import { ChatOpenAI } from '../nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI' import path from 'path' import { getStoragePath } from './utils' import fs from 'fs' +import { IVisionChatModal } from './IVisionChatModal' export const addImagesToMessages = ( nodeData: INodeData, @@ -12,7 +12,7 @@ export const addImagesToMessages = ( const imageContent: MessageContentImageUrl[] = [] let model = nodeData.inputs?.model - if (model instanceof ChatOpenAI && multiModalOption) { + if (llmSupportsVision(model) && multiModalOption) { // Image Uploaded if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) { const imageUploads = getImageUploads(options.uploads) @@ -46,3 +46,5 @@ export const getAudioUploads = (uploads: IFileUpload[]) => { export const getImageUploads = (uploads: IFileUpload[]) => { return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('image/')) } + +export const llmSupportsVision = (value: any): value is IVisionChatModal => !!value?.multiModalOption