feat: Add image upload support to ChatOpenRouter node (#5471)
* feat: Add image upload support to ChatOpenRouter node - Create FlowiseChatOpenRouter wrapper class implementing IVisionChatModal - Add allowImageUploads and imageResolution input fields - Support multimodal image inputs for OpenRouter models - Follows same pattern as ChatOpenAI implementation Resolves #5143 * lint fix --------- Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
parent
03ef28afbc
commit
4d79653741
|
|
@ -1,7 +1,8 @@
|
||||||
import { ChatOpenAI, ChatOpenAIFields } from '@langchain/openai'
|
import { ChatOpenAI as LangchainChatOpenAI, ChatOpenAIFields } from '@langchain/openai'
|
||||||
import { BaseCache } from '@langchain/core/caches'
|
import { BaseCache } from '@langchain/core/caches'
|
||||||
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||||
|
import { ChatOpenRouter } from './FlowiseChatOpenRouter'
|
||||||
|
|
||||||
class ChatOpenRouter_ChatModels implements INode {
|
class ChatOpenRouter_ChatModels implements INode {
|
||||||
label: string
|
label: string
|
||||||
|
|
@ -23,7 +24,7 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||||
this.icon = 'openRouter.svg'
|
this.icon = 'openRouter.svg'
|
||||||
this.category = 'Chat Models'
|
this.category = 'Chat Models'
|
||||||
this.description = 'Wrapper around Open Router Inference API'
|
this.description = 'Wrapper around Open Router Inference API'
|
||||||
this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]
|
this.baseClasses = [this.type, ...getBaseClasses(LangchainChatOpenAI)]
|
||||||
this.credential = {
|
this.credential = {
|
||||||
label: 'Connect Credential',
|
label: 'Connect Credential',
|
||||||
name: 'credential',
|
name: 'credential',
|
||||||
|
|
@ -114,6 +115,40 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||||
type: 'json',
|
type: 'json',
|
||||||
optional: true,
|
optional: true,
|
||||||
additionalParams: true
|
additionalParams: true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Allow Image Uploads',
|
||||||
|
name: 'allowImageUploads',
|
||||||
|
type: 'boolean',
|
||||||
|
description:
|
||||||
|
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||||
|
default: false,
|
||||||
|
optional: true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Image Resolution',
|
||||||
|
description: 'This parameter controls the resolution in which the model views the image.',
|
||||||
|
name: 'imageResolution',
|
||||||
|
type: 'options',
|
||||||
|
options: [
|
||||||
|
{
|
||||||
|
label: 'Low',
|
||||||
|
name: 'low'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'High',
|
||||||
|
name: 'high'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Auto',
|
||||||
|
name: 'auto'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
default: 'low',
|
||||||
|
optional: false,
|
||||||
|
show: {
|
||||||
|
allowImageUploads: true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
@ -130,6 +165,8 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||||
const basePath = (nodeData.inputs?.basepath as string) || 'https://openrouter.ai/api/v1'
|
const basePath = (nodeData.inputs?.basepath as string) || 'https://openrouter.ai/api/v1'
|
||||||
const baseOptions = nodeData.inputs?.baseOptions
|
const baseOptions = nodeData.inputs?.baseOptions
|
||||||
const cache = nodeData.inputs?.cache as BaseCache
|
const cache = nodeData.inputs?.cache as BaseCache
|
||||||
|
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
|
||||||
|
const imageResolution = nodeData.inputs?.imageResolution as string
|
||||||
|
|
||||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||||
const openRouterApiKey = getCredentialParam('openRouterApiKey', credentialData, nodeData)
|
const openRouterApiKey = getCredentialParam('openRouterApiKey', credentialData, nodeData)
|
||||||
|
|
@ -155,7 +192,7 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||||
try {
|
try {
|
||||||
parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
|
parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
|
||||||
} catch (exception) {
|
} catch (exception) {
|
||||||
throw new Error("Invalid JSON in the ChatCerebras's BaseOptions: " + exception)
|
throw new Error("Invalid JSON in the ChatOpenRouter's BaseOptions: " + exception)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -166,7 +203,15 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const model = new ChatOpenAI(obj)
|
const multiModalOption: IMultiModalOption = {
|
||||||
|
image: {
|
||||||
|
allowImageUploads: allowImageUploads ?? false,
|
||||||
|
imageResolution
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const model = new ChatOpenRouter(nodeData.id, obj)
|
||||||
|
model.setMultiModalOption(multiModalOption)
|
||||||
return model
|
return model
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
import { ChatOpenAI as LangchainChatOpenAI, ChatOpenAIFields } from '@langchain/openai'
|
||||||
|
import { IMultiModalOption, IVisionChatModal } from '../../../src'
|
||||||
|
|
||||||
|
export class ChatOpenRouter extends LangchainChatOpenAI implements IVisionChatModal {
|
||||||
|
configuredModel: string
|
||||||
|
configuredMaxToken?: number
|
||||||
|
multiModalOption: IMultiModalOption
|
||||||
|
id: string
|
||||||
|
|
||||||
|
constructor(id: string, fields?: ChatOpenAIFields) {
|
||||||
|
super(fields)
|
||||||
|
this.id = id
|
||||||
|
this.configuredModel = fields?.modelName ?? ''
|
||||||
|
this.configuredMaxToken = fields?.maxTokens
|
||||||
|
}
|
||||||
|
|
||||||
|
revertToOriginalModel(): void {
|
||||||
|
this.model = this.configuredModel
|
||||||
|
this.maxTokens = this.configuredMaxToken
|
||||||
|
}
|
||||||
|
|
||||||
|
setMultiModalOption(multiModalOption: IMultiModalOption): void {
|
||||||
|
this.multiModalOption = multiModalOption
|
||||||
|
}
|
||||||
|
|
||||||
|
setVisionModel(): void {
|
||||||
|
// pass - OpenRouter models don't need model switching
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue