Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1970 from vinodkiran/FEATURE/AzureOpenAI_Image 

Feature/azure open ai image
This commit is contained in:
Henry Heng 2024-03-16 16:48:56 +08:00 committed by GitHub
parent 3fbfd3d425 0c649c9ce3
commit 58122e985c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 54 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts
View File

@ -1,8 +1,9 @@
import { AzureOpenAIInput, ChatOpenAI, OpenAIChatInput } from '@langchain/openai' import { AzureOpenAIInput, ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from '@langchain/openai'
import { BaseCache } from '@langchain/core/caches' import { BaseCache } from '@langchain/core/caches'
import { BaseLLMParams } from '@langchain/core/language_models/llms' import { BaseLLMParams } from '@langchain/core/language_models/llms'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { ChatOpenAI } from '../ChatOpenAI/FlowiseChatOpenAI'
class AzureChatOpenAI_ChatModels implements INode { class AzureChatOpenAI_ChatModels implements INode {
label: string label: string
@ -19,12 +20,12 @@ class AzureChatOpenAI_ChatModels implements INode {
constructor() { constructor() {
this.label = 'Azure ChatOpenAI' this.label = 'Azure ChatOpenAI'
this.name = 'azureChatOpenAI' this.name = 'azureChatOpenAI'
this.version = 2.0 this.version = 3.0
this.type = 'AzureChatOpenAI' this.type = 'AzureChatOpenAI'
this.icon = 'Azure.svg' this.icon = 'Azure.svg'
this.category = 'Chat Models' this.category = 'Chat Models'
this.description = 'Wrapper around Azure OpenAI large language models that use the Chat endpoint' this.description = 'Wrapper around Azure OpenAI large language models that use the Chat endpoint'
this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)] this.baseClasses = [this.type, ...getBaseClasses(LangchainChatOpenAI)]
this.credential = { this.credential = {
label: 'Connect Credential', label: 'Connect Credential',
name: 'credential', name: 'credential',
@ -58,6 +59,10 @@ class AzureChatOpenAI_ChatModels implements INode {
{ {
label: 'gpt-35-turbo-16k', label: 'gpt-35-turbo-16k',
name: 'gpt-35-turbo-16k' name: 'gpt-35-turbo-16k'
},
{
label: 'gpt-4-vision-preview',
name: 'gpt-4-vision-preview'
} }
], ],
default: 'gpt-35-turbo', default: 'gpt-35-turbo',
@ -102,6 +107,38 @@ class AzureChatOpenAI_ChatModels implements INode {
step: 1, step: 1,
optional: true, optional: true,
additionalParams: true additionalParams: true
},
{
label: 'Allow Image Uploads',
name: 'allowImageUploads',
type: 'boolean',
description:
'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent',
default: false,
optional: true
},
{
label: 'Image Resolution',
description: 'This parameter controls the resolution in which the model views the image.',
name: 'imageResolution',
type: 'options',
options: [
{
label: 'Low',
name: 'low'
},
{
label: 'High',
name: 'high'
},
{
label: 'Auto',
name: 'auto'
}
],
default: 'low',
optional: false,
additionalParams: true
} }
] ]
} }
@ -122,6 +159,9 @@ class AzureChatOpenAI_ChatModels implements INode {
const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData)
const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData)
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
const imageResolution = nodeData.inputs?.imageResolution as string
const obj: Partial<AzureOpenAIInput> & BaseLLMParams & Partial<OpenAIChatInput> = { const obj: Partial<AzureOpenAIInput> & BaseLLMParams & Partial<OpenAIChatInput> = {
temperature: parseFloat(temperature), temperature: parseFloat(temperature),
modelName, modelName,
@ -138,7 +178,15 @@ class AzureChatOpenAI_ChatModels implements INode {
if (timeout) obj.timeout = parseInt(timeout, 10) if (timeout) obj.timeout = parseInt(timeout, 10)
if (cache) obj.cache = cache if (cache) obj.cache = cache
const model = new ChatOpenAI(obj) const multiModalOption: IMultiModalOption = {
image: {
allowImageUploads: allowImageUploads ?? false,
imageResolution
}
}
const model = new ChatOpenAI(nodeData.id, obj)
model.setMultiModalOption(multiModalOption)
return model return model
} }
} }

2
packages/server/src/index.ts
View File

@ -1675,7 +1675,7 @@ export class App {
if (!chatflow) return `Chatflow ${chatflowid} not found` if (!chatflow) return `Chatflow ${chatflowid} not found`
const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent'] const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent']
const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic', 'awsChatBedrock'] const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic', 'awsChatBedrock', 'azureChatOpenAI']
const flowObj = JSON.parse(chatflow.flowData) const flowObj = JSON.parse(chatflow.flowData)
const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = [] const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = []