diff --git a/packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts b/packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts index 4778b1b74..db834d4c2 100644 --- a/packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts +++ b/packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts @@ -208,6 +208,7 @@ class OpenAIAssistant_Agents implements INode { const usedTools: IUsedTool[] = [] const fileAnnotations = [] + const artifacts = [] const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({ id: selectedAssistantId @@ -439,21 +440,23 @@ class OpenAIAssistant_Agents implements INode { const fileId = chunk.image_file.file_id const fileObj = await openai.files.retrieve(fileId) - const buffer = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId) - const base64String = Buffer.from(buffer).toString('base64') - - // TODO: Use a file path and retrieve image on the fly. Storing as base64 to localStorage and database will easily hit limits - const imgHTML = `${fileObj.filename}
` - text += imgHTML + const filePath = await downloadImg( + openai, + fileId, + `${fileObj.filename}.png`, + options.chatflowid, + options.chatId + ) + artifacts.push({ type: 'png', data: filePath }) if (!isStreamingStarted) { isStreamingStarted = true if (sseStreamer) { - sseStreamer.streamStartEvent(chatId, imgHTML) + sseStreamer.streamStartEvent(chatId, ' ') } } if (sseStreamer) { - sseStreamer.streamTokenEvent(chatId, imgHTML) + sseStreamer.streamArtifactsEvent(chatId, artifacts) } } } @@ -565,6 +568,7 @@ class OpenAIAssistant_Agents implements INode { return { text, usedTools, + artifacts, fileAnnotations, assistant: { assistantId: openAIAssistantId, threadId, runId: runThreadId, messages: messageData } } @@ -769,12 +773,8 @@ class OpenAIAssistant_Agents implements INode { const fileId = content.image_file.file_id const fileObj = await openai.files.retrieve(fileId) - const buffer = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId) - const base64String = Buffer.from(buffer).toString('base64') - - // TODO: Use a file path and retrieve image on the fly. Storing as base64 to localStorage and database will easily hit limits - const imgHTML = `${fileObj.filename}
` - returnVal += imgHTML + const filePath = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId) + artifacts.push({ type: 'png', data: filePath }) } } @@ -787,6 +787,7 @@ class OpenAIAssistant_Agents implements INode { return { text: returnVal, usedTools, + artifacts, fileAnnotations, assistant: { assistantId: openAIAssistantId, threadId, runId: runThreadId, messages: messageData } } @@ -807,9 +808,9 @@ const downloadImg = async (openai: OpenAI, fileId: string, fileName: string, ... const image_data_buffer = Buffer.from(image_data) const mime = 'image/png' - await addSingleFileToStorage(mime, image_data_buffer, fileName, ...paths) + const res = await addSingleFileToStorage(mime, image_data_buffer, fileName, ...paths) - return image_data_buffer + return res } const downloadFile = async (openAIApiKey: string, fileObj: any, fileName: string, ...paths: string[]) => { diff --git a/packages/components/nodes/agents/ToolAgent/ToolAgent.ts b/packages/components/nodes/agents/ToolAgent/ToolAgent.ts index 012e18d6e..598ed17ea 100644 --- a/packages/components/nodes/agents/ToolAgent/ToolAgent.ts +++ b/packages/components/nodes/agents/ToolAgent/ToolAgent.ts @@ -134,6 +134,7 @@ class ToolAgent_Agents implements INode { let res: ChainValues = {} let sourceDocuments: ICommonObject[] = [] let usedTools: IUsedTool[] = [] + let artifacts = [] if (shouldStreamResponse) { const handler = new CustomChainHandler(sseStreamer, chatId) @@ -150,6 +151,12 @@ class ToolAgent_Agents implements INode { } usedTools = res.usedTools } + if (res.artifacts) { + if (sseStreamer) { + sseStreamer.streamArtifactsEvent(chatId, flatten(res.artifacts)) + } + artifacts = res.artifacts + } // If the tool is set to returnDirect, stream the output to the client if (res.usedTools && res.usedTools.length) { let inputTools = nodeData.inputs?.tools @@ -169,6 +176,9 @@ class ToolAgent_Agents implements INode { if (res.usedTools) { usedTools = res.usedTools } + if (res.artifacts) { + artifacts = res.artifacts + } } let output = res?.output @@ -203,7 +213,7 @@ class ToolAgent_Agents implements INode { let finalRes = output - if (sourceDocuments.length || usedTools.length) { + if (sourceDocuments.length || usedTools.length || artifacts.length) { const finalRes: ICommonObject = { text: output } if (sourceDocuments.length) { finalRes.sourceDocuments = flatten(sourceDocuments) @@ -211,6 +221,9 @@ class ToolAgent_Agents implements INode { if (usedTools.length) { finalRes.usedTools = usedTools } + if (artifacts.length) { + finalRes.artifacts = artifacts + } return finalRes } diff --git a/packages/components/nodes/sequentialagents/Agent/Agent.ts b/packages/components/nodes/sequentialagents/Agent/Agent.ts index e6b45490f..ff1525d1c 100644 --- a/packages/components/nodes/sequentialagents/Agent/Agent.ts +++ b/packages/components/nodes/sequentialagents/Agent/Agent.ts @@ -21,7 +21,7 @@ import { IDocument, IStateWithMessages } from '../../../src/Interface' -import { ToolCallingAgentOutputParser, AgentExecutor, SOURCE_DOCUMENTS_PREFIX } from '../../../src/agents' +import { ToolCallingAgentOutputParser, AgentExecutor, SOURCE_DOCUMENTS_PREFIX, ARTIFACTS_PREFIX } from '../../../src/agents' import { getInputVariables, getVars, handleEscapeCharacters, prepareSandboxVars } from '../../../src/utils' import { customGet, @@ -35,7 +35,6 @@ import { } from '../commonUtils' import { END, StateGraph } from '@langchain/langgraph' import { StructuredTool } from '@langchain/core/tools' -import { DynamicStructuredTool } from '../../tools/CustomTool/core' const defaultApprovalPrompt = `You are about to execute tool: {tools}. Ask if user want to proceed` const examplePrompt = 'You are a research assistant who can search for up-to-date info using search engine.' @@ -739,7 +738,12 @@ async function agentNode( // If the last message is a tool message and is an interrupted message, format output into standard agent output if (lastMessage._getType() === 'tool' && lastMessage.additional_kwargs?.nodeId === nodeData.id) { - let formattedAgentResult: { output?: string; usedTools?: IUsedTool[]; sourceDocuments?: IDocument[] } = {} + let formattedAgentResult: { + output?: string + usedTools?: IUsedTool[] + sourceDocuments?: IDocument[] + artifacts?: ICommonObject[] + } = {} formattedAgentResult.output = result.content if (lastMessage.additional_kwargs?.usedTools) { formattedAgentResult.usedTools = lastMessage.additional_kwargs.usedTools as IUsedTool[] @@ -747,6 +751,9 @@ async function agentNode( if (lastMessage.additional_kwargs?.sourceDocuments) { formattedAgentResult.sourceDocuments = lastMessage.additional_kwargs.sourceDocuments as IDocument[] } + if (lastMessage.additional_kwargs?.artifacts) { + formattedAgentResult.artifacts = lastMessage.additional_kwargs.artifacts as ICommonObject[] + } result = formattedAgentResult } else { result.name = name @@ -765,12 +772,18 @@ async function agentNode( if (result.sourceDocuments) { additional_kwargs.sourceDocuments = result.sourceDocuments } + if (result.artifacts) { + additional_kwargs.artifacts = result.artifacts + } if (result.output) { result.content = result.output delete result.output } - const outputContent = typeof result === 'string' ? result : result.content || result.output + let outputContent = typeof result === 'string' ? result : result.content || result.output + + // remove invalid markdown image pattern: ![]() + outputContent = typeof outputContent === 'string' ? outputContent.replace(/!\[.*?\]\(.*?\)/g, '') : outputContent if (nodeData.inputs?.updateStateMemoryUI || nodeData.inputs?.updateStateMemoryCode) { let formattedOutput = { @@ -931,6 +944,9 @@ class ToolNode extends RunnableCallable // Extract all properties except messages for IStateWithMessages const { messages: _, ...inputWithoutMessages } = Array.isArray(input) ? { messages: input } : input const ChannelsWithoutMessages = { + chatId: this.options.chatId, + sessionId: this.options.sessionId, + input: this.inputQuery, state: inputWithoutMessages } @@ -940,12 +956,14 @@ class ToolNode extends RunnableCallable if (tool === undefined) { throw new Error(`Tool ${call.name} not found.`) } - if (tool && tool instanceof DynamicStructuredTool) { + if (tool && (tool as any).setFlowObject) { // @ts-ignore tool.setFlowObject(ChannelsWithoutMessages) } let output = await tool.invoke(call.args, config) let sourceDocuments: Document[] = [] + let artifacts = [] + if (output?.includes(SOURCE_DOCUMENTS_PREFIX)) { const outputArray = output.split(SOURCE_DOCUMENTS_PREFIX) output = outputArray[0] @@ -956,12 +974,23 @@ class ToolNode extends RunnableCallable console.error('Error parsing source documents from tool') } } + if (output?.includes(ARTIFACTS_PREFIX)) { + const outputArray = output.split(ARTIFACTS_PREFIX) + output = outputArray[0] + try { + artifacts = JSON.parse(outputArray[1]) + } catch (e) { + console.error('Error parsing artifacts from tool') + } + } + return new ToolMessage({ name: tool.name, content: typeof output === 'string' ? output : JSON.stringify(output), tool_call_id: call.id!, additional_kwargs: { sourceDocuments, + artifacts, args: call.args, usedTools: [ { diff --git a/packages/components/nodes/sequentialagents/ToolNode/ToolNode.ts b/packages/components/nodes/sequentialagents/ToolNode/ToolNode.ts index 45ef49c1b..2c4daee48 100644 --- a/packages/components/nodes/sequentialagents/ToolNode/ToolNode.ts +++ b/packages/components/nodes/sequentialagents/ToolNode/ToolNode.ts @@ -12,13 +12,12 @@ import { import { AIMessage, AIMessageChunk, BaseMessage, ToolMessage } from '@langchain/core/messages' import { StructuredTool } from '@langchain/core/tools' import { RunnableConfig } from '@langchain/core/runnables' -import { SOURCE_DOCUMENTS_PREFIX } from '../../../src/agents' +import { ARTIFACTS_PREFIX, SOURCE_DOCUMENTS_PREFIX } from '../../../src/agents' import { Document } from '@langchain/core/documents' import { DataSource } from 'typeorm' import { MessagesState, RunnableCallable, customGet, getVM } from '../commonUtils' import { getVars, prepareSandboxVars } from '../../../src/utils' import { ChatPromptTemplate } from '@langchain/core/prompts' -import { DynamicStructuredTool } from '../../tools/CustomTool/core' const defaultApprovalPrompt = `You are about to execute tool: {tools}. Ask if user want to proceed` @@ -408,6 +407,9 @@ class ToolNode ext // Extract all properties except messages for IStateWithMessages const { messages: _, ...inputWithoutMessages } = Array.isArray(input) ? { messages: input } : input const ChannelsWithoutMessages = { + chatId: this.options.chatId, + sessionId: this.options.sessionId, + input: this.inputQuery, state: inputWithoutMessages } @@ -417,12 +419,13 @@ class ToolNode ext if (tool === undefined) { throw new Error(`Tool ${call.name} not found.`) } - if (tool && tool instanceof DynamicStructuredTool) { + if (tool && (tool as any).setFlowObject) { // @ts-ignore tool.setFlowObject(ChannelsWithoutMessages) } let output = await tool.invoke(call.args, config) let sourceDocuments: Document[] = [] + let artifacts = [] if (output?.includes(SOURCE_DOCUMENTS_PREFIX)) { const outputArray = output.split(SOURCE_DOCUMENTS_PREFIX) output = outputArray[0] @@ -433,12 +436,23 @@ class ToolNode ext console.error('Error parsing source documents from tool') } } + if (output?.includes(ARTIFACTS_PREFIX)) { + const outputArray = output.split(ARTIFACTS_PREFIX) + output = outputArray[0] + try { + artifacts = JSON.parse(outputArray[1]) + } catch (e) { + console.error('Error parsing artifacts from tool') + } + } + return new ToolMessage({ name: tool.name, content: typeof output === 'string' ? output : JSON.stringify(output), tool_call_id: call.id!, additional_kwargs: { sourceDocuments, + artifacts, args: call.args, usedTools: [ { @@ -489,7 +503,8 @@ const getReturnOutput = async ( tool: output.name, toolInput: output.additional_kwargs.args, toolOutput: output.content, - sourceDocuments: output.additional_kwargs.sourceDocuments + sourceDocuments: output.additional_kwargs.sourceDocuments, + artifacts: output.additional_kwargs.artifacts } as IUsedTool }) diff --git a/packages/components/nodes/tools/CodeInterpreterE2B/CodeInterpreterE2B.ts b/packages/components/nodes/tools/CodeInterpreterE2B/CodeInterpreterE2B.ts new file mode 100644 index 000000000..8c3967dc9 --- /dev/null +++ b/packages/components/nodes/tools/CodeInterpreterE2B/CodeInterpreterE2B.ts @@ -0,0 +1,269 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { StructuredTool, ToolInputParsingException, ToolParams } from '@langchain/core/tools' +import { CodeInterpreter } from '@e2b/code-interpreter' +import { z } from 'zod' +import { addSingleFileToStorage } from '../../../src/storageUtils' +import { CallbackManager, CallbackManagerForToolRun, Callbacks, parseCallbackConfigArg } from '@langchain/core/callbacks/manager' +import { RunnableConfig } from '@langchain/core/runnables' +import { ARTIFACTS_PREFIX } from '../../../src/agents' + +const DESC = `Evaluates python code in a sandbox environment. \ +The environment is long running and exists across multiple executions. \ +You must send the whole script every time and print your outputs. \ +Script should be pure python code that can be evaluated. \ +It should be in python format NOT markdown. \ +The code should NOT be wrapped in backticks. \ +All python packages including requests, matplotlib, scipy, numpy, pandas, \ +etc are available. Create and display chart using "plt.show()".` +const NAME = 'code_interpreter' + +class Code_Interpreter_Tools implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + badge: string + credential: INodeParams + + constructor() { + this.label = 'Code Interpreter by E2B' + this.name = 'codeInterpreterE2B' + this.version = 1.0 + this.type = 'CodeInterpreter' + this.icon = 'e2b.png' + this.category = 'Tools' + this.description = 'Execute code in a sandbox environment' + this.baseClasses = [this.type, 'Tool', ...getBaseClasses(E2BTool)] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['E2BApi'], + optional: true + } + this.inputs = [ + { + label: 'Tool Name', + name: 'toolName', + type: 'string', + description: 'Specify the name of the tool', + default: 'code_interpreter' + }, + { + label: 'Tool Description', + name: 'toolDesc', + type: 'string', + rows: 4, + description: 'Specify the description of the tool', + default: DESC + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const toolDesc = nodeData.inputs?.toolDesc as string + const toolName = nodeData.inputs?.toolName as string + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const e2bApiKey = getCredentialParam('e2bApiKey', credentialData, nodeData) + + return await E2BTool.initialize({ + description: toolDesc ?? DESC, + name: toolName ?? NAME, + apiKey: e2bApiKey, + schema: z.object({ + input: z.string().describe('Python code to be executed in the sandbox environment') + }), + chatflowid: options.chatflowid + }) + } +} + +type E2BToolParams = ToolParams +type E2BToolInput = { + name: string + description: string + apiKey: string + schema: any + chatflowid: string + templateCodeInterpreterE2B?: string + domainCodeInterpreterE2B?: string +} + +export class E2BTool extends StructuredTool { + static lc_name() { + return 'E2BTool' + } + + name = NAME + + description = DESC + + instance: CodeInterpreter + + apiKey: string + + schema + + chatflowid: string + + flowObj: ICommonObject + + templateCodeInterpreterE2B?: string + domainCodeInterpreterE2B?: string + + constructor(options: E2BToolParams & E2BToolInput) { + super(options) + this.description = options.description + this.name = options.name + this.apiKey = options.apiKey + this.schema = options.schema + this.chatflowid = options.chatflowid + this.templateCodeInterpreterE2B = options.templateCodeInterpreterE2B + this.domainCodeInterpreterE2B = options.domainCodeInterpreterE2B + } + + static async initialize(options: Partial & E2BToolInput) { + return new this({ + name: options.name, + description: options.description, + apiKey: options.apiKey, + schema: options.schema, + chatflowid: options.chatflowid, + templateCodeInterpreterE2B: options.templateCodeInterpreterE2B, + domainCodeInterpreterE2B: options.domainCodeInterpreterE2B + }) + } + + async call( + arg: z.infer, + configArg?: RunnableConfig | Callbacks, + tags?: string[], + flowConfig?: { sessionId?: string; chatId?: string; input?: string; state?: ICommonObject } + ): Promise { + const config = parseCallbackConfigArg(configArg) + if (config.runName === undefined) { + config.runName = this.name + } + let parsed + try { + parsed = await this.schema.parseAsync(arg) + } catch (e) { + throw new ToolInputParsingException(`Received tool input did not match expected schema`, JSON.stringify(arg)) + } + const callbackManager_ = await CallbackManager.configure( + config.callbacks, + this.callbacks, + config.tags || tags, + this.tags, + config.metadata, + this.metadata, + { verbose: this.verbose } + ) + const runManager = await callbackManager_?.handleToolStart( + this.toJSON(), + typeof parsed === 'string' ? parsed : JSON.stringify(parsed), + undefined, + undefined, + undefined, + undefined, + config.runName + ) + let result + try { + result = await this._call(parsed, runManager, flowConfig) + } catch (e) { + await runManager?.handleToolError(e) + throw e + } + if (result && typeof result !== 'string') { + result = JSON.stringify(result) + } + await runManager?.handleToolEnd(result) + return result + } + + // @ts-ignore + protected async _call( + arg: z.infer, + _?: CallbackManagerForToolRun, + flowConfig?: { sessionId?: string; chatId?: string; input?: string } + ): Promise { + flowConfig = { ...this.flowObj, ...flowConfig } + try { + if ('input' in arg) { + this.instance = await CodeInterpreter.create({ apiKey: this.apiKey }) + const execution = await this.instance.notebook.execCell(arg?.input) + + const artifacts = [] + for (const result of execution.results) { + for (const key in result) { + if (!(result as any)[key]) continue + + if (key === 'png') { + //@ts-ignore + const pngData = Buffer.from(result.png, 'base64') + + const filename = `artifact_${Date.now()}.png` + + const res = await addSingleFileToStorage( + 'image/png', + pngData, + filename, + this.chatflowid, + flowConfig!.chatId as string + ) + artifacts.push({ type: 'png', data: res }) + } else if (key === 'jpeg') { + //@ts-ignore + const jpegData = Buffer.from(result.jpeg, 'base64') + + const filename = `artifact_${Date.now()}.jpg` + + const res = await addSingleFileToStorage( + 'image/jpg', + jpegData, + filename, + this.chatflowid, + flowConfig!.chatId as string + ) + artifacts.push({ type: 'jpeg', data: res }) + } else if (key === 'html' || key === 'markdown' || key === 'latex' || key === 'json' || key === 'javascript') { + artifacts.push({ type: key, data: (result as any)[key] }) + } //TODO: support for pdf + } + } + + this.instance.close() + + let output = '' + + if (execution.text) output = execution.text + if (!execution.text && execution.logs.stdout.length) output = execution.logs.stdout.join('\n') + + if (execution.error) { + return `${execution.error.name}: ${execution.error.value}` + } + + return artifacts.length > 0 ? output + ARTIFACTS_PREFIX + JSON.stringify(artifacts) : output + } else { + return 'No input provided' + } + } catch (e) { + if (this.instance) this.instance.close() + return typeof e === 'string' ? e : JSON.stringify(e, null, 2) + } + } + + setFlowObject(flowObj: ICommonObject) { + this.flowObj = flowObj + } +} + +module.exports = { nodeClass: Code_Interpreter_Tools } diff --git a/packages/components/nodes/tools/E2B/e2b.png b/packages/components/nodes/tools/CodeInterpreterE2B/e2b.png similarity index 100% rename from packages/components/nodes/tools/E2B/e2b.png rename to packages/components/nodes/tools/CodeInterpreterE2B/e2b.png diff --git a/packages/components/nodes/tools/E2B/E2B.ts b/packages/components/nodes/tools/E2B/E2B.ts deleted file mode 100644 index 9954e4bd2..000000000 --- a/packages/components/nodes/tools/E2B/E2B.ts +++ /dev/null @@ -1,151 +0,0 @@ -/* -* TODO: Implement codeInterpreter column to chat_message table -import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' -import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' -import { StructuredTool, ToolParams } from '@langchain/core/tools' -import { CodeInterpreter } from '@e2b/code-interpreter' -import { z } from 'zod' - -const DESC = `Evaluates python code in a sandbox environment. \ -The environment is long running and exists across multiple executions. \ -You must send the whole script every time and print your outputs. \ -Script should be pure python code that can be evaluated. \ -It should be in python format NOT markdown. \ -The code should NOT be wrapped in backticks. \ -All python packages including requests, matplotlib, scipy, numpy, pandas, \ -etc are available. Create and display chart using "plt.show()".` -const NAME = 'code_interpreter' - -class E2B_Tools implements INode { - label: string - name: string - version: number - description: string - type: string - icon: string - category: string - baseClasses: string[] - inputs: INodeParams[] - badge: string - credential: INodeParams - - constructor() { - this.label = 'E2B' - this.name = 'e2b' - this.version = 1.0 - this.type = 'E2B' - this.icon = 'e2b.png' - this.category = 'Tools' - this.description = 'Execute code in E2B Code Intepreter' - this.baseClasses = [this.type, 'Tool', ...getBaseClasses(E2BTool)] - this.credential = { - label: 'Connect Credential', - name: 'credential', - type: 'credential', - credentialNames: ['E2BApi'] - } - this.inputs = [ - { - label: 'Tool Name', - name: 'toolName', - type: 'string', - description: 'Specify the name of the tool', - default: 'code_interpreter' - }, - { - label: 'Tool Description', - name: 'toolDesc', - type: 'string', - rows: 4, - description: 'Specify the description of the tool', - default: DESC - } - ] - } - - async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { - const toolDesc = nodeData.inputs?.toolDesc as string - const toolName = nodeData.inputs?.toolName as string - const credentialData = await getCredentialData(nodeData.credential ?? '', options) - const e2bApiKey = getCredentialParam('e2bApiKey', credentialData, nodeData) - const socketIO = options.socketIO - const socketIOClientId = options.socketIOClientId - - return await E2BTool.initialize({ - description: toolDesc ?? DESC, - name: toolName ?? NAME, - apiKey: e2bApiKey, - schema: z.object({ - input: z.string().describe('Python code to be executed in the sandbox environment') - }), - socketIO, - socketIOClientId - }) - } -} - -type E2BToolParams = ToolParams & { instance: CodeInterpreter } - -export class E2BTool extends StructuredTool { - static lc_name() { - return 'E2BTool' - } - - name = NAME - - description = DESC - - instance: CodeInterpreter - - apiKey: string - - schema - - socketIO - - socketIOClientId = '' - - constructor(options: E2BToolParams & { name: string; description: string, apiKey: string, schema: any, socketIO: any, socketIOClientId: string}) { - super(options) - this.instance = options.instance - this.description = options.description - this.name = options.name - this.apiKey = options.apiKey - this.schema = options.schema - this.returnDirect = true - this.socketIO = options.socketIO - this.socketIOClientId = options.socketIOClientId - } - - static async initialize(options: Partial & { name: string; description: string, apiKey: string, schema: any, socketIO: any, socketIOClientId: string }) { - const instance = await CodeInterpreter.create({ apiKey: options.apiKey }) - return new this({ instance, name: options.name, description: options.description, apiKey: options.apiKey, schema: options.schema, socketIO: options.socketIO, socketIOClientId: options.socketIOClientId}) - } - - async _call(args: any) { - try { - if ('input' in args) { - const execution = await this.instance.notebook.execCell(args?.input) - let imgHTML = '' - for (const result of execution.results) { - if (result.png) { - imgHTML += `\n\nimage
` - } - if (result.jpeg) { - imgHTML += `\n\nimage
` - } - } - const output = execution.text ? execution.text + imgHTML : imgHTML - if (this.socketIO && this.socketIOClientId) this.socketIO.to(this.socketIOClientId).emit('token', output) - return output - } else { - return 'No input provided' - } - } catch (e) { - return typeof e === 'string' ? e : JSON.stringify(e, null, 2) - } - } -} - -module.exports = { nodeClass: E2B_Tools } -*/ diff --git a/packages/components/nodes/tools/PythonInterpreter/PythonInterpreter.ts b/packages/components/nodes/tools/PythonInterpreter/PythonInterpreter.ts deleted file mode 100644 index fa577632e..000000000 --- a/packages/components/nodes/tools/PythonInterpreter/PythonInterpreter.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { INode, INodeData, INodeParams } from '../../../src/Interface' -import { getBaseClasses } from '../../../src/utils' -import { loadPyodide, type PyodideInterface } from 'pyodide' -import { Tool, ToolParams } from '@langchain/core/tools' -import * as path from 'path' -import { getUserHome } from '../../../src/utils' - -let pyodideInstance: PyodideInterface | undefined -const DESC = `Evaluates python code in a sandbox environment. The environment resets on every execution. You must send the whole script every time and print your outputs. Script should be pure python code that can be evaluated. Use only packages available in Pyodide.` -const NAME = 'python_interpreter' - -async function LoadPyodide(): Promise { - if (pyodideInstance === undefined) { - const obj = { packageCacheDir: path.join(getUserHome(), '.flowise', 'pyodideCacheDir') } - pyodideInstance = await loadPyodide(obj) - } - return pyodideInstance -} - -class PythonInterpreter_Tools implements INode { - label: string - name: string - version: number - description: string - type: string - icon: string - category: string - baseClasses: string[] - inputs: INodeParams[] - badge: string - - constructor() { - this.label = 'Python Interpreter' - this.name = 'pythonInterpreter' - this.version = 1.0 - this.type = 'PythonInterpreter' - this.icon = 'python.svg' - this.category = 'Tools' - this.description = 'Execute python code in Pyodide sandbox environment' - this.baseClasses = [this.type, 'Tool', ...getBaseClasses(PythonInterpreterTool)] - this.inputs = [ - { - label: 'Tool Name', - name: 'toolName', - type: 'string', - description: 'Specify the name of the tool', - default: 'python_interpreter' - }, - { - label: 'Tool Description', - name: 'toolDesc', - type: 'string', - rows: 4, - description: 'Specify the description of the tool', - default: DESC - } - ] - } - - async init(nodeData: INodeData): Promise { - const toolDesc = nodeData.inputs?.toolDesc as string - const toolName = nodeData.inputs?.toolName as string - - return await PythonInterpreterTool.initialize({ - description: toolDesc ?? DESC, - name: toolName ?? NAME - }) - } -} - -type PythonInterpreterToolParams = Parameters[0] & - ToolParams & { - instance: PyodideInterface - } - -export class PythonInterpreterTool extends Tool { - static lc_name() { - return 'PythonInterpreterTool' - } - - name = NAME - - description = DESC - - pyodideInstance: PyodideInterface - - stdout = '' - - stderr = '' - - constructor(options: PythonInterpreterToolParams & { name: string; description: string }) { - super(options) - this.description = options.description - this.name = options.name - this.pyodideInstance = options.instance - this.pyodideInstance.setStderr({ - batched: (text: string) => { - this.stderr += text - } - }) - this.pyodideInstance.setStdout({ - batched: (text: string) => { - this.stdout += text - } - }) - } - - static async initialize(options: Partial & { name: string; description: string }) { - const instance = await LoadPyodide() - return new this({ instance, name: options.name, description: options.description }) - } - - async _call(script: string) { - this.stdout = '' - this.stderr = '' - - try { - await this.pyodideInstance.loadPackagesFromImports(script) - await this.pyodideInstance.runPythonAsync(script) - return JSON.stringify({ stdout: this.stdout, stderr: this.stderr }, null, 2) - } catch (e) { - return typeof e === 'string' ? e : JSON.stringify(e, null, 2) - } - } -} - -module.exports = { nodeClass: PythonInterpreter_Tools } diff --git a/packages/components/nodes/tools/PythonInterpreter/python.svg b/packages/components/nodes/tools/PythonInterpreter/python.svg deleted file mode 100644 index 9cbbf9478..000000000 --- a/packages/components/nodes/tools/PythonInterpreter/python.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index ecbcb250e..f8058fe4f 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -415,7 +415,7 @@ export interface IServerSideEventStreamer { streamAgentReasoningEvent(chatId: string, data: any): void streamNextAgentEvent(chatId: string, data: any): void streamActionEvent(chatId: string, data: any): void - + streamArtifactsEvent(chatId: string, data: any): void streamAbortEvent(chatId: string): void streamEndEvent(chatId: string): void } diff --git a/packages/components/src/agents.ts b/packages/components/src/agents.ts index f464a6742..c8d473ac1 100644 --- a/packages/components/src/agents.ts +++ b/packages/components/src/agents.ts @@ -26,6 +26,8 @@ import { formatLogToString } from 'langchain/agents/format_scratchpad/log' import { IUsedTool } from './Interface' export const SOURCE_DOCUMENTS_PREFIX = '\n\n----FLOWISE_SOURCE_DOCUMENTS----\n\n' +export const ARTIFACTS_PREFIX = '\n\n----FLOWISE_ARTIFACTS----\n\n' + export type AgentFinish = { returnValues: Record log: string @@ -345,12 +347,14 @@ export class AgentExecutor extends BaseChain { let iterations = 0 let sourceDocuments: Array = [] const usedTools: IUsedTool[] = [] + let artifacts: any[] = [] const getOutput = async (finishStep: AgentFinish): Promise => { const { returnValues } = finishStep const additional = await this.agent.prepareForOutput(returnValues, steps) if (sourceDocuments.length) additional.sourceDocuments = flatten(sourceDocuments) if (usedTools.length) additional.usedTools = usedTools + if (artifacts.length) additional.artifacts = flatten(artifacts) if (this.returnIntermediateSteps) { return { ...returnValues, intermediateSteps: steps, ...additional } } @@ -426,13 +430,17 @@ export class AgentExecutor extends BaseChain { state: inputs } ) + let toolOutput = observation + if (typeof toolOutput === 'string' && toolOutput.includes(SOURCE_DOCUMENTS_PREFIX)) { + toolOutput = toolOutput.split(SOURCE_DOCUMENTS_PREFIX)[0] + } + if (typeof toolOutput === 'string' && toolOutput.includes(ARTIFACTS_PREFIX)) { + toolOutput = toolOutput.split(ARTIFACTS_PREFIX)[0] + } usedTools.push({ tool: tool.name, toolInput: action.toolInput as any, - toolOutput: - typeof observation === 'string' && observation.includes(SOURCE_DOCUMENTS_PREFIX) - ? observation.split(SOURCE_DOCUMENTS_PREFIX)[0] - : observation + toolOutput }) } else { observation = `${action.tool} is not a valid tool, try another one.` @@ -463,6 +471,16 @@ export class AgentExecutor extends BaseChain { console.error('Error parsing source documents from tool') } } + if (typeof observation === 'string' && observation.includes(ARTIFACTS_PREFIX)) { + const observationArray = observation.split(ARTIFACTS_PREFIX) + observation = observationArray[0] + try { + const artifact = JSON.parse(observationArray[1]) + artifacts.push(artifact) + } catch (e) { + console.error('Error parsing source documents from tool') + } + } return { action, observation: observation ?? '' } }) ) @@ -566,6 +584,10 @@ export class AgentExecutor extends BaseChain { const observationArray = observation.split(SOURCE_DOCUMENTS_PREFIX) observation = observationArray[0] } + if (typeof observation === 'string' && observation.includes(ARTIFACTS_PREFIX)) { + const observationArray = observation.split(ARTIFACTS_PREFIX) + observation = observationArray[0] + } } catch (e) { if (e instanceof ToolInputParsingException) { if (this.handleParsingErrors === true) { diff --git a/packages/server/src/Interface.ts b/packages/server/src/Interface.ts index f64cce360..a6bb205b7 100644 --- a/packages/server/src/Interface.ts +++ b/packages/server/src/Interface.ts @@ -42,6 +42,7 @@ export interface IChatMessage { fileAnnotations?: string agentReasoning?: string fileUploads?: string + artifacts?: string chatType: string chatId: string memoryType?: string diff --git a/packages/server/src/controllers/chat-messages/index.ts b/packages/server/src/controllers/chat-messages/index.ts index bc4b7003b..903a313bc 100644 --- a/packages/server/src/controllers/chat-messages/index.ts +++ b/packages/server/src/controllers/chat-messages/index.ts @@ -209,6 +209,9 @@ const parseAPIResponse = (apiResponse: ChatMessage | ChatMessage[]): ChatMessage if (parsedResponse.action) { parsedResponse.action = JSON.parse(parsedResponse.action) } + if (parsedResponse.artifacts) { + parsedResponse.artifacts = JSON.parse(parsedResponse.artifacts) + } return parsedResponse } diff --git a/packages/server/src/database/entities/ChatMessage.ts b/packages/server/src/database/entities/ChatMessage.ts index 6228b6699..ebd059f36 100644 --- a/packages/server/src/database/entities/ChatMessage.ts +++ b/packages/server/src/database/entities/ChatMessage.ts @@ -32,6 +32,9 @@ export class ChatMessage implements IChatMessage { @Column({ nullable: true, type: 'text' }) fileUploads?: string + @Column({ nullable: true, type: 'text' }) + artifacts?: string + @Column({ nullable: true, type: 'text' }) action?: string | null diff --git a/packages/server/src/database/migrations/sqlite/1726156258465-AddArtifactsToChatMessage.ts b/packages/server/src/database/migrations/sqlite/1726156258465-AddArtifactsToChatMessage.ts new file mode 100644 index 000000000..1056cb1ce --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1726156258465-AddArtifactsToChatMessage.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddArtifactsToChatMessage1726156258465 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN "artifacts" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "artifacts";`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index 0695ccfe1..33c04bdf7 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -23,6 +23,7 @@ import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-Add import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow' import { AddApiKey1720230151480 } from './1720230151480-AddApiKey' import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionToChatMessage' +import { AddArtifactsToChatMessage1726156258465 } from './1726156258465-AddArtifactsToChatMessage' import { AddCustomTemplate1725629836652 } from './1725629836652-AddCustomTemplate' export const sqliteMigrations = [ @@ -51,5 +52,6 @@ export const sqliteMigrations = [ AddVectorStoreConfigToDocStore1715861032479, AddApiKey1720230151480, AddActionToChatMessage1721078251523, + AddArtifactsToChatMessage1726156258465, AddCustomTemplate1725629836652 ] diff --git a/packages/server/src/utils/SSEStreamer.ts b/packages/server/src/utils/SSEStreamer.ts index 0e81b8380..4217925f2 100644 --- a/packages/server/src/utils/SSEStreamer.ts +++ b/packages/server/src/utils/SSEStreamer.ts @@ -97,6 +97,16 @@ export class SSEStreamer implements IServerSideEventStreamer { client.response.write('message:\ndata:' + JSON.stringify(clientResponse) + '\n\n') } } + streamArtifactsEvent(chatId: string, data: any) { + const client = this.clients[chatId] + if (client) { + const clientResponse = { + event: 'artifacts', + data: data + } + client.response.write('message:\ndata:' + JSON.stringify(clientResponse) + '\n\n') + } + } streamUsedToolsEvent(chatId: string, data: any): void { const client = this.clients[chatId] if (client) { diff --git a/packages/server/src/utils/buildAgentGraph.ts b/packages/server/src/utils/buildAgentGraph.ts index cd6fafe7a..ea44382bb 100644 --- a/packages/server/src/utils/buildAgentGraph.ts +++ b/packages/server/src/utils/buildAgentGraph.ts @@ -154,6 +154,7 @@ export const buildAgentGraph = async ( let finalAction: IAction = {} let totalSourceDocuments: IDocument[] = [] let totalUsedTools: IUsedTool[] = [] + let totalArtifacts: ICommonObject[] = [] const workerNodes = reactFlowNodes.filter((node) => node.data.name === 'worker') const supervisorNodes = reactFlowNodes.filter((node) => node.data.name === 'supervisor') @@ -221,6 +222,9 @@ export const buildAgentGraph = async ( const sourceDocuments = output[agentName]?.messages ? output[agentName].messages.map((msg: BaseMessage) => msg.additional_kwargs?.sourceDocuments) : [] + const artifacts = output[agentName]?.messages + ? output[agentName].messages.map((msg: BaseMessage) => msg.additional_kwargs?.artifacts) + : [] const messages = output[agentName]?.messages ? output[agentName].messages.map((msg: BaseMessage) => (typeof msg === 'string' ? msg : msg.content)) : [] @@ -240,6 +244,11 @@ export const buildAgentGraph = async ( if (cleanedDocs.length) totalSourceDocuments.push(...cleanedDocs) } + if (artifacts && artifacts.length) { + const cleanedArtifacts = artifacts.filter((artifact: ICommonObject) => artifact) + if (cleanedArtifacts.length) totalArtifacts.push(...cleanedArtifacts) + } + /* * Check if the next node is a condition node, if yes, then add the agent reasoning of the condition node */ @@ -273,6 +282,7 @@ export const buildAgentGraph = async ( instructions: output[agentName]?.instructions, usedTools: flatten(usedTools) as IUsedTool[], sourceDocuments: flatten(sourceDocuments) as Document[], + artifacts: flatten(artifacts) as ICommonObject[], state, nodeName: isSequential ? mapNameToLabel[agentName].nodeName : undefined, nodeId @@ -395,10 +405,12 @@ export const buildAgentGraph = async ( totalSourceDocuments = uniq(flatten(totalSourceDocuments)) totalUsedTools = uniq(flatten(totalUsedTools)) + totalArtifacts = uniq(flatten(totalArtifacts)) if (shouldStreamResponse && sseStreamer) { sseStreamer.streamUsedToolsEvent(chatId, totalUsedTools) sseStreamer.streamSourceDocumentsEvent(chatId, totalSourceDocuments) + sseStreamer.streamArtifactsEvent(chatId, totalArtifacts) sseStreamer.streamEndEvent(chatId) } @@ -406,6 +418,7 @@ export const buildAgentGraph = async ( finalResult, finalAction, sourceDocuments: totalSourceDocuments, + artifacts: totalArtifacts, usedTools: totalUsedTools, agentReasoning } diff --git a/packages/server/src/utils/buildChatflow.ts b/packages/server/src/utils/buildChatflow.ts index 7cd7b68f8..429492f2d 100644 --- a/packages/server/src/utils/buildChatflow.ts +++ b/packages/server/src/utils/buildChatflow.ts @@ -420,6 +420,8 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals if (result?.sourceDocuments) apiMessage.sourceDocuments = JSON.stringify(result.sourceDocuments) if (result?.usedTools) apiMessage.usedTools = JSON.stringify(result.usedTools) if (result?.fileAnnotations) apiMessage.fileAnnotations = JSON.stringify(result.fileAnnotations) + if (result?.artifacts) apiMessage.artifacts = JSON.stringify(result.artifacts) + const chatMessage = await utilAddChatMessage(apiMessage) logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`) @@ -481,7 +483,7 @@ const utilBuildAgentResponse = async ( shouldStreamResponse ) if (streamResults) { - const { finalResult, finalAction, sourceDocuments, usedTools, agentReasoning } = streamResults + const { finalResult, finalAction, sourceDocuments, artifacts, usedTools, agentReasoning } = streamResults const userMessage: Omit = { role: 'userMessage', content: incomingInput.question, @@ -506,6 +508,7 @@ const utilBuildAgentResponse = async ( sessionId } if (sourceDocuments?.length) apiMessage.sourceDocuments = JSON.stringify(sourceDocuments) + if (artifacts?.length) apiMessage.artifacts = JSON.stringify(artifacts) if (usedTools?.length) apiMessage.usedTools = JSON.stringify(usedTools) if (agentReasoning?.length) apiMessage.agentReasoning = JSON.stringify(agentReasoning) if (finalAction && Object.keys(finalAction).length) apiMessage.action = JSON.stringify(finalAction) diff --git a/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx b/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx index ef981cd92..5b40d1e7c 100644 --- a/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx +++ b/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx @@ -8,6 +8,7 @@ import rehypeRaw from 'rehype-raw' import remarkGfm from 'remark-gfm' import remarkMath from 'remark-math' import axios from 'axios' +import { cloneDeep } from 'lodash' // material-ui import { @@ -207,7 +208,16 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { if (chatmsg.fileAnnotations) msg.fileAnnotations = chatmsg.fileAnnotations if (chatmsg.feedback) msg.feedback = chatmsg.feedback?.content if (chatmsg.agentReasoning) msg.agentReasoning = chatmsg.agentReasoning - + if (chatmsg.artifacts) { + obj.artifacts = chatmsg.artifacts + obj.artifacts.forEach((artifact) => { + if (artifact.type === 'png' || artifact.type === 'jpeg') { + artifact.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatmsg.chatflowid}&chatId=${ + chatmsg.chatId + }&fileName=${artifact.data.replace('FILE-STORAGE::', '')}` + } + }) + } if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) { obj[chatPK] = { id: chatmsg.chatId, @@ -341,7 +351,16 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { if (chatmsg.usedTools) obj.usedTools = chatmsg.usedTools if (chatmsg.fileAnnotations) obj.fileAnnotations = chatmsg.fileAnnotations if (chatmsg.agentReasoning) obj.agentReasoning = chatmsg.agentReasoning - + if (chatmsg.artifacts) { + obj.artifacts = chatmsg.artifacts + obj.artifacts.forEach((artifact) => { + if (artifact.type === 'png' || artifact.type === 'jpeg') { + artifact.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatmsg.chatflowid}&chatId=${ + chatmsg.chatId + }&fileName=${artifact.data.replace('FILE-STORAGE::', '')}` + } + }) + } loadedMessages.push(obj) } setChatMessages(loadedMessages) @@ -574,6 +593,83 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [feedbackTypeFilter]) + const agentReasoningArtifacts = (artifacts) => { + const newArtifacts = cloneDeep(artifacts) + for (let i = 0; i < newArtifacts.length; i++) { + const artifact = newArtifacts[i] + if (artifact && (artifact.type === 'png' || artifact.type === 'jpeg')) { + const data = artifact.data + newArtifacts[i].data = `${baseURL}/api/v1/get-upload-file?chatflowId=${ + dialogProps.chatflow.id + }&chatId=${selectedChatId}&fileName=${data.replace('FILE-STORAGE::', '')}` + } + } + return newArtifacts + } + + const renderArtifacts = (item, index, isAgentReasoning) => { + if (item.type === 'png' || item.type === 'jpeg') { + return ( + + + + ) + } else if (item.type === 'html') { + return ( +
+
+
+ ) + } else { + return ( + + ) : ( + + {children} + + ) + } + }} + > + {item.data} + + ) + } + } + const component = show ? ( { width: '100%' }} > - {message.usedTools && ( -
- {message.usedTools.map((tool, index) => { - return ( - onSourceDialogClick(tool, 'Used Tools')} - /> - ) - })} -
- )} {message.fileUploads && message.fileUploads.length > 0 && (
{ />
)} + {agent.artifacts && ( +
+ {agentReasoningArtifacts( + agent.artifacts + ).map((item, index) => { + return item !== null ? ( + <> + {renderArtifacts( + item, + index, + true + )} + + ) : null + })} +
+ )} {agent.messages.length > 0 && ( { return !inline ? ( { })} )} + {message.usedTools && ( +
+ {message.usedTools.map((tool, index) => { + return ( + onSourceDialogClick(tool, 'Used Tools')} + /> + ) + })} +
+ )} + {message.artifacts && ( +
+ {message.artifacts.map((item, index) => { + return item !== null ? ( + <>{renderArtifacts(item, index)} + ) : null + })} +
+ )}
{/* Messages are being rendered in Markdown format */} { + artifacts.forEach((artifact) => { + if (artifact.type === 'png' || artifact.type === 'jpeg') { + artifact.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatflowid}&chatId=${chatId}&fileName=${artifact.data.replace( + 'FILE-STORAGE::', + '' + )}` + } + }) + setMessages((prevMessages) => { + let allMessages = [...cloneDeep(prevMessages)] + if (allMessages[allMessages.length - 1].type === 'userMessage') return allMessages + allMessages[allMessages.length - 1].artifacts = artifacts + return allMessages + }) + } + const updateLastMessageNextAgent = (nextAgent) => { setMessages((prevMessages) => { let allMessages = [...cloneDeep(prevMessages)] @@ -730,6 +747,7 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview fileAnnotations: data?.fileAnnotations, agentReasoning: data?.agentReasoning, action: data?.action, + artifacts: data?.artifacts, type: 'apiMessage', feedback: null } @@ -792,6 +810,9 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview case 'agentReasoning': updateLastMessageAgentReasoning(payload.data) break + case 'artifacts': + updateLastMessageArtifacts(payload.data) + break case 'action': updateLastMessageAction(payload.data) break @@ -913,6 +934,17 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview if (message.fileAnnotations) obj.fileAnnotations = message.fileAnnotations if (message.agentReasoning) obj.agentReasoning = message.agentReasoning if (message.action) obj.action = message.action + if (message.artifacts) { + obj.artifacts = message.artifacts + obj.artifacts.forEach((artifact) => { + if (artifact.type === 'png' || artifact.type === 'jpeg') { + artifact.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatflowid}&chatId=${chatId}&fileName=${artifact.data.replace( + 'FILE-STORAGE::', + '' + )}` + } + }) + } if (message.fileUploads) { obj.fileUploads = message.fileUploads obj.fileUploads.forEach((file) => { @@ -1260,6 +1292,84 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview } } + const agentReasoningArtifacts = (artifacts) => { + const newArtifacts = cloneDeep(artifacts) + for (let i = 0; i < newArtifacts.length; i++) { + const artifact = newArtifacts[i] + if (artifact && (artifact.type === 'png' || artifact.type === 'jpeg')) { + const data = artifact.data + newArtifacts[i].data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatflowid}&chatId=${chatId}&fileName=${data.replace( + 'FILE-STORAGE::', + '' + )}` + } + } + return newArtifacts + } + + const renderArtifacts = (item, index, isAgentReasoning) => { + if (item.type === 'png' || item.type === 'jpeg') { + return ( + + + + ) + } else if (item.type === 'html') { + return ( +
+
+
+ ) + } else { + return ( + + ) : ( + + {children} + + ) + } + }} + > + {item.data} + + ) + } + } + return (
{isDragActive && ( @@ -1459,6 +1569,23 @@ export const ChatMessage = ({ open, chatflowid, isAgentCanvas, isDialog, preview />
)} + {agent.artifacts && ( +
+ {agentReasoningArtifacts(agent.artifacts).map((item, index) => { + return item !== null ? ( + <>{renderArtifacts(item, index, true)} + ) : null + })} +
+ )} {agent.messages.length > 0 && ( )} + {message.artifacts && ( +
+ {message.artifacts.map((item, index) => { + return item !== null ? <>{renderArtifacts(item, index)} : null + })} +
+ )}
{message.type === 'leadCaptureMessage' && !getLocalStorageChatflow(chatflowid)?.lead &&