import { BaseMessage, MessageContentImageUrl, AIMessageChunk } from '@langchain/core/messages' import { getImageUploads } from '../../src/multiModalUtils' import { addSingleFileToStorage, getFileFromStorage } from '../../src/storageUtils' import { ICommonObject, IFileUpload, INodeData } from '../../src/Interface' import { BaseMessageLike } from '@langchain/core/messages' import { IFlowState } from './Interface.Agentflow' import { getCredentialData, getCredentialParam, handleEscapeCharacters, mapMimeTypeToInputField } from '../../src/utils' import fetch from 'node-fetch' export const addImagesToMessages = async ( options: ICommonObject, allowImageUploads: boolean, imageResolution?: 'auto' | 'low' | 'high' ): Promise => { const imageContent: MessageContentImageUrl[] = [] if (allowImageUploads && options?.uploads && options?.uploads.length > 0) { const imageUploads = getImageUploads(options.uploads) for (const upload of imageUploads) { let bf = upload.data if (upload.type == 'stored-file') { const fileName = upload.name.replace(/^FILE-STORAGE::/, '') const contents = await getFileFromStorage(fileName, options.orgId, options.chatflowid, options.chatId) // as the image is stored in the server, read the file and convert it to base64 bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64') imageContent.push({ type: 'image_url', image_url: { url: bf, detail: imageResolution ?? 'low' } }) } else if (upload.type == 'url' && bf) { imageContent.push({ type: 'image_url', image_url: { url: bf, detail: imageResolution ?? 'low' } }) } } } return imageContent } /** * Process message array to replace stored file references with base64 image data * @param messages Array of messages that may contain image references * @param options Common options object containing chatflowid and chatId * @returns Object containing updated messages array and transformed original messages */ export const processMessagesWithImages = async ( messages: BaseMessageLike[], options: ICommonObject ): Promise<{ updatedMessages: BaseMessageLike[] transformedMessages: BaseMessageLike[] }> => { if (!messages || !options.chatflowid || !options.chatId) { return { updatedMessages: messages, transformedMessages: [] } } // Create a deep copy of the messages to avoid mutating the original const updatedMessages = JSON.parse(JSON.stringify(messages)) // Track which messages were transformed const transformedMessages: BaseMessageLike[] = [] // Scan through all messages looking for stored-file references for (let i = 0; i < updatedMessages.length; i++) { const message = updatedMessages[i] // Skip non-user messages or messages without content if (message.role !== 'user' || !message.content) { continue } // Handle array content (typically containing file references) if (Array.isArray(message.content)) { const imageContents: MessageContentImageUrl[] = [] let hasImageReferences = false // Process each content item for (const item of message.content) { // Look for stored-file type items if (item.type === 'stored-file' && item.name && item.mime.startsWith('image/')) { hasImageReferences = true try { const fileName = item.name.replace(/^FILE-STORAGE::/, '') // Get file contents from storage const contents = await getFileFromStorage(fileName, options.orgId, options.chatflowid, options.chatId) // Create base64 data URL const base64Data = 'data:' + item.mime + ';base64,' + contents.toString('base64') // Add to image content array imageContents.push({ type: 'image_url', image_url: { url: base64Data, detail: item.imageResolution ?? 'low' } }) } catch (error) { console.error(`Failed to load image ${item.name}:`, error) } } } // Replace the content with the image content array if (imageContents.length > 0) { // Store the original message before modifying if (hasImageReferences) { transformedMessages.push(JSON.parse(JSON.stringify(messages[i]))) } updatedMessages[i].content = imageContents } } } return { updatedMessages, transformedMessages } } /** * Replace base64 image data in messages with file references * @param messages Array of messages that may contain base64 image data * @param uniqueImageMessages Array of messages with file references for new images * @param pastImageMessages Array of messages with file references for previous images * @returns Updated messages array with file references instead of base64 data */ export const replaceBase64ImagesWithFileReferences = ( messages: BaseMessageLike[], uniqueImageMessages: BaseMessageLike[] = [], pastImageMessages: BaseMessageLike[] = [] ): BaseMessageLike[] => { // Create a deep copy to avoid mutating the original const updatedMessages = JSON.parse(JSON.stringify(messages)) // Track positions in replacement arrays let pastMessageIndex = 0 let pastContentIndex = 0 let uniqueMessageIndex = 0 let uniqueContentIndex = 0 for (let i = 0; i < updatedMessages.length; i++) { const message = updatedMessages[i] if (message.content && Array.isArray(message.content)) { for (let j = 0; j < message.content.length; j++) { const item = message.content[j] if (item.type === 'image_url') { // Try past images first let replacement = null if (pastMessageIndex < pastImageMessages.length) { const pastMessage = pastImageMessages[pastMessageIndex] as BaseMessage | undefined if (pastMessage && Array.isArray(pastMessage.content)) { if (pastContentIndex < pastMessage.content.length) { replacement = pastMessage.content[pastContentIndex] pastContentIndex++ // Move to next message if we've used all content in current one if (pastContentIndex >= pastMessage.content.length) { pastMessageIndex++ pastContentIndex = 0 } } else { // Current message has no more content, move to next pastMessageIndex++ pastContentIndex = 0 // Try again with the next message if (pastMessageIndex < pastImageMessages.length) { const nextPastMessage = pastImageMessages[pastMessageIndex] as BaseMessage | undefined if (nextPastMessage && Array.isArray(nextPastMessage.content) && nextPastMessage.content.length > 0) { replacement = nextPastMessage.content[0] pastContentIndex = 1 } } } } } // Try unique images if no past image replacement found if (!replacement && uniqueMessageIndex < uniqueImageMessages.length) { const uniqueMessage = uniqueImageMessages[uniqueMessageIndex] as BaseMessage | undefined if (uniqueMessage && Array.isArray(uniqueMessage.content)) { if (uniqueContentIndex < uniqueMessage.content.length) { replacement = uniqueMessage.content[uniqueContentIndex] uniqueContentIndex++ // Move to next message if we've used all content in current one if (uniqueContentIndex >= uniqueMessage.content.length) { uniqueMessageIndex++ uniqueContentIndex = 0 } } else { // Current message has no more content, move to next uniqueMessageIndex++ uniqueContentIndex = 0 // Try again with the next message if (uniqueMessageIndex < uniqueImageMessages.length) { const nextUniqueMessage = uniqueImageMessages[uniqueMessageIndex] as BaseMessage | undefined if ( nextUniqueMessage && Array.isArray(nextUniqueMessage.content) && nextUniqueMessage.content.length > 0 ) { replacement = nextUniqueMessage.content[0] uniqueContentIndex = 1 } } } } } // Apply replacement if found if (replacement) { message.content[j] = { ...replacement } } } } } } return updatedMessages } /** * Get unique image messages from uploads * @param options Common options object containing uploads * @param messages Array of messages to check for existing images * @param modelConfig Model configuration object containing allowImageUploads and imageResolution * @returns Object containing imageMessageWithFileRef and imageMessageWithBase64 */ export const getUniqueImageMessages = async ( options: ICommonObject, messages: BaseMessageLike[], modelConfig?: ICommonObject ): Promise<{ imageMessageWithFileRef: BaseMessageLike; imageMessageWithBase64: BaseMessageLike } | undefined> => { if (!options.uploads) return undefined // Get images from uploads const images = await addImagesToMessages(options, modelConfig?.allowImageUploads, modelConfig?.imageResolution) // Filter out images that are already in previous messages const uniqueImages = images.filter((image) => { // Check if this image is already in any existing message return !messages.some((msg: any) => { // For multimodal content (arrays with image objects) if (Array.isArray(msg.content)) { return msg.content.some( (item: any) => // Compare by image URL/content for image objects item.type === 'image_url' && image.type === 'image_url' && JSON.stringify(item) === JSON.stringify(image) ) } // For direct comparison of simple content return JSON.stringify(msg.content) === JSON.stringify(image) }) }) if (uniqueImages.length === 0) { return undefined } // Create messages with the original file references for storage/display const imageMessageWithFileRef = { role: 'user', content: options.uploads.map((upload: IFileUpload) => ({ type: upload.type, name: upload.name, mime: upload.mime, imageResolution: modelConfig?.imageResolution })) } // Create messages with base64 data for the LLM const imageMessageWithBase64 = { role: 'user', content: uniqueImages } return { imageMessageWithFileRef, imageMessageWithBase64 } } /** * Get past chat history image messages * @param pastChatHistory Array of past chat history messages * @param options Common options object * @returns Object containing updatedPastMessages and transformedPastMessages */ export const getPastChatHistoryImageMessages = async ( pastChatHistory: BaseMessageLike[], options: ICommonObject ): Promise<{ updatedPastMessages: BaseMessageLike[]; transformedPastMessages: BaseMessageLike[] }> => { const chatHistory = [] const transformedPastMessages = [] for (let i = 0; i < pastChatHistory.length; i++) { const message = pastChatHistory[i] as BaseMessage & { role: string } const messageRole = message.role || 'user' if (message.additional_kwargs && message.additional_kwargs.fileUploads) { // example: [{"type":"stored-file","name":"0_DiXc4ZklSTo3M8J4.jpg","mime":"image/jpeg"}] const fileUploads = message.additional_kwargs.fileUploads const artifacts = message.additional_kwargs.artifacts const fileAnnotations = message.additional_kwargs.fileAnnotations const usedTools = message.additional_kwargs.usedTools try { let messageWithFileUploads = '' const uploads: IFileUpload[] = typeof fileUploads === 'string' ? JSON.parse(fileUploads) : fileUploads const imageContents: MessageContentImageUrl[] = [] for (const upload of uploads) { if (upload.type === 'stored-file' && upload.mime.startsWith('image/')) { const fileName = upload.name.replace(/^FILE-STORAGE::/, '') const fileData = await getFileFromStorage(fileName, options.orgId, options.chatflowid, options.chatId) // as the image is stored in the server, read the file and convert it to base64 const bf = 'data:' + upload.mime + ';base64,' + fileData.toString('base64') imageContents.push({ type: 'image_url', image_url: { url: bf } }) } else if (upload.type === 'url' && upload.mime.startsWith('image') && upload.data) { imageContents.push({ type: 'image_url', image_url: { url: upload.data } }) } else if (upload.type === 'stored-file:full') { const fileLoaderNodeModule = await import('../../nodes/documentloaders/File/File') // @ts-ignore const fileLoaderNodeInstance = new fileLoaderNodeModule.nodeClass() const nodeOptions = { retrieveAttachmentChatId: true, chatflowid: options.chatflowid, chatId: options.chatId, orgId: options.orgId } let fileInputFieldFromMimeType = 'txtFile' fileInputFieldFromMimeType = mapMimeTypeToInputField(upload.mime) const nodeData = { inputs: { [fileInputFieldFromMimeType]: `FILE-STORAGE::${JSON.stringify([upload.name])}` } } const documents: string = await fileLoaderNodeInstance.init(nodeData, '', nodeOptions) messageWithFileUploads += `${handleEscapeCharacters(documents, true)}\n\n` } } const messageContent = messageWithFileUploads ? `${messageWithFileUploads}\n\n${message.content}` : message.content const hasArtifacts = artifacts && Array.isArray(artifacts) && artifacts.length > 0 const hasFileAnnotations = fileAnnotations && Array.isArray(fileAnnotations) && fileAnnotations.length > 0 const hasUsedTools = usedTools && Array.isArray(usedTools) && usedTools.length > 0 if (imageContents.length > 0) { const imageMessage: any = { role: messageRole, content: imageContents } if (hasArtifacts || hasFileAnnotations || hasUsedTools) { imageMessage.additional_kwargs = {} if (hasArtifacts) imageMessage.additional_kwargs.artifacts = artifacts if (hasFileAnnotations) imageMessage.additional_kwargs.fileAnnotations = fileAnnotations if (hasUsedTools) imageMessage.additional_kwargs.usedTools = usedTools } chatHistory.push(imageMessage) transformedPastMessages.push({ role: messageRole, content: [...JSON.parse((pastChatHistory[i] as any).additional_kwargs.fileUploads)] }) } const contentMessage: any = { role: messageRole, content: messageContent } if (hasArtifacts || hasFileAnnotations || hasUsedTools) { contentMessage.additional_kwargs = {} if (hasArtifacts) contentMessage.additional_kwargs.artifacts = artifacts if (hasFileAnnotations) contentMessage.additional_kwargs.fileAnnotations = fileAnnotations if (hasUsedTools) contentMessage.additional_kwargs.usedTools = usedTools } chatHistory.push(contentMessage) } catch (e) { // failed to parse fileUploads, continue with text only const hasArtifacts = artifacts && Array.isArray(artifacts) && artifacts.length > 0 const hasFileAnnotations = fileAnnotations && Array.isArray(fileAnnotations) && fileAnnotations.length > 0 const hasUsedTools = usedTools && Array.isArray(usedTools) && usedTools.length > 0 const errorMessage: any = { role: messageRole, content: message.content } if (hasArtifacts || hasFileAnnotations || hasUsedTools) { errorMessage.additional_kwargs = {} if (hasArtifacts) errorMessage.additional_kwargs.artifacts = artifacts if (hasFileAnnotations) errorMessage.additional_kwargs.fileAnnotations = fileAnnotations if (hasUsedTools) errorMessage.additional_kwargs.usedTools = usedTools } chatHistory.push(errorMessage) } } else if (message.additional_kwargs) { const hasArtifacts = message.additional_kwargs.artifacts && Array.isArray(message.additional_kwargs.artifacts) && message.additional_kwargs.artifacts.length > 0 const hasFileAnnotations = message.additional_kwargs.fileAnnotations && Array.isArray(message.additional_kwargs.fileAnnotations) && message.additional_kwargs.fileAnnotations.length > 0 const hasUsedTools = message.additional_kwargs.usedTools && Array.isArray(message.additional_kwargs.usedTools) && message.additional_kwargs.usedTools.length > 0 if (hasArtifacts || hasFileAnnotations || hasUsedTools) { const messageAdditionalKwargs: any = {} if (hasArtifacts) messageAdditionalKwargs.artifacts = message.additional_kwargs.artifacts if (hasFileAnnotations) messageAdditionalKwargs.fileAnnotations = message.additional_kwargs.fileAnnotations if (hasUsedTools) messageAdditionalKwargs.usedTools = message.additional_kwargs.usedTools chatHistory.push({ role: messageRole, content: message.content, additional_kwargs: messageAdditionalKwargs }) } else { chatHistory.push({ role: messageRole, content: message.content }) } } else { chatHistory.push({ role: messageRole, content: message.content }) } } return { updatedPastMessages: chatHistory, transformedPastMessages } } /** * Gets MIME type from filename extension */ export const getMimeTypeFromFilename = (filename: string): string => { const extension = filename.toLowerCase().split('.').pop() const mimeTypes: { [key: string]: string } = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', pdf: 'application/pdf', txt: 'text/plain', csv: 'text/csv', json: 'application/json', html: 'text/html', xml: 'application/xml' } return mimeTypes[extension || ''] || 'application/octet-stream' } /** * Gets artifact type from filename extension for UI rendering */ export const getArtifactTypeFromFilename = (filename: string): string => { const extension = filename.toLowerCase().split('.').pop() const artifactTypes: { [key: string]: string } = { png: 'png', jpg: 'jpeg', jpeg: 'jpeg', html: 'html', htm: 'html', md: 'markdown', markdown: 'markdown', json: 'json', js: 'javascript', javascript: 'javascript', tex: 'latex', latex: 'latex', txt: 'text', csv: 'text', pdf: 'text' } return artifactTypes[extension || ''] || 'text' } /** * Saves base64 image data to storage and returns file information */ export const saveBase64Image = async ( outputItem: any, options: ICommonObject ): Promise<{ filePath: string; fileName: string; totalSize: number } | null> => { try { if (!outputItem.result) { return null } // Extract base64 data and create buffer const base64Data = outputItem.result const imageBuffer = Buffer.from(base64Data, 'base64') // Determine file extension and MIME type const outputFormat = outputItem.output_format || 'png' const fileName = `generated_image_${outputItem.id || Date.now()}.${outputFormat}` const mimeType = outputFormat === 'png' ? 'image/png' : 'image/jpeg' // Save the image using the existing storage utility const { path, totalSize } = await addSingleFileToStorage( mimeType, imageBuffer, fileName, options.orgId, options.chatflowid, options.chatId ) return { filePath: path, fileName, totalSize } } catch (error) { console.error('Error saving base64 image:', error) return null } } /** * Saves Gemini inline image data to storage and returns file information */ export const saveGeminiInlineImage = async ( inlineItem: any, options: ICommonObject ): Promise<{ filePath: string; fileName: string; totalSize: number } | null> => { try { if (!inlineItem.data || !inlineItem.mimeType) { return null } // Extract base64 data and create buffer const base64Data = inlineItem.data const imageBuffer = Buffer.from(base64Data, 'base64') // Determine file extension from MIME type const mimeType = inlineItem.mimeType let extension = 'png' if (mimeType.includes('jpeg') || mimeType.includes('jpg')) { extension = 'jpg' } else if (mimeType.includes('png')) { extension = 'png' } else if (mimeType.includes('gif')) { extension = 'gif' } else if (mimeType.includes('webp')) { extension = 'webp' } const fileName = `gemini_generated_image_${Date.now()}.${extension}` // Save the image using the existing storage utility const { path, totalSize } = await addSingleFileToStorage( mimeType, imageBuffer, fileName, options.orgId, options.chatflowid, options.chatId ) return { filePath: path, fileName, totalSize } } catch (error) { console.error('Error saving Gemini inline image:', error) return null } } /** * Downloads file content from container file citation */ export const downloadContainerFile = async ( containerId: string, fileId: string, filename: string, modelNodeData: INodeData, options: ICommonObject ): Promise<{ filePath: string; totalSize: number } | null> => { try { const credentialData = await getCredentialData(modelNodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, modelNodeData) if (!openAIApiKey) { console.warn('No OpenAI API key available for downloading container file') return null } // Download the file using OpenAI Container API const response = await fetch(`https://api.openai.com/v1/containers/${containerId}/files/${fileId}/content`, { method: 'GET', headers: { Accept: '*/*', Authorization: `Bearer ${openAIApiKey}` } }) if (!response.ok) { console.warn( `Failed to download container file ${fileId} from container ${containerId}: ${response.status} ${response.statusText}` ) return null } // Extract the binary data from the Response object const data = await response.arrayBuffer() const dataBuffer = Buffer.from(data) const mimeType = getMimeTypeFromFilename(filename) // Store the file using the same storage utility as OpenAIAssistant const { path, totalSize } = await addSingleFileToStorage( mimeType, dataBuffer, filename, options.orgId, options.chatflowid, options.chatId ) return { filePath: path, totalSize } } catch (error) { console.error('Error downloading container file:', error) return null } } /** * Replace inlineData base64 with file references in the response content */ export const replaceInlineDataWithFileReferences = ( response: AIMessageChunk, savedInlineImages: Array<{ filePath: string; fileName: string; mimeType: string }> ): void => { // Check if content is an array if (!Array.isArray(response.content)) { return } // Replace base64 data with file references in response content let savedImageIndex = 0 for (let i = 0; i < response.content.length; i++) { const contentItem = response.content[i] if ( typeof contentItem === 'object' && contentItem.type === 'inlineData' && contentItem.inlineData && savedImageIndex < savedInlineImages.length ) { const savedImage = savedInlineImages[savedImageIndex] // Replace with file reference response.content[i] = { type: 'stored-file', name: savedImage.fileName, mime: savedImage.mimeType, path: savedImage.filePath } savedImageIndex++ } } // Clear the inlineData from response_metadata to avoid duplication if (response.response_metadata?.inlineData) { delete response.response_metadata.inlineData } } /** * Extracts artifacts from response metadata (both annotations and built-in tools) */ export const extractArtifactsFromResponse = async ( responseMetadata: any, modelNodeData: INodeData, options: ICommonObject ): Promise<{ artifacts: any[] fileAnnotations: any[] savedInlineImages?: Array<{ filePath: string; fileName: string; mimeType: string }> }> => { const artifacts: any[] = [] const fileAnnotations: any[] = [] const savedInlineImages: Array<{ filePath: string; fileName: string; mimeType: string }> = [] // Handle Gemini inline data (image generation) if (responseMetadata?.inlineData && Array.isArray(responseMetadata.inlineData)) { for (const inlineItem of responseMetadata.inlineData) { if (inlineItem.type === 'gemini_inline_data' && inlineItem.data && inlineItem.mimeType) { try { const savedImageResult = await saveGeminiInlineImage(inlineItem, options) if (savedImageResult) { // Create artifact in the same format as other image artifacts const fileType = getArtifactTypeFromFilename(savedImageResult.fileName) artifacts.push({ type: fileType, data: savedImageResult.filePath }) // Track saved image for replacing base64 data in content savedInlineImages.push({ filePath: savedImageResult.filePath, fileName: savedImageResult.fileName, mimeType: inlineItem.mimeType }) } } catch (error) { console.error('Error processing Gemini inline image artifact:', error) } } } } if (!responseMetadata?.output || !Array.isArray(responseMetadata.output)) { return { artifacts, fileAnnotations, savedInlineImages: savedInlineImages.length > 0 ? savedInlineImages : undefined } } for (const outputItem of responseMetadata.output) { // Handle container file citations from annotations if (outputItem.type === 'message' && outputItem.content && Array.isArray(outputItem.content)) { for (const contentItem of outputItem.content) { if (contentItem.annotations && Array.isArray(contentItem.annotations)) { for (const annotation of contentItem.annotations) { if (annotation.type === 'container_file_citation' && annotation.file_id && annotation.filename) { try { // Download and store the file content const downloadResult = await downloadContainerFile( annotation.container_id, annotation.file_id, annotation.filename, modelNodeData, options ) if (downloadResult) { const fileType = getArtifactTypeFromFilename(annotation.filename) if (fileType === 'png' || fileType === 'jpeg' || fileType === 'jpg') { const artifact = { type: fileType, data: downloadResult.filePath } artifacts.push(artifact) } else { fileAnnotations.push({ filePath: downloadResult.filePath, fileName: annotation.filename }) } } } catch (error) { console.error('Error processing annotation:', error) } } } } } } // Handle built-in tool artifacts (like image generation) if (outputItem.type === 'image_generation_call' && outputItem.result) { try { const savedImageResult = await saveBase64Image(outputItem, options) if (savedImageResult) { // Replace the base64 result with the file path in the response metadata outputItem.result = savedImageResult.filePath // Create artifact in the same format as other image artifacts const fileType = getArtifactTypeFromFilename(savedImageResult.fileName) artifacts.push({ type: fileType, data: savedImageResult.filePath }) } } catch (error) { console.error('Error processing image generation artifact:', error) } } } return { artifacts, fileAnnotations, savedInlineImages: savedInlineImages.length > 0 ? savedInlineImages : undefined } } /** * Add image artifacts from previous assistant messages as user messages * This allows the LLM to see and reference the generated images in the conversation * Messages are marked with a special flag for later removal */ export const addImageArtifactsToMessages = async (messages: BaseMessageLike[], options: ICommonObject): Promise => { const imageExtensions = ['png', 'jpg', 'jpeg', 'gif', 'webp'] const messagesToInsert: Array<{ index: number; message: any }> = [] // Iterate through messages to find assistant messages with image artifacts for (let i = 0; i < messages.length; i++) { const message = messages[i] as any // Check if this is an assistant message with artifacts if ( (message.role === 'assistant' || message.role === 'ai') && message.additional_kwargs?.artifacts && Array.isArray(message.additional_kwargs.artifacts) ) { const artifacts = message.additional_kwargs.artifacts const imageArtifacts: Array<{ type: string; name: string; mime: string }> = [] // Extract image artifacts for (const artifact of artifacts) { if (artifact.type && artifact.data) { // Check if this is an image artifact by file type if (imageExtensions.includes(artifact.type.toLowerCase())) { // Extract filename from the file path const fileName = artifact.data.split('/').pop() || artifact.data const mimeType = `image/${artifact.type.toLowerCase()}` imageArtifacts.push({ type: 'stored-file', name: fileName, mime: mimeType }) } } } // If we found image artifacts, prepare to insert a user message after this assistant message if (imageArtifacts.length > 0) { // Check if the next message already contains these image artifacts to avoid duplicates const nextMessage = messages[i + 1] as any const shouldInsert = !nextMessage || nextMessage.role !== 'user' || !Array.isArray(nextMessage.content) || !nextMessage.content.some( (item: any) => (item.type === 'stored-file' || item.type === 'image_url') && imageArtifacts.some((artifact) => { // Compare with and without FILE-STORAGE:: prefix const artifactName = artifact.name.replace('FILE-STORAGE::', '') const itemName = item.name?.replace('FILE-STORAGE::', '') || '' return artifactName === itemName }) ) if (shouldInsert) { messagesToInsert.push({ index: i + 1, message: { role: 'user', content: imageArtifacts, _isTemporaryImageMessage: true // Mark for later removal } }) } } } } // Insert messages in reverse order to maintain correct indices for (let i = messagesToInsert.length - 1; i >= 0; i--) { const { index, message } = messagesToInsert[i] messages.splice(index, 0, message) } // Convert stored-file references to base64 image_url format if (messagesToInsert.length > 0) { const { updatedMessages } = await processMessagesWithImages(messages, options) // Replace the messages array content with the updated messages messages.length = 0 messages.push(...updatedMessages) } } /** * Updates the flow state with new values */ export const updateFlowState = (state: ICommonObject, updateState: IFlowState[]): ICommonObject => { let newFlowState: Record = {} for (const state of updateState) { newFlowState[state.key] = state.value } return { ...state, ...newFlowState } }