Evaluations for Agentflows v2 & Assistants (#4589)

* New Feature: Evaluations for AgentFlow v2

* New Feature: Evaluations for Assistants and minor tweaks on other evaluations.

* do not store messages during evaluation for agent flows.

* common cost formatting

* moving the category names to description (in create dialog) and adjusting the side drawer label

* lint fixes

* Enhancement: Add auto-refresh toggle for evaluations with 5-second interval and adjust grid item size for metrics display.

* 1) chatflow types are stored in additional config
2) messages are now stored with type "Evaluations"
3) Message Dialog has a new Type in the ChatType Filter Dropdown
4) Chatflow badges on the view page, have the right canvas URL
5) outdated API returns chatflow type along with the stale indicator.
6) UI - Flow Indicator Icons are shown in the Chatflows Used chips & side drawer

* Refactor JWT error handling to return 401 status for expired refresh tokens. Update chat message ID assignment to remove UUID fallback. Enhance ViewMessagesDialog to set default chat type filters and implement a new method for determining chat type sources. Modify EvalsResultDialog to open links in a new tab and adjust icon sizes for better consistency. Clean up unused imports in EvaluationResultSideDrawer.

* handling on Click for deleted flows and minor code cleanup

* evals ui fix

* Refactor evaluation service to improve error handling and data parsing. Update additionalConfig handling to default to an empty object if not present. Enhance type definitions for better clarity. Adjust MetricsItemCard to prevent overflow and improve layout consistency.

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/components/evaluation/EvaluationRunner.ts

@@ -6,6 +6,26 @@ import { getModelConfigByModelName, MODEL_TYPE } from '../src/modelLoader'
 
 export class EvaluationRunner {
     static metrics = new Map<string, string[]>()
+
+    static getCostMetrics = async (selectedProvider: string, selectedModel: string) => {
+        let modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, selectedProvider, selectedModel)
+        if (modelConfig) {
+            if (modelConfig['cost_values']) {
+                return modelConfig.cost_values
+            }
+            return { cost_values: modelConfig }
+        } else {
+            modelConfig = await getModelConfigByModelName(MODEL_TYPE.LLM, selectedProvider, selectedModel)
+            if (modelConfig) {
+                if (modelConfig['cost_values']) {
+                    return modelConfig.cost_values
+                }
+                return { cost_values: modelConfig }
+            }
+        }
+        return undefined
+    }
+
     static async getAndDeleteMetrics(id: string) {
         const val = EvaluationRunner.metrics.get(id)
         if (val) {
@@ -34,11 +54,8 @@ export class EvaluationRunner {
                         }
                     }
                 }
-                let modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, selectedProvider, selectedModel)
+                if (selectedProvider && selectedModel) {
-                if (modelConfig) {
+                    const modelConfig = await EvaluationRunner.getCostMetrics(selectedProvider, selectedModel)
-                    val.push(JSON.stringify({ cost_values: modelConfig }))
-                } else {
-                    modelConfig = await getModelConfigByModelName(MODEL_TYPE.LLM, selectedProvider, selectedModel)
                     if (modelConfig) {
                         val.push(JSON.stringify({ cost_values: modelConfig }))
                     }
@@ -116,6 +133,40 @@ export class EvaluationRunner {
             }
             try {
                 let response = await axios.post(`${this.baseURL}/api/v1/prediction/${chatflowId}`, postData, axiosConfig)
+                let agentFlowMetrics: any[] = []
+                if (response?.data?.agentFlowExecutedData) {
+                    for (let i = 0; i < response.data.agentFlowExecutedData.length; i++) {
+                        const agentFlowExecutedData = response.data.agentFlowExecutedData[i]
+                        const input_tokens = agentFlowExecutedData?.data?.output?.usageMetadata?.input_tokens || 0
+                        const output_tokens = agentFlowExecutedData?.data?.output?.usageMetadata?.output_tokens || 0
+                        const total_tokens =
+                            agentFlowExecutedData?.data?.output?.usageMetadata?.total_tokens || input_tokens + output_tokens
+                        const metrics: any = {
+                            promptTokens: input_tokens,
+                            completionTokens: output_tokens,
+                            totalTokens: total_tokens,
+                            provider:
+                                agentFlowExecutedData.data?.input?.llmModelConfig?.llmModel ||
+                                agentFlowExecutedData.data?.input?.agentModelConfig?.agentModel,
+                            model:
+                                agentFlowExecutedData.data?.input?.llmModelConfig?.modelName ||
+                                agentFlowExecutedData.data?.input?.agentModelConfig?.modelName,
+                            nodeLabel: agentFlowExecutedData?.nodeLabel,
+                            nodeId: agentFlowExecutedData?.nodeId
+                        }
+                        if (metrics.provider && metrics.model) {
+                            const modelConfig = await EvaluationRunner.getCostMetrics(metrics.provider, metrics.model)
+                            if (modelConfig) {
+                                metrics.cost_values = {
+                                    input_cost: (modelConfig.cost_values.input_cost || 0) * (input_tokens / 1000),
+                                    output_cost: (modelConfig.cost_values.output_cost || 0) * (output_tokens / 1000)
+                                }
+                                metrics.cost_values.total_cost = metrics.cost_values.input_cost + metrics.cost_values.output_cost
+                            }
+                        }
+                        agentFlowMetrics.push(metrics)
+                    }
+                }
                 const endTime = performance.now()
                 const timeTaken = (endTime - startTime).toFixed(2)
                 if (response?.data?.metrics) {
@@ -130,6 +181,9 @@ export class EvaluationRunner {
                         }
                     ]
                 }
+                if (agentFlowMetrics.length > 0) {
+                    runData.nested_metrics = agentFlowMetrics
+                }
                 runData.status = 'complete'
                 let resultText = ''
                 if (response.data.text) resultText = response.data.text

packages/server/src/enterprise/middleware/passport/index.ts

@@ -218,7 +218,7 @@ export const initializeJwtCookieMiddleware = async (app: express.Application, id
         if (!refreshToken) return res.sendStatus(401)
 
         jwt.verify(refreshToken, jwtRefreshSecret, async (err: any, payload: any) => {
-            if (err || !payload) return res.status(403).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED })
+            if (err || !payload) return res.status(401).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED })
             // @ts-ignore
             const loggedInUser = req.user as LoggedInUser
             let isSSO = false
@@ -227,16 +227,16 @@ export const initializeJwtCookieMiddleware = async (app: express.Application, id
                 try {
                     newTokenResponse = await identityManager.getRefreshToken(loggedInUser.ssoProvider, loggedInUser.ssoRefreshToken)
                     if (newTokenResponse.error) {
-                        return res.status(403).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED })
+                        return res.status(401).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED })
                     }
                     isSSO = true
                 } catch (error) {
-                    return res.status(403).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED })
+                    return res.status(401).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED })
                 }
             }
             const meta = decryptToken(payload.meta)
             if (!meta) {
-                return res.status(403).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED })
+                return res.status(401).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED })
             }
             if (isSSO) {
                 loggedInUser.ssoToken = newTokenResponse.access_token

packages/server/src/services/evaluations/CostCalculator.ts

@@ -18,39 +18,29 @@ export const calculateCost = (metricsArray: ICommonObject[]) => {
         let completionTokensCost: string = '0'
         let totalTokensCost = '0'
         if (metric.cost_values) {
-            const costValues = metric.cost_values
+            let costValues: any = {}
+            if (metric.cost_values?.cost_values) {
+                costValues = metric.cost_values.cost_values
+            } else {
+                costValues = metric.cost_values
+            }
+
             if (costValues.total_price > 0) {
                 let cost = costValues.total_cost * (totalTokens / 1000)
-                if (cost < 0.01) {
+                totalTokensCost = formatCost(cost)
-                    totalTokensCost = '$ <0.01'
-                } else {
-                    totalTokensCost = '$ ' + cost.toFixed(fractionDigits)
-                }
             } else {
                 let totalCost = 0
                 if (promptTokens) {
                     const cost = costValues.input_cost * (promptTokens / 1000)
                     totalCost += cost
-                    if (cost < 0.01) {
+                    promptTokensCost = formatCost(cost)
-                        promptTokensCost = '$ <0.01'
-                    } else {
-                        promptTokensCost = '$ ' + cost.toFixed(fractionDigits)
-                    }
                 }
                 if (completionTokens) {
                     const cost = costValues.output_cost * (completionTokens / 1000)
                     totalCost += cost
-                    if (cost < 0.01) {
+                    completionTokensCost = formatCost(cost)
-                        completionTokensCost = '$ <0.01'
-                    } else {
-                        completionTokensCost = '$ ' + cost.toFixed(fractionDigits)
-                    }
-                }
-                if (totalCost < 0.01) {
-                    totalTokensCost = '$ <0.01'
-                } else {
-                    totalTokensCost = '$ ' + totalCost.toFixed(fractionDigits)
                 }
+                totalTokensCost = formatCost(totalCost)
             }
         }
         metric['totalCost'] = totalTokensCost
@@ -58,3 +48,10 @@ export const calculateCost = (metricsArray: ICommonObject[]) => {
         metric['completionCost'] = completionTokensCost
     }
 }
+
+export const formatCost = (cost: number) => {
+    if (cost == 0) {
+        return '$ 0'
+    }
+    return cost < 0.01 ? '$ <0.01' : '$ ' + cost.toFixed(fractionDigits)
+}

packages/server/src/services/evaluations/index.ts

@@ -15,10 +15,11 @@ import { getAppVersion } from '../../utils'
 import { In } from 'typeorm'
 import { getWorkspaceSearchOptions } from '../../enterprise/utils/ControllerServiceUtils'
 import { v4 as uuidv4 } from 'uuid'
-import { calculateCost } from './CostCalculator'
+import { calculateCost, formatCost } from './CostCalculator'
 import { runAdditionalEvaluators } from './EvaluatorRunner'
 import evaluatorsService from '../evaluator'
 import { LLMEvaluationRunner } from './LLMEvaluationRunner'
+import { Assistant } from '../../database/entities/Assistant'
 
 const runAgain = async (id: string, baseURL: string, orgId: string) => {
     try {
@@ -27,7 +28,7 @@ const runAgain = async (id: string, baseURL: string, orgId: string) => {
             id: id
         })
         if (!evaluation) throw new Error(`Evaluation ${id} not found`)
-        const additionalConfig: any = JSON.parse(evaluation.additionalConfig)
+        const additionalConfig = evaluation.additionalConfig ? JSON.parse(evaluation.additionalConfig) : {}
         const data: ICommonObject = {
             chatflowId: evaluation.chatflowId,
             chatflowName: evaluation.chatflowName,
@@ -35,7 +36,8 @@ const runAgain = async (id: string, baseURL: string, orgId: string) => {
             datasetId: evaluation.datasetId,
             evaluationType: evaluation.evaluationType,
             selectedSimpleEvaluators: JSON.stringify(additionalConfig.simpleEvaluators),
-            datasetAsOneConversation: additionalConfig.datasetAsOneConversation
+            datasetAsOneConversation: additionalConfig.datasetAsOneConversation,
+            chatflowType: JSON.stringify(additionalConfig.chatflowTypes ? additionalConfig.chatflowTypes : [])
         }
         data.name = evaluation.name
         data.workspaceId = evaluation.workspaceId
@@ -69,7 +71,8 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str
         const row = appServer.AppDataSource.getRepository(Evaluation).create(newEval)
         row.average_metrics = JSON.stringify({})
 
-        const additionalConfig: any = {
+        const additionalConfig: ICommonObject = {
+            chatflowTypes: body.chatflowType ? JSON.parse(body.chatflowType) : [],
             datasetAsOneConversation: body.datasetAsOneConversation,
             simpleEvaluators: body.selectedSimpleEvaluators.length > 0 ? JSON.parse(body.selectedSimpleEvaluators) : []
         }
@@ -152,7 +155,7 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str
         let evalMetrics = { passCount: 0, failCount: 0, errorCount: 0 }
         evalRunner
             .runEvaluations(data)
-            .then(async (result: any) => {
+            .then(async (result) => {
                 let totalTime = 0
                 // let us assume that the eval is successful
                 let allRowsSuccessful = true
@@ -171,8 +174,48 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str
                             totalTime += parseFloat(evaluationRow.latency)
                             let metricsObjFromRun: ICommonObject = {}
 
+                            let nested_metrics = evaluationRow.nested_metrics
+
+                            let promptTokens = 0,
+                                completionTokens = 0,
+                                totalTokens = 0
+                            let inputCost = 0,
+                                outputCost = 0,
+                                totalCost = 0
+                            if (nested_metrics && nested_metrics.length > 0) {
+                                for (let i = 0; i < nested_metrics.length; i++) {
+                                    const nested_metric = nested_metrics[i]
+                                    if (nested_metric.model && nested_metric.promptTokens > 0) {
+                                        promptTokens += nested_metric.promptTokens
+                                        completionTokens += nested_metric.completionTokens
+                                        totalTokens += nested_metric.totalTokens
+
+                                        inputCost += nested_metric.cost_values.input_cost
+                                        outputCost += nested_metric.cost_values.output_cost
+                                        totalCost += nested_metric.cost_values.total_cost
+
+                                        nested_metric['totalCost'] = formatCost(nested_metric.cost_values.total_cost)
+                                        nested_metric['promptCost'] = formatCost(nested_metric.cost_values.input_cost)
+                                        nested_metric['completionCost'] = formatCost(nested_metric.cost_values.output_cost)
+                                    }
+                                }
+                                nested_metrics = nested_metrics.filter((metric: any) => {
+                                    return metric.model && metric.provider
+                                })
+                            }
                             const metrics = evaluationRow.metrics
                             if (metrics) {
+                                if (nested_metrics && nested_metrics.length > 0) {
+                                    metrics.push({
+                                        promptTokens: promptTokens,
+                                        completionTokens: completionTokens,
+                                        totalTokens: totalTokens,
+                                        totalCost: formatCost(totalCost),
+                                        promptCost: formatCost(inputCost),
+                                        completionCost: formatCost(outputCost)
+                                    })
+                                    metricsObjFromRun.nested_metrics = nested_metrics
+                                }
                                 metrics.map((metric: any) => {
                                     if (metric) {
                                         const json = typeof metric === 'object' ? metric : JSON.parse(metric)
@@ -211,7 +254,7 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str
                         if (body.evaluationType === 'llm') {
                             resultRow.llmConfig = additionalConfig.llmConfig
                             resultRow.LLMEvaluators = body.selectedLLMEvaluators.length > 0 ? JSON.parse(body.selectedLLMEvaluators) : []
-                            const llmEvaluatorMap: any = []
+                            const llmEvaluatorMap: { evaluatorId: string; evaluator: any }[] = []
                             for (let i = 0; i < resultRow.LLMEvaluators.length; i++) {
                                 const evaluatorId = resultRow.LLMEvaluators[i]
                                 const evaluator = await evaluatorsService.getEvaluator(evaluatorId)
@@ -243,23 +286,27 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str
                     }
                     appServer.AppDataSource.getRepository(Evaluation)
                         .findOneBy({ id: newEvaluation.id })
-                        .then((evaluation: any) => {
+                        .then((evaluation) => {
-                            evaluation.status = allRowsSuccessful ? EvaluationStatus.COMPLETED : EvaluationStatus.ERROR
+                            if (evaluation) {
-                            evaluation.average_metrics = JSON.stringify({
+                                evaluation.status = allRowsSuccessful ? EvaluationStatus.COMPLETED : EvaluationStatus.ERROR
-                                averageLatency: (totalTime / result.rows.length).toFixed(3),
+                                evaluation.average_metrics = JSON.stringify({
-                                totalRuns: result.rows.length,
+                                    averageLatency: (totalTime / result.rows.length).toFixed(3),
-                                ...evalMetrics,
+                                    totalRuns: result.rows.length,
-                                passPcnt: passPercent.toFixed(2)
+                                    ...evalMetrics,
-                            })
+                                    passPcnt: passPercent.toFixed(2)
-                            appServer.AppDataSource.getRepository(Evaluation).save(evaluation)
+                                })
+                                appServer.AppDataSource.getRepository(Evaluation).save(evaluation)
+                            }
                         })
                 } catch (error) {
                     //update the evaluation with status as error
                     appServer.AppDataSource.getRepository(Evaluation)
                         .findOneBy({ id: newEvaluation.id })
-                        .then((evaluation: any) => {
+                        .then((evaluation) => {
-                            evaluation.status = EvaluationStatus.ERROR
+                            if (evaluation) {
-                            appServer.AppDataSource.getRepository(Evaluation).save(evaluation)
+                                evaluation.status = EvaluationStatus.ERROR
+                                appServer.AppDataSource.getRepository(Evaluation).save(evaluation)
+                            }
                         })
                 }
             })
@@ -268,12 +315,14 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str
                 console.error('Error running evaluations:', getErrorMessage(error))
                 appServer.AppDataSource.getRepository(Evaluation)
                     .findOneBy({ id: newEvaluation.id })
-                    .then((evaluation: any) => {
+                    .then((evaluation) => {
-                        evaluation.status = EvaluationStatus.ERROR
+                        if (evaluation) {
-                        evaluation.average_metrics = JSON.stringify({
+                            evaluation.status = EvaluationStatus.ERROR
-                            error: getErrorMessage(error)
+                            evaluation.average_metrics = JSON.stringify({
-                        })
+                                error: getErrorMessage(error)
-                        appServer.AppDataSource.getRepository(Evaluation).save(evaluation)
+                            })
+                            appServer.AppDataSource.getRepository(Evaluation).save(evaluation)
+                        }
                     })
                     .catch((dbError) => {
                         console.error('Error updating evaluation status:', getErrorMessage(dbError))
@@ -378,18 +427,31 @@ const isOutdated = async (id: string) => {
                 returnObj.dataset = dataset
             }
         } else {
-            returnObj.errors.push(`Dataset ${evaluation.datasetName} not found`)
+            returnObj.errors.push({
+                message: `Dataset ${evaluation.datasetName} not found`,
+                id: evaluation.datasetId
+            })
             isOutdated = true
         }
-        const chatflows = JSON.parse(evaluation.chatflowId)
+        const chatflowIds = evaluation.chatflowId ? JSON.parse(evaluation.chatflowId) : []
-        const chatflowNames = JSON.parse(evaluation.chatflowName)
+        const chatflowNames = evaluation.chatflowName ? JSON.parse(evaluation.chatflowName) : []
-
+        const chatflowTypes = evaluation.additionalConfig ? JSON.parse(evaluation.additionalConfig).chatflowTypes : []
-        for (let i = 0; i < chatflows.length; i++) {
+        for (let i = 0; i < chatflowIds.length; i++) {
+            // check for backward compatibility, as previous versions did not the types in additionalConfig
+            if (chatflowTypes && chatflowTypes.length >= 0) {
+                if (chatflowTypes[i] === 'Custom Assistant') {
+                    // if the chatflow type is custom assistant, then we should NOT check in the chatflows table
+                    continue
+                }
+            }
             const chatflow = await appServer.AppDataSource.getRepository(ChatFlow).findOneBy({
-                id: chatflows[i]
+                id: chatflowIds[i]
             })
             if (!chatflow) {
-                returnObj.errors.push(`Chatflow ${chatflowNames[i]} not found`)
+                returnObj.errors.push({
+                    message: `Chatflow ${chatflowNames[i]} not found`,
+                    id: chatflowIds[i]
+                })
                 isOutdated = true
             } else {
                 const chatflowLastUpdated = chatflow.updatedDate.getTime()
@@ -397,12 +459,42 @@ const isOutdated = async (id: string) => {
                     isOutdated = true
                     returnObj.chatflows.push({
                         chatflowName: chatflowNames[i],
-                        chatflowId: chatflows[i],
+                        chatflowId: chatflowIds[i],
+                        chatflowType: chatflow.type === 'AGENTFLOW' ? 'Agentflow v2' : 'Chatflow',
                         isOutdated: true
                     })
                 }
             }
         }
+        if (chatflowTypes && chatflowTypes.length > 0) {
+            for (let i = 0; i < chatflowIds.length; i++) {
+                if (chatflowTypes[i] !== 'Custom Assistant') {
+                    // if the chatflow type is NOT custom assistant, then bail out for this item
+                    continue
+                }
+                const assistant = await appServer.AppDataSource.getRepository(Assistant).findOneBy({
+                    id: chatflowIds[i]
+                })
+                if (!assistant) {
+                    returnObj.errors.push({
+                        message: `Custom Assistant ${chatflowNames[i]} not found`,
+                        id: chatflowIds[i]
+                    })
+                    isOutdated = true
+                } else {
+                    const chatflowLastUpdated = assistant.updatedDate.getTime()
+                    if (chatflowLastUpdated > evaluationRunDate) {
+                        isOutdated = true
+                        returnObj.chatflows.push({
+                            chatflowName: chatflowNames[i],
+                            chatflowId: chatflowIds[i],
+                            chatflowType: 'Custom Assistant',
+                            isOutdated: true
+                        })
+                    }
+                }
+            }
+        }
         returnObj.isOutdated = isOutdated
         return returnObj
     } catch (error) {
@@ -424,7 +516,7 @@ const getEvaluation = async (id: string) => {
             where: { evaluationId: id }
         })
         const versions = (await getVersions(id)).versions
-        const versionNo = versions.findIndex((version: any) => version.id === id) + 1
+        const versionNo = versions.findIndex((version) => version.id === id) + 1
         return {
             ...evaluation,
             versionCount: versionCount,
@@ -451,7 +543,7 @@ const getVersions = async (id: string) => {
                 runDate: 'ASC'
             }
         })
-        const returnResults: any[] = []
+        const returnResults: { id: string; runDate: Date; version: number }[] = []
         versions.map((version, index) => {
             returnResults.push({
                 id: version.id,

packages/server/src/utils/buildAgentflow.ts

@@ -1805,7 +1805,7 @@ export const executeAgentFlow = async ({
         role: 'userMessage',
         content: finalUserInput,
         chatflowid,
-        chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
+        chatType: evaluationRunId ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
         chatId,
         sessionId,
         createdDate: userMessageDateTime,
@@ -1820,7 +1820,7 @@ export const executeAgentFlow = async ({
         role: 'apiMessage',
         content: content,
         chatflowid,
-        chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
+        chatType: evaluationRunId ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
         chatId,
         sessionId,
         executionId: newExecution.id
@@ -1856,7 +1856,7 @@ export const executeAgentFlow = async ({
             version: await getAppVersion(),
             chatflowId: chatflowid,
             chatId,
-            type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
+            type: evaluationRunId ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
             flowGraph: getTelemetryFlowObj(nodes, edges)
         },
         orgId

packages/server/src/utils/buildChatflow.ts

@@ -551,7 +551,7 @@ export const executeFlow = async ({
                 role: 'userMessage',
                 content: incomingInput.question,
                 chatflowid: agentflow.id,
-                chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
+                chatType: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
                 chatId,
                 memoryType,
                 sessionId,
@@ -566,7 +566,7 @@ export const executeFlow = async ({
                 role: 'apiMessage',
                 content: finalResult,
                 chatflowid: agentflow.id,
-                chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
+                chatType: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
                 chatId,
                 memoryType,
                 sessionId
@@ -598,7 +598,7 @@ export const executeFlow = async ({
                     version: await getAppVersion(),
                     agentflowId: agentflow.id,
                     chatId,
-                    type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
+                    type: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
                     flowGraph: getTelemetryFlowObj(nodes, edges)
                 },
                 orgId
@@ -807,7 +807,7 @@ export const executeFlow = async ({
                 version: await getAppVersion(),
                 chatflowId: chatflowid,
                 chatId,
-                type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
+                type: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
                 flowGraph: getTelemetryFlowObj(nodes, edges)
             },
             orgId
@@ -905,17 +905,17 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
     const isTool = req.get('flowise-tool') === 'true'
     const isEvaluation: boolean = req.headers['X-Flowise-Evaluation'] || req.body.evaluation
     let evaluationRunId = ''
-    if (isEvaluation) {
+    evaluationRunId = req.body.evaluationRunId
-        evaluationRunId = req.body.evaluationRunId
+    if (isEvaluation && chatflow.type !== 'AGENTFLOW' && req.body.evaluationRunId) {
-        if (evaluationRunId) {
+        // this is needed for the collection of token metrics for non-agent flows,
-            const newEval = {
+        // for agentflows the execution trace has the info needed
-                evaluation: {
+        const newEval = {
-                    status: true,
+            evaluation: {
-                    evaluationRunId
+                status: true,
-                }
+                evaluationRunId
             }
-            chatflow.analytic = JSON.stringify(newEval)
         }
+        chatflow.analytic = JSON.stringify(newEval)
     }
 
     try {

packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx

@@ -149,7 +149,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
     const [sourceDialogProps, setSourceDialogProps] = useState({})
     const [hardDeleteDialogOpen, setHardDeleteDialogOpen] = useState(false)
     const [hardDeleteDialogProps, setHardDeleteDialogProps] = useState({})
-    const [chatTypeFilter, setChatTypeFilter] = useState([])
+    const [chatTypeFilter, setChatTypeFilter] = useState(['INTERNAL', 'EXTERNAL'])
     const [feedbackTypeFilter, setFeedbackTypeFilter] = useState([])
     const [startDate, setStartDate] = useState(new Date(new Date().setMonth(new Date().getMonth() - 1)))
     const [endDate, setEndDate] = useState(new Date())
@@ -310,6 +310,15 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
         }
     }
 
+    const getChatType = (chatType) => {
+        if (chatType === 'INTERNAL') {
+            return 'UI'
+        } else if (chatType === 'EVALUATION') {
+            return 'Evaluation'
+        }
+        return 'API/Embed'
+    }
+
     const exportMessages = async () => {
         if (!storagePath && getStoragePathFromServer.data) {
             storagePath = getStoragePathFromServer.data.storagePath
@@ -356,7 +365,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
             if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) {
                 obj[chatPK] = {
                     id: chatmsg.chatId,
-                    source: chatmsg.chatType === 'INTERNAL' ? 'UI' : 'API/Embed',
+                    source: getChatType(chatmsg.chatType),
                     sessionId: chatmsg.sessionId ?? null,
                     memoryType: chatmsg.memoryType ?? null,
                     email: chatmsg.leadEmail ?? null,
@@ -716,7 +725,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
             setChatLogs([])
             setAllChatLogs([])
             setChatMessages([])
-            setChatTypeFilter([])
+            setChatTypeFilter(['INTERNAL', 'EXTERNAL'])
             setFeedbackTypeFilter([])
             setSelectedMessageIndex(0)
             setSelectedChatId('')
@@ -880,6 +889,10 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                                     {
                                         label: 'API/Embed',
                                         name: 'EXTERNAL'
+                                    },
+                                    {
+                                        label: 'Evaluations',
+                                        name: 'EVALUATION'
                                     }
                                 ]}
                                 onSelect={(newValue) => onChatTypeSelected(newValue)}
@@ -1016,7 +1029,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
                                             )}
                                             {chatMessages[1].chatType && (
                                                 <div>
-                                                    Source:&nbsp;<b>{chatMessages[1].chatType === 'INTERNAL' ? 'UI' : 'API/Embed'}</b>
+                                                    Source:&nbsp;<b>{getChatType(chatMessages[1].chatType)}</b>
                                                 </div>
                                             )}
                                             {chatMessages[1].memoryType && (

packages/ui/src/views/evaluations/CreateEvaluationDialog.jsx

@@ -21,7 +21,8 @@ import {
     Switch,
     StepLabel,
     IconButton,
-    FormControlLabel
+    FormControlLabel,
+    Checkbox
 } from '@mui/material'
 import { useTheme } from '@mui/material/styles'
 
@@ -42,6 +43,7 @@ import useApi from '@/hooks/useApi'
 import datasetsApi from '@/api/dataset'
 import evaluatorsApi from '@/api/evaluators'
 import nodesApi from '@/api/nodes'
+import assistantsApi from '@/api/assistants'
 
 // utils
 import useNotifier from '@/utils/useNotifier'
@@ -57,14 +59,18 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
     useNotifier()
 
     const getAllChatflowsApi = useApi(chatflowsApi.getAllChatflows)
+    const getAllAgentflowsApi = useApi(chatflowsApi.getAllAgentflows)
+
     const getAllDatasetsApi = useApi(datasetsApi.getAllDatasets)
     const getAllEvaluatorsApi = useApi(evaluatorsApi.getAllEvaluators)
     const getNodesByCategoryApi = useApi(nodesApi.getNodesByCategory)
     const getModelsApi = useApi(nodesApi.executeNodeLoadMethod)
+    const getAssistantsApi = useApi(assistantsApi.getAllAssistants)
 
     const [chatflow, setChatflow] = useState([])
     const [dataset, setDataset] = useState('')
     const [datasetAsOneConversation, setDatasetAsOneConversation] = useState(false)
+    const [flowTypes, setFlowTypes] = useState([])
 
     const [flows, setFlows] = useState([])
     const [datasets, setDatasets] = useState([])
@@ -163,6 +169,10 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
         for (let i = 0; i < selectedChatflows.length; i += 1) {
             selectedChatflowNames.push(flows.find((f) => f.name === selectedChatflows[i])?.label)
         }
+        const selectedChatflowTypes = []
+        for (let i = 0; i < selectedChatflows.length; i += 1) {
+            selectedChatflowTypes.push(flows.find((f) => f.name === selectedChatflows[i])?.type)
+        }
         const chatflowName = JSON.stringify(selectedChatflowNames)
         const datasetName = datasets.find((f) => f.name === dataset)?.label
         const obj = {
@@ -173,6 +183,7 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
             datasetName: datasetName,
             chatflowId: chatflow,
             chatflowName: chatflowName,
+            chatflowType: JSON.stringify(selectedChatflowTypes),
             selectedSimpleEvaluators: selectedSimpleEvaluators,
             selectedLLMEvaluators: selectedLLMEvaluators,
             model: selectedModel,
@@ -216,6 +227,8 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
         getNodesByCategoryApi.request('Chat Models')
         if (flows.length === 0) {
             getAllChatflowsApi.request()
+            getAssistantsApi.request('CUSTOM')
+            getAllAgentflowsApi.request('AGENTFLOW')
         }
         if (datasets.length === 0) {
             getAllDatasetsApi.request()
@@ -225,23 +238,18 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
     }, [])
 
     useEffect(() => {
-        if (getAllChatflowsApi.data) {
+        if (getAllAgentflowsApi.data && getAllChatflowsApi.data && getAssistantsApi.data) {
             try {
-                const chatflows = getAllChatflowsApi.data
+                const agentFlows = populateFlowNames(getAllAgentflowsApi.data, 'Agentflow v2')
-                let flowNames = []
+                const chatFlows = populateFlowNames(getAllChatflowsApi.data, 'Chatflow')
-                for (let i = 0; i < chatflows.length; i += 1) {
+                const assistants = populateAssistants(getAssistantsApi.data)
-                    const flow = chatflows[i]
+                setFlows([...agentFlows, ...chatFlows, ...assistants])
-                    flowNames.push({
+                setFlowTypes(['Agentflow v2', 'Chatflow', 'Custom Assistant'])
-                        label: flow.name,
-                        name: flow.id
-                    })
-                }
-                setFlows(flowNames)
             } catch (e) {
                 console.error(e)
             }
         }
-    }, [getAllChatflowsApi.data])
+    }, [getAllAgentflowsApi.data, getAllChatflowsApi.data, getAssistantsApi.data])
 
     useEffect(() => {
         if (getNodesByCategoryApi.data) {
@@ -337,6 +345,44 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
         if (llm !== 'no_grading') getModelsApi.request(llm, { loadMethod: 'listModels' })
     }
 
+    const onChangeFlowType = (flowType) => {
+        const selected = flowType.target.checked
+        const flowTypeValue = flowType.target.value
+        if (selected) {
+            setFlowTypes([...flowTypes, flowTypeValue])
+        } else {
+            setFlowTypes(flowTypes.filter((f) => f !== flowTypeValue))
+        }
+    }
+
+    const populateFlowNames = (data, type) => {
+        let flowNames = []
+        for (let i = 0; i < data.length; i += 1) {
+            const flow = data[i]
+            flowNames.push({
+                label: flow.name,
+                name: flow.id,
+                type: type,
+                description: type
+            })
+        }
+        return flowNames
+    }
+
+    const populateAssistants = (assistants) => {
+        let assistantNames = []
+        for (let i = 0; i < assistants.length; i += 1) {
+            const assistant = assistants[i]
+            assistantNames.push({
+                label: JSON.parse(assistant.details).name || '',
+                name: assistant.id,
+                type: 'Custom Assistant',
+                description: 'Custom Assistant'
+            })
+        }
+        return assistantNames
+    }
+
     const component = show ? (
         <Dialog
             fullWidth
@@ -476,18 +522,42 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => {
                                     Treat all dataset rows as one conversation ?
                                 </Typography>
                                 <FormControlLabel
+                                    label=''
                                     control={<Switch />}
                                     value={datasetAsOneConversation}
                                     onChange={() => setDatasetAsOneConversation(!datasetAsOneConversation)}
                                 />
                             </Box>
                             <Box>
-                                <Typography variant='overline'>
+                                <div style={{ display: 'flex', justifyContent: 'space-between' }}>
-                                    Chatflow(s) to Evaluate<span style={{ color: 'red' }}>&nbsp;*</span>
+                                    <Typography variant='overline'>
-                                </Typography>
+                                        Select your flows to Evaluate
+                                        <span style={{ color: 'red' }}>&nbsp;*</span>
+                                    </Typography>
+                                    <Typography variant='overline'>
+                                        <Checkbox defaultChecked size='small' label='All' value='Chatflow' onChange={onChangeFlowType} />{' '}
+                                        Chatflows
+                                        <Checkbox
+                                            defaultChecked
+                                            size='small'
+                                            label='All'
+                                            value='Agentflow v2'
+                                            onChange={onChangeFlowType}
+                                        />{' '}
+                                        Agentflows (v2)
+                                        <Checkbox
+                                            defaultChecked
+                                            size='small'
+                                            label='All'
+                                            value='Custom Assistant'
+                                            onChange={onChangeFlowType}
+                                        />{' '}
+                                        Custom Assistants
+                                    </Typography>
+                                </div>
                                 <MultiDropdown
                                     name={'chatflow1'}
-                                    options={flows}
+                                    options={flows.filter((f) => flowTypes.includes(f.type))}
                                     onSelect={(newValue) => setChatflow(newValue)}
                                     value={chatflow ?? chatflow ?? 'choose an option'}
                                 />

packages/ui/src/views/evaluations/EvalsResultDialog.jsx

@@ -2,7 +2,6 @@ import React from 'react'
 import { createPortal } from 'react-dom'
 import PropTypes from 'prop-types'
 import { useSelector } from 'react-redux'
-import { useNavigate } from 'react-router-dom'
 
 // Material
 import {
@@ -36,7 +35,6 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) =
     const portalElement = document.getElementById('portal')
     const customization = useSelector((state) => state.customization)
     const theme = useTheme()
-    const navigate = useNavigate()
 
     const getColSpan = (evaluationsShown, llmEvaluations) => {
         let colSpan = 1
@@ -45,6 +43,23 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) =
         return colSpan
     }
 
+    const getOpenLink = (index) => {
+        if (index === undefined) {
+            return ''
+        }
+        if (dialogProps.data?.additionalConfig?.chatflowTypes) {
+            switch (dialogProps.data.additionalConfig.chatflowTypes[index]) {
+                case 'Chatflow':
+                    return '/canvas/' + dialogProps.data.evaluation.chatflowId[index]
+                case 'Custom Assistant':
+                    return '/assistants/custom/' + dialogProps.data.evaluation.chatflowId[index]
+                case 'Agentflow v2':
+                    return '/v2/agentcanvas/' + dialogProps.data.evaluation.chatflowId[index]
+            }
+        }
+        return '/canvas/' + dialogProps.data.evaluation.chatflowId[index]
+    }
+
     const component = show ? (
         <Dialog fullScreen open={show} onClose={onCancel} aria-labelledby='alert-dialog-title' aria-describedby='alert-dialog-description'>
             <DialogTitle id='alert-dialog-title'>
@@ -65,7 +80,7 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) =
                                 }}
                             >
                                 <IconVectorBezier2 style={{ marginRight: 5 }} size={17} />
-                                Chatflows Used:
+                                Flows Used:
                             </div>
                             {(dialogProps.data.evaluation.chatflowName || []).map((chatflowUsed, index) => (
                                 <Chip
@@ -79,7 +94,7 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) =
                                             : '0 2px 14px 0 rgb(32 40 45 / 10%)'
                                     }}
                                     label={chatflowUsed}
-                                    onClick={() => navigate('/canvas/' + dialogProps.data.evaluation.chatflowId[index])}
+                                    onClick={() => window.open(getOpenLink(index), '_blank')}
                                 ></Chip>
                             ))}
                         </Stack>

packages/ui/src/views/evaluations/EvaluationResult.jsx

@@ -25,6 +25,7 @@ import {
 import { useTheme } from '@mui/material/styles'
 import moment from 'moment'
 import PaidIcon from '@mui/icons-material/Paid'
+import { IconHierarchy, IconUsersGroup, IconRobot } from '@tabler/icons-react'
 import LLMIcon from '@mui/icons-material/ModelTraining'
 import AlarmIcon from '@mui/icons-material/AlarmOn'
 import TokensIcon from '@mui/icons-material/AutoAwesomeMotion'
@@ -116,10 +117,13 @@ const EvalEvaluationRows = () => {
     const [expandTableProps, setExpandTableProps] = useState({})
     const [isTableLoading, setTableLoading] = useState(false)
 
+    const [additionalConfig, setAdditionalConfig] = useState({})
+
     const openDetailsDrawer = (item) => {
         setSideDrawerDialogProps({
             type: 'View',
             data: item,
+            additionalConfig: additionalConfig,
             evaluationType: evaluation.evaluationType,
             evaluationChatflows: evaluation.chatflowName
         })
@@ -169,7 +173,8 @@ const EvalEvaluationRows = () => {
                 showCustomEvals,
                 showTokenMetrics,
                 showLatencyMetrics,
-                showCostMetrics
+                showCostMetrics,
+                additionalConfig
             }
         })
         setShowExpandTableDialog(true)
@@ -239,6 +244,9 @@ const EvalEvaluationRows = () => {
             const data = getEvaluation.data
             setSelectedEvaluationName(data.name)
             getIsOutdatedApi.request(data.id)
+            if (data.additionalConfig) {
+                setAdditionalConfig(JSON.parse(data.additionalConfig))
+            }
             data.chatflowId = typeof data.chatflowId === 'object' ? data.chatflowId : JSON.parse(data.chatflowId)
             data.chatflowName = typeof data.chatflowName === 'object' ? data.chatflowName : JSON.parse(data.chatflowName)
             const rows = getEvaluation.data.rows
@@ -314,6 +322,51 @@ const EvalEvaluationRows = () => {
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [getEvaluation.data])
 
+    const getOpenLink = (index) => {
+        if (index === undefined) {
+            return undefined
+        }
+        const id = evaluation.chatflowId[index]
+        // this is to check if the evaluation is deleted!
+        if (outdated?.errors?.length > 0 && outdated.errors.find((e) => e.id === id)) {
+            return undefined
+        }
+        if (additionalConfig.chatflowTypes) {
+            switch (additionalConfig.chatflowTypes[index]) {
+                case 'Chatflow':
+                    return '/canvas/' + evaluation.chatflowId[index]
+                case 'Custom Assistant':
+                    return '/assistants/custom/' + evaluation.chatflowId[index]
+                case 'Agentflow v2':
+                    return '/v2/agentcanvas/' + evaluation.chatflowId[index]
+            }
+        }
+        return '/canvas/' + evaluation.chatflowId[index]
+    }
+
+    const openFlow = (index) => {
+        const url = getOpenLink(index)
+        if (url) {
+            window.open(getOpenLink(index), '_blank')
+        }
+    }
+
+    const getFlowIcon = (index) => {
+        if (index === undefined) {
+            return <IconHierarchy size={17} />
+        }
+        if (additionalConfig.chatflowTypes) {
+            switch (additionalConfig.chatflowTypes[index]) {
+                case 'Chatflow':
+                    return <IconHierarchy size={17} />
+                case 'Custom Assistant':
+                    return <IconRobot size={17} />
+                case 'Agentflow v2':
+                    return <IconUsersGroup size={17} />
+            }
+        }
+        return <IconHierarchy />
+    }
     return (
         <>
             <MainCard>
@@ -405,14 +458,14 @@ const EvalEvaluationRows = () => {
                                                 }}
                                                 variant='outlined'
                                                 label={outdated.dataset.name}
-                                                onClick={() => navigate(`/dataset_rows/${outdated.dataset.id}`)}
+                                                onClick={() => window.open(`/dataset_rows/${outdated.dataset.id}`, '_blank')}
                                             ></Chip>
                                         </>
                                     )}
                                     {outdated.chatflows && outdated?.errors?.length === 0 && outdated.chatflows.length > 0 && (
                                         <>
                                             <br />
-                                            <b style={{ color: 'rgb(116,66,16)' }}>Chatflows:</b>
+                                            <b style={{ color: 'rgb(116,66,16)' }}>Flows:</b>
                                             <Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
                                                 {outdated.chatflows.map((chatflow, index) => (
                                                     <Chip
@@ -429,14 +482,23 @@ const EvalEvaluationRows = () => {
                                                         }}
                                                         variant='outlined'
                                                         label={chatflow.chatflowName}
-                                                        onClick={() => navigate(`/canvas/${chatflow.chatflowId}`)}
+                                                        onClick={() =>
+                                                            window.open(
+                                                                chatflow.chatflowType === 'Chatflow'
+                                                                    ? '/canvas/' + chatflow.chatflowId
+                                                                    : chatflow.chatflowType === 'Custom Assistant'
+                                                                    ? '/assistants/custom/' + chatflow.chatflowId
+                                                                    : '/v2/agentcanvas/' + chatflow.chatflowId,
+                                                                '_blank'
+                                                            )
+                                                        }
                                                     ></Chip>
                                                 ))}
                                             </Stack>
                                         </>
                                     )}
                                     {outdated.errors.length > 0 &&
-                                        outdated.errors.map((error, index) => <ListItem key={index}>{error}</ListItem>)}
+                                        outdated.errors.map((error, index) => <ListItem key={index}>{error.message}</ListItem>)}
                                     <IconButton
                                         style={{ position: 'absolute', top: 10, right: 10 }}
                                         size='small'
@@ -501,7 +563,7 @@ const EvalEvaluationRows = () => {
                         {showCharts && (
                             <Grid container={true} spacing={2}>
                                 {customEvalsDefined && (
-                                    <Grid item={true} xs={12} sm={6} md={4} lg={4}>
+                                    <Grid item={true} xs={12} sm={12} md={4} lg={4}>
                                         <MetricsItemCard
                                             data={{
                                                 header: 'PASS RATE',
@@ -566,11 +628,12 @@ const EvalEvaluationRows = () => {
                                     }}
                                 >
                                     <IconVectorBezier2 style={{ marginRight: 5 }} size={17} />
-                                    Chatflows Used:
+                                    Flows Used:
                                 </div>
                                 {(evaluation.chatflowName || []).map((chatflowUsed, index) => (
                                     <Chip
                                         key={index}
+                                        icon={getFlowIcon(index)}
                                         clickable
                                         style={{
                                             width: 'max-content',
@@ -580,7 +643,7 @@ const EvalEvaluationRows = () => {
                                                 : '0 2px 14px 0 rgb(32 40 45 / 10%)'
                                         }}
                                         label={chatflowUsed}
-                                        onClick={() => navigate('/canvas/' + evaluation.chatflowId[index])}
+                                        onClick={() => openFlow(index)}
                                     ></Chip>
                                 ))}
                             </Stack>

packages/ui/src/views/evaluations/EvaluationResultSideDrawer.jsx

@@ -1,8 +1,25 @@
 import PropTypes from 'prop-types'
-import { CardContent, Card, Box, SwipeableDrawer, Stack, Button, Chip, Divider, Typography } from '@mui/material'
+import {
+    CardContent,
+    Card,
+    Box,
+    SwipeableDrawer,
+    Stack,
+    Button,
+    Chip,
+    Divider,
+    Typography,
+    Table,
+    TableHead,
+    TableRow,
+    TableBody
+} from '@mui/material'
+import { IconHierarchy, IconUsersGroup, IconRobot } from '@tabler/icons-react'
+
 import { useSelector } from 'react-redux'
-import { IconSquareRoundedChevronsRight } from '@tabler/icons-react'
 import { evaluators as evaluatorsOptions, numericOperators } from '../evaluators/evaluatorConstant'
+import TableCell from '@mui/material/TableCell'
+import { Close } from '@mui/icons-material'
 
 const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {
     const onOpen = () => {}
@@ -19,12 +36,32 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {
         return ''
     }
 
+    const getFlowIcon = (index) => {
+        if (index === undefined) {
+            return <IconHierarchy size={24} />
+        }
+        if (dialogProps.additionalConfig.chatflowTypes) {
+            switch (dialogProps.additionalConfig.chatflowTypes[index]) {
+                case 'Chatflow':
+                    return <IconHierarchy size={20} />
+                case 'Custom Assistant':
+                    return <IconRobot size={20} />
+                case 'Agentflow v2':
+                    return <IconUsersGroup size={20} />
+            }
+        }
+        return <IconHierarchy />
+    }
+
     return (
         <SwipeableDrawer sx={{ zIndex: 2000 }} anchor='right' open={show} onClose={() => onClickFunction()} onOpen={onOpen}>
-            <Button startIcon={<IconSquareRoundedChevronsRight />} onClick={() => onClickFunction()}>
+            <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', borderBottom: '1px solid #ccc' }}>
-                Close
+                <Typography variant='overline' sx={{ margin: 1, fontWeight: 'bold' }}>
-            </Button>
+                    Evaluation Details
-            <Box sx={{ width: 450, p: 3 }} role='presentation'>
+                </Typography>
+                <Button endIcon={<Close />} onClick={() => onClickFunction()} />
+            </div>
+            <Box sx={{ width: 600, p: 2 }} role='presentation'>
                 <Box>
                     <Typography variant='overline' sx={{ fontWeight: 'bold' }}>
                         Evaluation Id
@@ -61,13 +98,19 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {
                             <CardContent>
                                 {dialogProps.evaluationChatflows?.length > 0 && (
                                     <>
-                                        <Box>
+                                        <div
-                                            <Typography variant='overline' sx={{ fontWeight: 'bold' }}>
+                                            style={{
-                                                Chatflow
+                                                display: 'flex',
+                                                justifyContent: 'start',
+                                                alignItems: 'center',
+                                                marginBottom: 5
+                                            }}
+                                        >
+                                            {getFlowIcon(index)}
+                                            <Typography variant='overline' sx={{ fontWeight: 'bold', fontSize: '1.1rem', marginLeft: 1 }}>
+                                                {dialogProps.evaluationChatflows[index]}
                                             </Typography>
-                                            <Typography variant='body2'>{dialogProps.evaluationChatflows[index]}</Typography>
+                                        </div>
-                                        </Box>
-                                        <br />
                                         <Divider />
                                     </>
                                 )}
@@ -153,79 +196,222 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {
                                 <br />
                                 <Divider />
                                 <br />
-                                <Box>
+                                {dialogProps.data.metrics[index]?.nested_metrics ? (
-                                    <Typography variant='overline' style={{ fontWeight: 'bold' }}>
+                                    <Box>
-                                        Tokens
+                                        <Typography variant='overline' style={{ fontWeight: 'bold' }}>
-                                    </Typography>
+                                            Tokens
-                                    <Typography variant='body2'>
+                                        </Typography>
-                                        <Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
+                                        <Table size='small' style={{ border: '1px solid #ccc' }}>
-                                            <Chip
+                                            <TableHead>
-                                                variant='outlined'
+                                                <TableRow>
-                                                size='small'
+                                                    <TableCell align='left' style={{ fontSize: '11px', fontWeight: 'bold' }}>
-                                                label={
+                                                        Node
-                                                    dialogProps.data.metrics[index]?.totalTokens
+                                                    </TableCell>
-                                                        ? 'Total: ' + dialogProps.data.metrics[index]?.totalTokens
+                                                    <TableCell align='left' style={{ fontSize: '11px', fontWeight: 'bold' }}>
-                                                        : 'Total: N/A'
+                                                        Provider & Model
-                                                }
+                                                    </TableCell>
-                                            />
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold', width: '15%' }}>
-                                            <Chip
+                                                        Input
-                                                variant='outlined'
+                                                    </TableCell>
-                                                size='small'
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold', width: '15%' }}>
-                                                label={
+                                                        Output
-                                                    dialogProps.data.metrics[index]?.promptTokens
+                                                    </TableCell>
-                                                        ? 'Prompt: ' + dialogProps.data.metrics[index]?.promptTokens
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold', width: '15%' }}>
-                                                        : 'Completion: N/A'
+                                                        Total
-                                                }
+                                                    </TableCell>
-                                            />
+                                                </TableRow>
-                                            <Chip
+                                            </TableHead>
-                                                variant='outlined'
+                                            <TableBody style={{ fontSize: '8px' }}>
-                                                size='small'
+                                                {dialogProps.data.metrics[index]?.nested_metrics?.map((metric, index) => (
-                                                label={
+                                                    <TableRow key={index}>
-                                                    dialogProps.data.metrics[index]?.completionTokens
+                                                        <TableCell component='th' scope='row' style={{ fontSize: '11px' }}>
-                                                        ? 'Completion: ' + dialogProps.data.metrics[index]?.completionTokens
+                                                            {metric.nodeLabel}
-                                                        : 'Completion: N/A'
+                                                        </TableCell>
-                                                }
+                                                        <TableCell component='th' scope='row' style={{ fontSize: '11px' }}>
-                                            />
+                                                            {metric.provider}
-                                        </Stack>
+                                                            <br />
-                                    </Typography>
+                                                            {metric.model}
-                                </Box>
+                                                        </TableCell>
+                                                        <TableCell align='right' style={{ fontSize: '11px' }}>
+                                                            {metric.promptTokens}
+                                                        </TableCell>
+                                                        <TableCell align='right' style={{ fontSize: '11px' }}>
+                                                            {metric.completionTokens}
+                                                        </TableCell>
+                                                        <TableCell align='right' style={{ fontSize: '11px' }}>
+                                                            {metric.totalTokens}
+                                                        </TableCell>
+                                                    </TableRow>
+                                                ))}
+                                                <TableRow key={index}>
+                                                    <TableCell
+                                                        align='right'
+                                                        style={{ fontSize: '11px', fontWeight: 'bold' }}
+                                                        component='th'
+                                                        scope='row'
+                                                        colspan={2}
+                                                    >
+                                                        Total
+                                                    </TableCell>
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
+                                                        {dialogProps.data.metrics[index].promptTokens}
+                                                    </TableCell>
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
+                                                        {dialogProps.data.metrics[index].completionTokens}
+                                                    </TableCell>
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
+                                                        {dialogProps.data.metrics[index].totalTokens}
+                                                    </TableCell>
+                                                </TableRow>
+                                            </TableBody>
+                                        </Table>
+                                    </Box>
+                                ) : (
+                                    <Box>
+                                        <Typography variant='overline' style={{ fontWeight: 'bold' }}>
+                                            Tokens
+                                        </Typography>
+                                        <Typography variant='body2'>
+                                            <Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
+                                                <Chip
+                                                    variant='outlined'
+                                                    size='small'
+                                                    label={
+                                                        dialogProps.data.metrics[index]?.totalTokens
+                                                            ? 'Total: ' + dialogProps.data.metrics[index]?.totalTokens
+                                                            : 'Total: N/A'
+                                                    }
+                                                />
+                                                <Chip
+                                                    variant='outlined'
+                                                    size='small'
+                                                    label={
+                                                        dialogProps.data.metrics[index]?.promptTokens
+                                                            ? 'Prompt: ' + dialogProps.data.metrics[index]?.promptTokens
+                                                            : 'Prompt: N/A'
+                                                    }
+                                                />
+                                                <Chip
+                                                    variant='outlined'
+                                                    size='small'
+                                                    label={
+                                                        dialogProps.data.metrics[index]?.completionTokens
+                                                            ? 'Completion: ' + dialogProps.data.metrics[index]?.completionTokens
+                                                            : 'Completion: N/A'
+                                                    }
+                                                />
+                                            </Stack>
+                                        </Typography>
+                                    </Box>
+                                )}
                                 <br />
-                                <Box>
+                                {dialogProps.data.metrics[index]?.nested_metrics ? (
-                                    <Typography variant='overline' style={{ fontWeight: 'bold' }}>
+                                    <Box>
-                                        Cost
+                                        <Typography variant='overline' style={{ fontWeight: 'bold' }}>
-                                    </Typography>
+                                            Cost
-                                    <Typography variant='body2'>
+                                        </Typography>
-                                        <Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
+                                        <Table size='small' style={{ border: '1px solid #ccc' }}>
-                                            <Chip
+                                            <TableHead>
-                                                variant='outlined'
+                                                <TableRow>
-                                                size='small'
+                                                    <TableCell align='left' style={{ fontSize: '11px', fontWeight: 'bold' }}>
-                                                label={
+                                                        Node
-                                                    dialogProps.data.metrics[index]?.totalCost
+                                                    </TableCell>
-                                                        ? 'Total: ' + dialogProps.data.metrics[index]?.totalCost
+                                                    <TableCell align='left' style={{ fontSize: '11px', fontWeight: 'bold' }}>
-                                                        : 'Total: N/A'
+                                                        Provider & Model
-                                                }
+                                                    </TableCell>
-                                            />
+                                                    <TableCell align='right' style={{ fontSize: '11px', width: '15%', fontWeight: 'bold' }}>
-                                            <Chip
+                                                        Input
-                                                variant='outlined'
+                                                    </TableCell>
-                                                size='small'
+                                                    <TableCell align='right' style={{ fontSize: '11px', width: '15%', fontWeight: 'bold' }}>
-                                                label={
+                                                        Output
-                                                    dialogProps.data.metrics[index]?.promptCost
+                                                    </TableCell>
-                                                        ? 'Prompt: ' + dialogProps.data.metrics[index]?.promptCost
+                                                    <TableCell align='right' style={{ fontSize: '11px', width: '15%', fontWeight: 'bold' }}>
-                                                        : 'Completion: N/A'
+                                                        Total
-                                                }
+                                                    </TableCell>
-                                            />
+                                                </TableRow>
-                                            <Chip
+                                            </TableHead>
-                                                variant='outlined'
+                                            <TableBody style={{ fontSize: '8px' }}>
-                                                size='small'
+                                                {dialogProps.data.metrics[index]?.nested_metrics?.map((metric, index) => (
-                                                label={
+                                                    <TableRow key={index}>
-                                                    dialogProps.data.metrics[index]?.completionCost
+                                                        <TableCell component='th' scope='row' style={{ fontSize: '11px' }}>
-                                                        ? 'Completion: ' + dialogProps.data.metrics[index]?.completionCost
+                                                            {metric.nodeLabel}
-                                                        : 'Completion: N/A'
+                                                        </TableCell>
-                                                }
+                                                        <TableCell component='th' scope='row' style={{ fontSize: '11px' }}>
-                                            />
+                                                            {metric.provider} <br />
-                                        </Stack>
+                                                            {metric.model}
-                                    </Typography>
+                                                        </TableCell>
-                                </Box>
+                                                        <TableCell align='right' style={{ fontSize: '11px' }}>
+                                                            {metric.promptCost}
+                                                        </TableCell>
+                                                        <TableCell align='right' style={{ fontSize: '11px' }}>
+                                                            {metric.completionCost}
+                                                        </TableCell>
+                                                        <TableCell align='right' style={{ fontSize: '11px' }}>
+                                                            {metric.totalCost}
+                                                        </TableCell>
+                                                    </TableRow>
+                                                ))}
+                                                <TableRow key={index}>
+                                                    <TableCell
+                                                        align='right'
+                                                        style={{ fontSize: '11px', fontWeight: 'bold' }}
+                                                        component='th'
+                                                        scope='row'
+                                                        colspan={2}
+                                                    >
+                                                        Total
+                                                    </TableCell>
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
+                                                        {dialogProps.data.metrics[index].promptCost}
+                                                    </TableCell>
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
+                                                        {dialogProps.data.metrics[index].completionCost}
+                                                    </TableCell>
+                                                    <TableCell align='right' style={{ fontSize: '11px', fontWeight: 'bold' }}>
+                                                        {dialogProps.data.metrics[index].totalCost}
+                                                    </TableCell>
+                                                </TableRow>
+                                            </TableBody>
+                                        </Table>
+                                    </Box>
+                                ) : (
+                                    <Box>
+                                        <Typography variant='overline' style={{ fontWeight: 'bold' }}>
+                                            Cost
+                                        </Typography>
+                                        <Typography variant='body2'>
+                                            <Stack sx={{ mt: 1, alignItems: 'center', flexWrap: 'wrap' }} flexDirection='row' gap={1}>
+                                                <Chip
+                                                    variant='outlined'
+                                                    size='small'
+                                                    label={
+                                                        dialogProps.data.metrics[index]?.totalCost
+                                                            ? 'Total: ' + dialogProps.data.metrics[index]?.totalCost
+                                                            : 'Total: N/A'
+                                                    }
+                                                />
+                                                <Chip
+                                                    variant='outlined'
+                                                    size='small'
+                                                    label={
+                                                        dialogProps.data.metrics[index]?.promptCost
+                                                            ? 'Prompt: ' + dialogProps.data.metrics[index]?.promptCost
+                                                            : 'Completion: N/A'
+                                                    }
+                                                />
+                                                <Chip
+                                                    variant='outlined'
+                                                    size='small'
+                                                    label={
+                                                        dialogProps.data.metrics[index]?.completionCost
+                                                            ? 'Completion: ' + dialogProps.data.metrics[index]?.completionCost
+                                                            : 'Completion: N/A'
+                                                    }
+                                                />
+                                            </Stack>
+                                        </Typography>
+                                    </Box>
+                                )}
                                 <br />
                                 <Divider />
                                 <br />

packages/ui/src/views/evaluations/MetricsItemCard.jsx

@@ -11,7 +11,7 @@ import SkeletonChatflowCard from '@/ui-component/cards/Skeleton/ChatflowCard'
 const CardWrapper = styled(MainCard)(({ theme }) => ({
     background: theme.palette.card.main,
     color: theme.darkTextPrimary,
-    overflow: 'auto',
+    overflow: 'hidden',
     position: 'relative',
     boxShadow: '0 2px 14px 0 rgb(32 40 45 / 8%)',
     cursor: 'pointer',

packages/ui/src/views/evaluations/index.jsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useState } from 'react'
+import React, { useEffect, useState, useCallback } from 'react'
 import * as PropTypes from 'prop-types'
 import moment from 'moment/moment'
 import { useNavigate } from 'react-router-dom'
@@ -20,7 +20,8 @@ import {
     TableBody,
     TableContainer,
     TableHead,
-    TableRow
+    TableRow,
+    ToggleButton
 } from '@mui/material'
 import { useTheme } from '@mui/material/styles'
 import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
@@ -35,7 +36,6 @@ import useNotifier from '@/utils/useNotifier'
 
 // project
 import MainCard from '@/ui-component/cards/MainCard'
-import { StyledButton } from '@/ui-component/button/StyledButton'
 import { BackdropLoader } from '@/ui-component/loading/BackdropLoader'
 import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog'
 import ErrorBoundary from '@/ErrorBoundary'
@@ -53,7 +53,9 @@ import {
     IconTrash,
     IconX,
     IconChevronsUp,
-    IconChevronsDown
+    IconChevronsDown,
+    IconPlayerPlay,
+    IconPlayerPause
 } from '@tabler/icons-react'
 import empty_evalSVG from '@/assets/images/empty_evals.svg'
 
@@ -79,6 +81,7 @@ const EvalsEvaluation = () => {
     const [loading, setLoading] = useState(false)
     const [isTableLoading, setTableLoading] = useState(false)
     const [selected, setSelected] = useState([])
+    const [autoRefresh, setAutoRefresh] = useState(false)
 
     const onSelectAllClick = (event) => {
         if (event.target.checked) {
@@ -240,14 +243,34 @@ const EvalsEvaluation = () => {
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [createNewEvaluation.error])
 
-    const onRefresh = () => {
+    const onRefresh = useCallback(() => {
         getAllEvaluations.request()
-    }
+    }, [getAllEvaluations])
 
     useEffect(() => {
         setTableLoading(getAllEvaluations.loading)
     }, [getAllEvaluations.loading])
 
+    useEffect(() => {
+        let intervalId = null
+
+        if (autoRefresh) {
+            intervalId = setInterval(() => {
+                onRefresh()
+            }, 5000)
+        }
+
+        return () => {
+            if (intervalId) {
+                clearInterval(intervalId)
+            }
+        }
+    }, [autoRefresh, onRefresh])
+
+    const toggleAutoRefresh = () => {
+        setAutoRefresh(!autoRefresh)
+    }
+
     return (
         <>
             <MainCard>
@@ -256,15 +279,52 @@ const EvalsEvaluation = () => {
                 ) : (
                     <Stack flexDirection='column' sx={{ gap: 3 }}>
                         <ViewHeader isBackButton={false} isEditButton={false} search={false} title={'Evaluations'} description=''>
-                            <StyledButton
+                            <ToggleButton
-                                color='secondary'
+                                value='auto-refresh'
-                                variant='outlined'
+                                selected={autoRefresh}
-                                sx={{ borderRadius: 2, height: '100%' }}
+                                onChange={toggleAutoRefresh}
-                                onClick={onRefresh}
+                                size='small'
-                                startIcon={<IconRefresh />}
+                                sx={{
+                                    borderRadius: 2,
+                                    height: '100%',
+                                    backgroundColor: 'transparent',
+                                    color: autoRefresh ? '#ff9800' : '#4caf50',
+                                    border: '1px solid transparent',
+                                    '&:hover': {
+                                        backgroundColor: 'rgba(0, 0, 0, 0.04)',
+                                        color: autoRefresh ? '#f57c00' : '#388e3c',
+                                        border: '1px solid transparent'
+                                    },
+                                    '&.Mui-selected': {
+                                        backgroundColor: 'transparent',
+                                        color: '#ff9800',
+                                        border: '1px solid transparent',
+                                        '&:hover': {
+                                            backgroundColor: 'rgba(0, 0, 0, 0.04)',
+                                            color: '#f57c00',
+                                            border: '1px solid transparent'
+                                        }
+                                    }
+                                }}
+                                title={autoRefresh ? 'Disable auto-refresh' : 'Enable auto-refresh (every 5s)'}
                             >
-                                Refresh
+                                {autoRefresh ? <IconPlayerPause /> : <IconPlayerPlay />}
-                            </StyledButton>
+                            </ToggleButton>
+                            <IconButton
+                                sx={{
+                                    borderRadius: 2,
+                                    height: '100%',
+                                    color: theme.palette.secondary.main,
+                                    '&:hover': {
+                                        backgroundColor: 'rgba(0, 0, 0, 0.04)',
+                                        color: theme.palette.secondary.dark
+                                    }
+                                }}
+                                onClick={onRefresh}
+                                title='Refresh'
+                            >
+                                <IconRefresh />
+                            </IconButton>
                             <StyledPermissionButton
                                 permissionId={'evaluations:create'}
                                 sx={{ borderRadius: 2, height: '100%' }}
@@ -327,7 +387,7 @@ const EvalsEvaluation = () => {
                                             <TableCell>Latest Version</TableCell>
                                             <TableCell>Average Metrics</TableCell>
                                             <TableCell>Last Evaluated</TableCell>
-                                            <TableCell>Chatflow(s)</TableCell>
+                                            <TableCell>Flow(s)</TableCell>
                                             <TableCell>Dataset</TableCell>
                                             <TableCell> </TableCell>
                                         </TableRow>
@@ -438,7 +498,7 @@ function EvaluationRunRow(props) {
     }
 
     const goToDataset = (id) => {
-        navigate(`/dataset_rows/${id}`)
+        window.open(`/dataset_rows/${id}`, '_blank')
     }
 
     const onSelectAllChildClick = (event) => {
@@ -513,10 +573,6 @@ function EvaluationRunRow(props) {
         }
     }
 
-    const goToCanvas = (id) => {
-        navigate(`/canvas/${id}`)
-    }
-
     const getStatusColor = (status) => {
         switch (status) {
             case 'pending':
@@ -619,16 +675,11 @@ function EvaluationRunRow(props) {
                         {props.item?.usedFlows?.map((usedFlow, index) => (
                             <Chip
                                 key={index}
-                                clickable
                                 style={{
                                     width: 'max-content',
-                                    borderRadius: '25px',
+                                    borderRadius: '25px'
-                                    boxShadow: props.customization.isDarkMode
-                                        ? '0 2px 14px 0 rgb(255 255 255 / 10%)'
-                                        : '0 2px 14px 0 rgb(32 40 45 / 10%)'
                                 }}
                                 label={usedFlow}
-                                onClick={() => goToCanvas(props.item.chatIds[index])}
                             ></Chip>
                         ))}
                     </Stack>
@@ -637,6 +688,7 @@ function EvaluationRunRow(props) {
                     <Chip
                         clickable
                         style={{
+                            border: 'none',
                             width: 'max-content',
                             borderRadius: '25px',
                             boxShadow: props.customization.isDarkMode