From e17994d8fe86717780d5bc01267bb541292fedf1 Mon Sep 17 00:00:00 2001 From: Vinod Kiran Date: Tue, 10 Jun 2025 20:41:22 +0530 Subject: [PATCH] Evaluations for Agentflows v2 & Assistants (#4589) * New Feature: Evaluations for AgentFlow v2 * New Feature: Evaluations for Assistants and minor tweaks on other evaluations. * do not store messages during evaluation for agent flows. * common cost formatting * moving the category names to description (in create dialog) and adjusting the side drawer label * lint fixes * Enhancement: Add auto-refresh toggle for evaluations with 5-second interval and adjust grid item size for metrics display. * 1) chatflow types are stored in additional config 2) messages are now stored with type "Evaluations" 3) Message Dialog has a new Type in the ChatType Filter Dropdown 4) Chatflow badges on the view page, have the right canvas URL 5) outdated API returns chatflow type along with the stale indicator. 6) UI - Flow Indicator Icons are shown in the Chatflows Used chips & side drawer * Refactor JWT error handling to return 401 status for expired refresh tokens. Update chat message ID assignment to remove UUID fallback. Enhance ViewMessagesDialog to set default chat type filters and implement a new method for determining chat type sources. Modify EvalsResultDialog to open links in a new tab and adjust icon sizes for better consistency. Clean up unused imports in EvaluationResultSideDrawer. * handling on Click for deleted flows and minor code cleanup * evals ui fix * Refactor evaluation service to improve error handling and data parsing. Update additionalConfig handling to default to an empty object if not present. Enhance type definitions for better clarity. Adjust MetricsItemCard to prevent overflow and improve layout consistency. --------- Co-authored-by: Henry --- .../components/evaluation/EvaluationRunner.ts | 64 +++- .../enterprise/middleware/passport/index.ts | 8 +- .../services/evaluations/CostCalculator.ts | 39 +- .../server/src/services/evaluations/index.ts | 160 ++++++-- packages/server/src/utils/buildAgentflow.ts | 6 +- packages/server/src/utils/buildChatflow.ts | 26 +- .../dialog/ViewMessagesDialog.jsx | 21 +- .../evaluations/CreateEvaluationDialog.jsx | 104 ++++- .../views/evaluations/EvalsResultDialog.jsx | 23 +- .../views/evaluations/EvaluationResult.jsx | 79 +++- .../EvaluationResultSideDrawer.jsx | 354 +++++++++++++----- .../src/views/evaluations/MetricsItemCard.jsx | 2 +- packages/ui/src/views/evaluations/index.jsx | 104 +++-- 13 files changed, 766 insertions(+), 224 deletions(-) diff --git a/packages/components/evaluation/EvaluationRunner.ts b/packages/components/evaluation/EvaluationRunner.ts index 73f22b70c..acde79446 100644 --- a/packages/components/evaluation/EvaluationRunner.ts +++ b/packages/components/evaluation/EvaluationRunner.ts @@ -6,6 +6,26 @@ import { getModelConfigByModelName, MODEL_TYPE } from '../src/modelLoader' export class EvaluationRunner { static metrics = new Map() + + static getCostMetrics = async (selectedProvider: string, selectedModel: string) => { + let modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, selectedProvider, selectedModel) + if (modelConfig) { + if (modelConfig['cost_values']) { + return modelConfig.cost_values + } + return { cost_values: modelConfig } + } else { + modelConfig = await getModelConfigByModelName(MODEL_TYPE.LLM, selectedProvider, selectedModel) + if (modelConfig) { + if (modelConfig['cost_values']) { + return modelConfig.cost_values + } + return { cost_values: modelConfig } + } + } + return undefined + } + static async getAndDeleteMetrics(id: string) { const val = EvaluationRunner.metrics.get(id) if (val) { @@ -34,11 +54,8 @@ export class EvaluationRunner { } } } - let modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, selectedProvider, selectedModel) - if (modelConfig) { - val.push(JSON.stringify({ cost_values: modelConfig })) - } else { - modelConfig = await getModelConfigByModelName(MODEL_TYPE.LLM, selectedProvider, selectedModel) + if (selectedProvider && selectedModel) { + const modelConfig = await EvaluationRunner.getCostMetrics(selectedProvider, selectedModel) if (modelConfig) { val.push(JSON.stringify({ cost_values: modelConfig })) } @@ -116,6 +133,40 @@ export class EvaluationRunner { } try { let response = await axios.post(`${this.baseURL}/api/v1/prediction/${chatflowId}`, postData, axiosConfig) + let agentFlowMetrics: any[] = [] + if (response?.data?.agentFlowExecutedData) { + for (let i = 0; i < response.data.agentFlowExecutedData.length; i++) { + const agentFlowExecutedData = response.data.agentFlowExecutedData[i] + const input_tokens = agentFlowExecutedData?.data?.output?.usageMetadata?.input_tokens || 0 + const output_tokens = agentFlowExecutedData?.data?.output?.usageMetadata?.output_tokens || 0 + const total_tokens = + agentFlowExecutedData?.data?.output?.usageMetadata?.total_tokens || input_tokens + output_tokens + const metrics: any = { + promptTokens: input_tokens, + completionTokens: output_tokens, + totalTokens: total_tokens, + provider: + agentFlowExecutedData.data?.input?.llmModelConfig?.llmModel || + agentFlowExecutedData.data?.input?.agentModelConfig?.agentModel, + model: + agentFlowExecutedData.data?.input?.llmModelConfig?.modelName || + agentFlowExecutedData.data?.input?.agentModelConfig?.modelName, + nodeLabel: agentFlowExecutedData?.nodeLabel, + nodeId: agentFlowExecutedData?.nodeId + } + if (metrics.provider && metrics.model) { + const modelConfig = await EvaluationRunner.getCostMetrics(metrics.provider, metrics.model) + if (modelConfig) { + metrics.cost_values = { + input_cost: (modelConfig.cost_values.input_cost || 0) * (input_tokens / 1000), + output_cost: (modelConfig.cost_values.output_cost || 0) * (output_tokens / 1000) + } + metrics.cost_values.total_cost = metrics.cost_values.input_cost + metrics.cost_values.output_cost + } + } + agentFlowMetrics.push(metrics) + } + } const endTime = performance.now() const timeTaken = (endTime - startTime).toFixed(2) if (response?.data?.metrics) { @@ -130,6 +181,9 @@ export class EvaluationRunner { } ] } + if (agentFlowMetrics.length > 0) { + runData.nested_metrics = agentFlowMetrics + } runData.status = 'complete' let resultText = '' if (response.data.text) resultText = response.data.text diff --git a/packages/server/src/enterprise/middleware/passport/index.ts b/packages/server/src/enterprise/middleware/passport/index.ts index 3381018e5..b93d24132 100644 --- a/packages/server/src/enterprise/middleware/passport/index.ts +++ b/packages/server/src/enterprise/middleware/passport/index.ts @@ -218,7 +218,7 @@ export const initializeJwtCookieMiddleware = async (app: express.Application, id if (!refreshToken) return res.sendStatus(401) jwt.verify(refreshToken, jwtRefreshSecret, async (err: any, payload: any) => { - if (err || !payload) return res.status(403).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED }) + if (err || !payload) return res.status(401).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED }) // @ts-ignore const loggedInUser = req.user as LoggedInUser let isSSO = false @@ -227,16 +227,16 @@ export const initializeJwtCookieMiddleware = async (app: express.Application, id try { newTokenResponse = await identityManager.getRefreshToken(loggedInUser.ssoProvider, loggedInUser.ssoRefreshToken) if (newTokenResponse.error) { - return res.status(403).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED }) + return res.status(401).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED }) } isSSO = true } catch (error) { - return res.status(403).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED }) + return res.status(401).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED }) } } const meta = decryptToken(payload.meta) if (!meta) { - return res.status(403).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED }) + return res.status(401).json({ message: ErrorMessage.REFRESH_TOKEN_EXPIRED }) } if (isSSO) { loggedInUser.ssoToken = newTokenResponse.access_token diff --git a/packages/server/src/services/evaluations/CostCalculator.ts b/packages/server/src/services/evaluations/CostCalculator.ts index 8813ade95..4013706ef 100644 --- a/packages/server/src/services/evaluations/CostCalculator.ts +++ b/packages/server/src/services/evaluations/CostCalculator.ts @@ -18,39 +18,29 @@ export const calculateCost = (metricsArray: ICommonObject[]) => { let completionTokensCost: string = '0' let totalTokensCost = '0' if (metric.cost_values) { - const costValues = metric.cost_values + let costValues: any = {} + if (metric.cost_values?.cost_values) { + costValues = metric.cost_values.cost_values + } else { + costValues = metric.cost_values + } + if (costValues.total_price > 0) { let cost = costValues.total_cost * (totalTokens / 1000) - if (cost < 0.01) { - totalTokensCost = '$ <0.01' - } else { - totalTokensCost = '$ ' + cost.toFixed(fractionDigits) - } + totalTokensCost = formatCost(cost) } else { let totalCost = 0 if (promptTokens) { const cost = costValues.input_cost * (promptTokens / 1000) totalCost += cost - if (cost < 0.01) { - promptTokensCost = '$ <0.01' - } else { - promptTokensCost = '$ ' + cost.toFixed(fractionDigits) - } + promptTokensCost = formatCost(cost) } if (completionTokens) { const cost = costValues.output_cost * (completionTokens / 1000) totalCost += cost - if (cost < 0.01) { - completionTokensCost = '$ <0.01' - } else { - completionTokensCost = '$ ' + cost.toFixed(fractionDigits) - } - } - if (totalCost < 0.01) { - totalTokensCost = '$ <0.01' - } else { - totalTokensCost = '$ ' + totalCost.toFixed(fractionDigits) + completionTokensCost = formatCost(cost) } + totalTokensCost = formatCost(totalCost) } } metric['totalCost'] = totalTokensCost @@ -58,3 +48,10 @@ export const calculateCost = (metricsArray: ICommonObject[]) => { metric['completionCost'] = completionTokensCost } } + +export const formatCost = (cost: number) => { + if (cost == 0) { + return '$ 0' + } + return cost < 0.01 ? '$ <0.01' : '$ ' + cost.toFixed(fractionDigits) +} diff --git a/packages/server/src/services/evaluations/index.ts b/packages/server/src/services/evaluations/index.ts index acca639cc..2d44b3ce2 100644 --- a/packages/server/src/services/evaluations/index.ts +++ b/packages/server/src/services/evaluations/index.ts @@ -15,10 +15,11 @@ import { getAppVersion } from '../../utils' import { In } from 'typeorm' import { getWorkspaceSearchOptions } from '../../enterprise/utils/ControllerServiceUtils' import { v4 as uuidv4 } from 'uuid' -import { calculateCost } from './CostCalculator' +import { calculateCost, formatCost } from './CostCalculator' import { runAdditionalEvaluators } from './EvaluatorRunner' import evaluatorsService from '../evaluator' import { LLMEvaluationRunner } from './LLMEvaluationRunner' +import { Assistant } from '../../database/entities/Assistant' const runAgain = async (id: string, baseURL: string, orgId: string) => { try { @@ -27,7 +28,7 @@ const runAgain = async (id: string, baseURL: string, orgId: string) => { id: id }) if (!evaluation) throw new Error(`Evaluation ${id} not found`) - const additionalConfig: any = JSON.parse(evaluation.additionalConfig) + const additionalConfig = evaluation.additionalConfig ? JSON.parse(evaluation.additionalConfig) : {} const data: ICommonObject = { chatflowId: evaluation.chatflowId, chatflowName: evaluation.chatflowName, @@ -35,7 +36,8 @@ const runAgain = async (id: string, baseURL: string, orgId: string) => { datasetId: evaluation.datasetId, evaluationType: evaluation.evaluationType, selectedSimpleEvaluators: JSON.stringify(additionalConfig.simpleEvaluators), - datasetAsOneConversation: additionalConfig.datasetAsOneConversation + datasetAsOneConversation: additionalConfig.datasetAsOneConversation, + chatflowType: JSON.stringify(additionalConfig.chatflowTypes ? additionalConfig.chatflowTypes : []) } data.name = evaluation.name data.workspaceId = evaluation.workspaceId @@ -69,7 +71,8 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str const row = appServer.AppDataSource.getRepository(Evaluation).create(newEval) row.average_metrics = JSON.stringify({}) - const additionalConfig: any = { + const additionalConfig: ICommonObject = { + chatflowTypes: body.chatflowType ? JSON.parse(body.chatflowType) : [], datasetAsOneConversation: body.datasetAsOneConversation, simpleEvaluators: body.selectedSimpleEvaluators.length > 0 ? JSON.parse(body.selectedSimpleEvaluators) : [] } @@ -152,7 +155,7 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str let evalMetrics = { passCount: 0, failCount: 0, errorCount: 0 } evalRunner .runEvaluations(data) - .then(async (result: any) => { + .then(async (result) => { let totalTime = 0 // let us assume that the eval is successful let allRowsSuccessful = true @@ -171,8 +174,48 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str totalTime += parseFloat(evaluationRow.latency) let metricsObjFromRun: ICommonObject = {} + let nested_metrics = evaluationRow.nested_metrics + + let promptTokens = 0, + completionTokens = 0, + totalTokens = 0 + let inputCost = 0, + outputCost = 0, + totalCost = 0 + if (nested_metrics && nested_metrics.length > 0) { + for (let i = 0; i < nested_metrics.length; i++) { + const nested_metric = nested_metrics[i] + if (nested_metric.model && nested_metric.promptTokens > 0) { + promptTokens += nested_metric.promptTokens + completionTokens += nested_metric.completionTokens + totalTokens += nested_metric.totalTokens + + inputCost += nested_metric.cost_values.input_cost + outputCost += nested_metric.cost_values.output_cost + totalCost += nested_metric.cost_values.total_cost + + nested_metric['totalCost'] = formatCost(nested_metric.cost_values.total_cost) + nested_metric['promptCost'] = formatCost(nested_metric.cost_values.input_cost) + nested_metric['completionCost'] = formatCost(nested_metric.cost_values.output_cost) + } + } + nested_metrics = nested_metrics.filter((metric: any) => { + return metric.model && metric.provider + }) + } const metrics = evaluationRow.metrics if (metrics) { + if (nested_metrics && nested_metrics.length > 0) { + metrics.push({ + promptTokens: promptTokens, + completionTokens: completionTokens, + totalTokens: totalTokens, + totalCost: formatCost(totalCost), + promptCost: formatCost(inputCost), + completionCost: formatCost(outputCost) + }) + metricsObjFromRun.nested_metrics = nested_metrics + } metrics.map((metric: any) => { if (metric) { const json = typeof metric === 'object' ? metric : JSON.parse(metric) @@ -211,7 +254,7 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str if (body.evaluationType === 'llm') { resultRow.llmConfig = additionalConfig.llmConfig resultRow.LLMEvaluators = body.selectedLLMEvaluators.length > 0 ? JSON.parse(body.selectedLLMEvaluators) : [] - const llmEvaluatorMap: any = [] + const llmEvaluatorMap: { evaluatorId: string; evaluator: any }[] = [] for (let i = 0; i < resultRow.LLMEvaluators.length; i++) { const evaluatorId = resultRow.LLMEvaluators[i] const evaluator = await evaluatorsService.getEvaluator(evaluatorId) @@ -243,23 +286,27 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str } appServer.AppDataSource.getRepository(Evaluation) .findOneBy({ id: newEvaluation.id }) - .then((evaluation: any) => { - evaluation.status = allRowsSuccessful ? EvaluationStatus.COMPLETED : EvaluationStatus.ERROR - evaluation.average_metrics = JSON.stringify({ - averageLatency: (totalTime / result.rows.length).toFixed(3), - totalRuns: result.rows.length, - ...evalMetrics, - passPcnt: passPercent.toFixed(2) - }) - appServer.AppDataSource.getRepository(Evaluation).save(evaluation) + .then((evaluation) => { + if (evaluation) { + evaluation.status = allRowsSuccessful ? EvaluationStatus.COMPLETED : EvaluationStatus.ERROR + evaluation.average_metrics = JSON.stringify({ + averageLatency: (totalTime / result.rows.length).toFixed(3), + totalRuns: result.rows.length, + ...evalMetrics, + passPcnt: passPercent.toFixed(2) + }) + appServer.AppDataSource.getRepository(Evaluation).save(evaluation) + } }) } catch (error) { //update the evaluation with status as error appServer.AppDataSource.getRepository(Evaluation) .findOneBy({ id: newEvaluation.id }) - .then((evaluation: any) => { - evaluation.status = EvaluationStatus.ERROR - appServer.AppDataSource.getRepository(Evaluation).save(evaluation) + .then((evaluation) => { + if (evaluation) { + evaluation.status = EvaluationStatus.ERROR + appServer.AppDataSource.getRepository(Evaluation).save(evaluation) + } }) } }) @@ -268,12 +315,14 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str console.error('Error running evaluations:', getErrorMessage(error)) appServer.AppDataSource.getRepository(Evaluation) .findOneBy({ id: newEvaluation.id }) - .then((evaluation: any) => { - evaluation.status = EvaluationStatus.ERROR - evaluation.average_metrics = JSON.stringify({ - error: getErrorMessage(error) - }) - appServer.AppDataSource.getRepository(Evaluation).save(evaluation) + .then((evaluation) => { + if (evaluation) { + evaluation.status = EvaluationStatus.ERROR + evaluation.average_metrics = JSON.stringify({ + error: getErrorMessage(error) + }) + appServer.AppDataSource.getRepository(Evaluation).save(evaluation) + } }) .catch((dbError) => { console.error('Error updating evaluation status:', getErrorMessage(dbError)) @@ -378,18 +427,31 @@ const isOutdated = async (id: string) => { returnObj.dataset = dataset } } else { - returnObj.errors.push(`Dataset ${evaluation.datasetName} not found`) + returnObj.errors.push({ + message: `Dataset ${evaluation.datasetName} not found`, + id: evaluation.datasetId + }) isOutdated = true } - const chatflows = JSON.parse(evaluation.chatflowId) - const chatflowNames = JSON.parse(evaluation.chatflowName) - - for (let i = 0; i < chatflows.length; i++) { + const chatflowIds = evaluation.chatflowId ? JSON.parse(evaluation.chatflowId) : [] + const chatflowNames = evaluation.chatflowName ? JSON.parse(evaluation.chatflowName) : [] + const chatflowTypes = evaluation.additionalConfig ? JSON.parse(evaluation.additionalConfig).chatflowTypes : [] + for (let i = 0; i < chatflowIds.length; i++) { + // check for backward compatibility, as previous versions did not the types in additionalConfig + if (chatflowTypes && chatflowTypes.length >= 0) { + if (chatflowTypes[i] === 'Custom Assistant') { + // if the chatflow type is custom assistant, then we should NOT check in the chatflows table + continue + } + } const chatflow = await appServer.AppDataSource.getRepository(ChatFlow).findOneBy({ - id: chatflows[i] + id: chatflowIds[i] }) if (!chatflow) { - returnObj.errors.push(`Chatflow ${chatflowNames[i]} not found`) + returnObj.errors.push({ + message: `Chatflow ${chatflowNames[i]} not found`, + id: chatflowIds[i] + }) isOutdated = true } else { const chatflowLastUpdated = chatflow.updatedDate.getTime() @@ -397,12 +459,42 @@ const isOutdated = async (id: string) => { isOutdated = true returnObj.chatflows.push({ chatflowName: chatflowNames[i], - chatflowId: chatflows[i], + chatflowId: chatflowIds[i], + chatflowType: chatflow.type === 'AGENTFLOW' ? 'Agentflow v2' : 'Chatflow', isOutdated: true }) } } } + if (chatflowTypes && chatflowTypes.length > 0) { + for (let i = 0; i < chatflowIds.length; i++) { + if (chatflowTypes[i] !== 'Custom Assistant') { + // if the chatflow type is NOT custom assistant, then bail out for this item + continue + } + const assistant = await appServer.AppDataSource.getRepository(Assistant).findOneBy({ + id: chatflowIds[i] + }) + if (!assistant) { + returnObj.errors.push({ + message: `Custom Assistant ${chatflowNames[i]} not found`, + id: chatflowIds[i] + }) + isOutdated = true + } else { + const chatflowLastUpdated = assistant.updatedDate.getTime() + if (chatflowLastUpdated > evaluationRunDate) { + isOutdated = true + returnObj.chatflows.push({ + chatflowName: chatflowNames[i], + chatflowId: chatflowIds[i], + chatflowType: 'Custom Assistant', + isOutdated: true + }) + } + } + } + } returnObj.isOutdated = isOutdated return returnObj } catch (error) { @@ -424,7 +516,7 @@ const getEvaluation = async (id: string) => { where: { evaluationId: id } }) const versions = (await getVersions(id)).versions - const versionNo = versions.findIndex((version: any) => version.id === id) + 1 + const versionNo = versions.findIndex((version) => version.id === id) + 1 return { ...evaluation, versionCount: versionCount, @@ -451,7 +543,7 @@ const getVersions = async (id: string) => { runDate: 'ASC' } }) - const returnResults: any[] = [] + const returnResults: { id: string; runDate: Date; version: number }[] = [] versions.map((version, index) => { returnResults.push({ id: version.id, diff --git a/packages/server/src/utils/buildAgentflow.ts b/packages/server/src/utils/buildAgentflow.ts index 378054979..40953bf3d 100644 --- a/packages/server/src/utils/buildAgentflow.ts +++ b/packages/server/src/utils/buildAgentflow.ts @@ -1805,7 +1805,7 @@ export const executeAgentFlow = async ({ role: 'userMessage', content: finalUserInput, chatflowid, - chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, + chatType: evaluationRunId ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, chatId, sessionId, createdDate: userMessageDateTime, @@ -1820,7 +1820,7 @@ export const executeAgentFlow = async ({ role: 'apiMessage', content: content, chatflowid, - chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, + chatType: evaluationRunId ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, chatId, sessionId, executionId: newExecution.id @@ -1856,7 +1856,7 @@ export const executeAgentFlow = async ({ version: await getAppVersion(), chatflowId: chatflowid, chatId, - type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, + type: evaluationRunId ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, flowGraph: getTelemetryFlowObj(nodes, edges) }, orgId diff --git a/packages/server/src/utils/buildChatflow.ts b/packages/server/src/utils/buildChatflow.ts index b138922ea..df5b64e64 100644 --- a/packages/server/src/utils/buildChatflow.ts +++ b/packages/server/src/utils/buildChatflow.ts @@ -551,7 +551,7 @@ export const executeFlow = async ({ role: 'userMessage', content: incomingInput.question, chatflowid: agentflow.id, - chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, + chatType: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, chatId, memoryType, sessionId, @@ -566,7 +566,7 @@ export const executeFlow = async ({ role: 'apiMessage', content: finalResult, chatflowid: agentflow.id, - chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, + chatType: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, chatId, memoryType, sessionId @@ -598,7 +598,7 @@ export const executeFlow = async ({ version: await getAppVersion(), agentflowId: agentflow.id, chatId, - type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, + type: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, flowGraph: getTelemetryFlowObj(nodes, edges) }, orgId @@ -807,7 +807,7 @@ export const executeFlow = async ({ version: await getAppVersion(), chatflowId: chatflowid, chatId, - type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, + type: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, flowGraph: getTelemetryFlowObj(nodes, edges) }, orgId @@ -905,17 +905,17 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals const isTool = req.get('flowise-tool') === 'true' const isEvaluation: boolean = req.headers['X-Flowise-Evaluation'] || req.body.evaluation let evaluationRunId = '' - if (isEvaluation) { - evaluationRunId = req.body.evaluationRunId - if (evaluationRunId) { - const newEval = { - evaluation: { - status: true, - evaluationRunId - } + evaluationRunId = req.body.evaluationRunId + if (isEvaluation && chatflow.type !== 'AGENTFLOW' && req.body.evaluationRunId) { + // this is needed for the collection of token metrics for non-agent flows, + // for agentflows the execution trace has the info needed + const newEval = { + evaluation: { + status: true, + evaluationRunId } - chatflow.analytic = JSON.stringify(newEval) } + chatflow.analytic = JSON.stringify(newEval) } try { diff --git a/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx b/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx index de36dd17b..7e336bfa4 100644 --- a/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx +++ b/packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx @@ -149,7 +149,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { const [sourceDialogProps, setSourceDialogProps] = useState({}) const [hardDeleteDialogOpen, setHardDeleteDialogOpen] = useState(false) const [hardDeleteDialogProps, setHardDeleteDialogProps] = useState({}) - const [chatTypeFilter, setChatTypeFilter] = useState([]) + const [chatTypeFilter, setChatTypeFilter] = useState(['INTERNAL', 'EXTERNAL']) const [feedbackTypeFilter, setFeedbackTypeFilter] = useState([]) const [startDate, setStartDate] = useState(new Date(new Date().setMonth(new Date().getMonth() - 1))) const [endDate, setEndDate] = useState(new Date()) @@ -310,6 +310,15 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { } } + const getChatType = (chatType) => { + if (chatType === 'INTERNAL') { + return 'UI' + } else if (chatType === 'EVALUATION') { + return 'Evaluation' + } + return 'API/Embed' + } + const exportMessages = async () => { if (!storagePath && getStoragePathFromServer.data) { storagePath = getStoragePathFromServer.data.storagePath @@ -356,7 +365,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) { obj[chatPK] = { id: chatmsg.chatId, - source: chatmsg.chatType === 'INTERNAL' ? 'UI' : 'API/Embed', + source: getChatType(chatmsg.chatType), sessionId: chatmsg.sessionId ?? null, memoryType: chatmsg.memoryType ?? null, email: chatmsg.leadEmail ?? null, @@ -716,7 +725,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { setChatLogs([]) setAllChatLogs([]) setChatMessages([]) - setChatTypeFilter([]) + setChatTypeFilter(['INTERNAL', 'EXTERNAL']) setFeedbackTypeFilter([]) setSelectedMessageIndex(0) setSelectedChatId('') @@ -880,6 +889,10 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { { label: 'API/Embed', name: 'EXTERNAL' + }, + { + label: 'Evaluations', + name: 'EVALUATION' } ]} onSelect={(newValue) => onChatTypeSelected(newValue)} @@ -1016,7 +1029,7 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { )} {chatMessages[1].chatType && (
- Source: {chatMessages[1].chatType === 'INTERNAL' ? 'UI' : 'API/Embed'} + Source: {getChatType(chatMessages[1].chatType)}
)} {chatMessages[1].memoryType && ( diff --git a/packages/ui/src/views/evaluations/CreateEvaluationDialog.jsx b/packages/ui/src/views/evaluations/CreateEvaluationDialog.jsx index a18453989..1eb8df654 100644 --- a/packages/ui/src/views/evaluations/CreateEvaluationDialog.jsx +++ b/packages/ui/src/views/evaluations/CreateEvaluationDialog.jsx @@ -21,7 +21,8 @@ import { Switch, StepLabel, IconButton, - FormControlLabel + FormControlLabel, + Checkbox } from '@mui/material' import { useTheme } from '@mui/material/styles' @@ -42,6 +43,7 @@ import useApi from '@/hooks/useApi' import datasetsApi from '@/api/dataset' import evaluatorsApi from '@/api/evaluators' import nodesApi from '@/api/nodes' +import assistantsApi from '@/api/assistants' // utils import useNotifier from '@/utils/useNotifier' @@ -57,14 +59,18 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => { useNotifier() const getAllChatflowsApi = useApi(chatflowsApi.getAllChatflows) + const getAllAgentflowsApi = useApi(chatflowsApi.getAllAgentflows) + const getAllDatasetsApi = useApi(datasetsApi.getAllDatasets) const getAllEvaluatorsApi = useApi(evaluatorsApi.getAllEvaluators) const getNodesByCategoryApi = useApi(nodesApi.getNodesByCategory) const getModelsApi = useApi(nodesApi.executeNodeLoadMethod) + const getAssistantsApi = useApi(assistantsApi.getAllAssistants) const [chatflow, setChatflow] = useState([]) const [dataset, setDataset] = useState('') const [datasetAsOneConversation, setDatasetAsOneConversation] = useState(false) + const [flowTypes, setFlowTypes] = useState([]) const [flows, setFlows] = useState([]) const [datasets, setDatasets] = useState([]) @@ -163,6 +169,10 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => { for (let i = 0; i < selectedChatflows.length; i += 1) { selectedChatflowNames.push(flows.find((f) => f.name === selectedChatflows[i])?.label) } + const selectedChatflowTypes = [] + for (let i = 0; i < selectedChatflows.length; i += 1) { + selectedChatflowTypes.push(flows.find((f) => f.name === selectedChatflows[i])?.type) + } const chatflowName = JSON.stringify(selectedChatflowNames) const datasetName = datasets.find((f) => f.name === dataset)?.label const obj = { @@ -173,6 +183,7 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => { datasetName: datasetName, chatflowId: chatflow, chatflowName: chatflowName, + chatflowType: JSON.stringify(selectedChatflowTypes), selectedSimpleEvaluators: selectedSimpleEvaluators, selectedLLMEvaluators: selectedLLMEvaluators, model: selectedModel, @@ -216,6 +227,8 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => { getNodesByCategoryApi.request('Chat Models') if (flows.length === 0) { getAllChatflowsApi.request() + getAssistantsApi.request('CUSTOM') + getAllAgentflowsApi.request('AGENTFLOW') } if (datasets.length === 0) { getAllDatasetsApi.request() @@ -225,23 +238,18 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => { }, []) useEffect(() => { - if (getAllChatflowsApi.data) { + if (getAllAgentflowsApi.data && getAllChatflowsApi.data && getAssistantsApi.data) { try { - const chatflows = getAllChatflowsApi.data - let flowNames = [] - for (let i = 0; i < chatflows.length; i += 1) { - const flow = chatflows[i] - flowNames.push({ - label: flow.name, - name: flow.id - }) - } - setFlows(flowNames) + const agentFlows = populateFlowNames(getAllAgentflowsApi.data, 'Agentflow v2') + const chatFlows = populateFlowNames(getAllChatflowsApi.data, 'Chatflow') + const assistants = populateAssistants(getAssistantsApi.data) + setFlows([...agentFlows, ...chatFlows, ...assistants]) + setFlowTypes(['Agentflow v2', 'Chatflow', 'Custom Assistant']) } catch (e) { console.error(e) } } - }, [getAllChatflowsApi.data]) + }, [getAllAgentflowsApi.data, getAllChatflowsApi.data, getAssistantsApi.data]) useEffect(() => { if (getNodesByCategoryApi.data) { @@ -337,6 +345,44 @@ const CreateEvaluationDialog = ({ show, dialogProps, onCancel, onConfirm }) => { if (llm !== 'no_grading') getModelsApi.request(llm, { loadMethod: 'listModels' }) } + const onChangeFlowType = (flowType) => { + const selected = flowType.target.checked + const flowTypeValue = flowType.target.value + if (selected) { + setFlowTypes([...flowTypes, flowTypeValue]) + } else { + setFlowTypes(flowTypes.filter((f) => f !== flowTypeValue)) + } + } + + const populateFlowNames = (data, type) => { + let flowNames = [] + for (let i = 0; i < data.length; i += 1) { + const flow = data[i] + flowNames.push({ + label: flow.name, + name: flow.id, + type: type, + description: type + }) + } + return flowNames + } + + const populateAssistants = (assistants) => { + let assistantNames = [] + for (let i = 0; i < assistants.length; i += 1) { + const assistant = assistants[i] + assistantNames.push({ + label: JSON.parse(assistant.details).name || '', + name: assistant.id, + type: 'Custom Assistant', + description: 'Custom Assistant' + }) + } + return assistantNames + } + const component = show ? ( { Treat all dataset rows as one conversation ? } value={datasetAsOneConversation} onChange={() => setDatasetAsOneConversation(!datasetAsOneConversation)} /> - - Chatflow(s) to Evaluate * - +
+ + Select your flows to Evaluate +  * + + + {' '} + Chatflows + {' '} + Agentflows (v2) + {' '} + Custom Assistants + +
flowTypes.includes(f.type))} onSelect={(newValue) => setChatflow(newValue)} value={chatflow ?? chatflow ?? 'choose an option'} /> diff --git a/packages/ui/src/views/evaluations/EvalsResultDialog.jsx b/packages/ui/src/views/evaluations/EvalsResultDialog.jsx index 3fb013f0d..a74c2b76e 100644 --- a/packages/ui/src/views/evaluations/EvalsResultDialog.jsx +++ b/packages/ui/src/views/evaluations/EvalsResultDialog.jsx @@ -2,7 +2,6 @@ import React from 'react' import { createPortal } from 'react-dom' import PropTypes from 'prop-types' import { useSelector } from 'react-redux' -import { useNavigate } from 'react-router-dom' // Material import { @@ -36,7 +35,6 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) = const portalElement = document.getElementById('portal') const customization = useSelector((state) => state.customization) const theme = useTheme() - const navigate = useNavigate() const getColSpan = (evaluationsShown, llmEvaluations) => { let colSpan = 1 @@ -45,6 +43,23 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) = return colSpan } + const getOpenLink = (index) => { + if (index === undefined) { + return '' + } + if (dialogProps.data?.additionalConfig?.chatflowTypes) { + switch (dialogProps.data.additionalConfig.chatflowTypes[index]) { + case 'Chatflow': + return '/canvas/' + dialogProps.data.evaluation.chatflowId[index] + case 'Custom Assistant': + return '/assistants/custom/' + dialogProps.data.evaluation.chatflowId[index] + case 'Agentflow v2': + return '/v2/agentcanvas/' + dialogProps.data.evaluation.chatflowId[index] + } + } + return '/canvas/' + dialogProps.data.evaluation.chatflowId[index] + } + const component = show ? ( @@ -65,7 +80,7 @@ const EvalsResultDialog = ({ show, dialogProps, onCancel, openDetailsDrawer }) = }} > - Chatflows Used: + Flows Used: {(dialogProps.data.evaluation.chatflowName || []).map((chatflowUsed, index) => ( navigate('/canvas/' + dialogProps.data.evaluation.chatflowId[index])} + onClick={() => window.open(getOpenLink(index), '_blank')} > ))} diff --git a/packages/ui/src/views/evaluations/EvaluationResult.jsx b/packages/ui/src/views/evaluations/EvaluationResult.jsx index e6d35079b..6fdde95b7 100644 --- a/packages/ui/src/views/evaluations/EvaluationResult.jsx +++ b/packages/ui/src/views/evaluations/EvaluationResult.jsx @@ -25,6 +25,7 @@ import { import { useTheme } from '@mui/material/styles' import moment from 'moment' import PaidIcon from '@mui/icons-material/Paid' +import { IconHierarchy, IconUsersGroup, IconRobot } from '@tabler/icons-react' import LLMIcon from '@mui/icons-material/ModelTraining' import AlarmIcon from '@mui/icons-material/AlarmOn' import TokensIcon from '@mui/icons-material/AutoAwesomeMotion' @@ -116,10 +117,13 @@ const EvalEvaluationRows = () => { const [expandTableProps, setExpandTableProps] = useState({}) const [isTableLoading, setTableLoading] = useState(false) + const [additionalConfig, setAdditionalConfig] = useState({}) + const openDetailsDrawer = (item) => { setSideDrawerDialogProps({ type: 'View', data: item, + additionalConfig: additionalConfig, evaluationType: evaluation.evaluationType, evaluationChatflows: evaluation.chatflowName }) @@ -169,7 +173,8 @@ const EvalEvaluationRows = () => { showCustomEvals, showTokenMetrics, showLatencyMetrics, - showCostMetrics + showCostMetrics, + additionalConfig } }) setShowExpandTableDialog(true) @@ -239,6 +244,9 @@ const EvalEvaluationRows = () => { const data = getEvaluation.data setSelectedEvaluationName(data.name) getIsOutdatedApi.request(data.id) + if (data.additionalConfig) { + setAdditionalConfig(JSON.parse(data.additionalConfig)) + } data.chatflowId = typeof data.chatflowId === 'object' ? data.chatflowId : JSON.parse(data.chatflowId) data.chatflowName = typeof data.chatflowName === 'object' ? data.chatflowName : JSON.parse(data.chatflowName) const rows = getEvaluation.data.rows @@ -314,6 +322,51 @@ const EvalEvaluationRows = () => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [getEvaluation.data]) + const getOpenLink = (index) => { + if (index === undefined) { + return undefined + } + const id = evaluation.chatflowId[index] + // this is to check if the evaluation is deleted! + if (outdated?.errors?.length > 0 && outdated.errors.find((e) => e.id === id)) { + return undefined + } + if (additionalConfig.chatflowTypes) { + switch (additionalConfig.chatflowTypes[index]) { + case 'Chatflow': + return '/canvas/' + evaluation.chatflowId[index] + case 'Custom Assistant': + return '/assistants/custom/' + evaluation.chatflowId[index] + case 'Agentflow v2': + return '/v2/agentcanvas/' + evaluation.chatflowId[index] + } + } + return '/canvas/' + evaluation.chatflowId[index] + } + + const openFlow = (index) => { + const url = getOpenLink(index) + if (url) { + window.open(getOpenLink(index), '_blank') + } + } + + const getFlowIcon = (index) => { + if (index === undefined) { + return + } + if (additionalConfig.chatflowTypes) { + switch (additionalConfig.chatflowTypes[index]) { + case 'Chatflow': + return + case 'Custom Assistant': + return + case 'Agentflow v2': + return + } + } + return + } return ( <> @@ -405,14 +458,14 @@ const EvalEvaluationRows = () => { }} variant='outlined' label={outdated.dataset.name} - onClick={() => navigate(`/dataset_rows/${outdated.dataset.id}`)} + onClick={() => window.open(`/dataset_rows/${outdated.dataset.id}`, '_blank')} > )} {outdated.chatflows && outdated?.errors?.length === 0 && outdated.chatflows.length > 0 && ( <>
- Chatflows: + Flows: {outdated.chatflows.map((chatflow, index) => ( { }} variant='outlined' label={chatflow.chatflowName} - onClick={() => navigate(`/canvas/${chatflow.chatflowId}`)} + onClick={() => + window.open( + chatflow.chatflowType === 'Chatflow' + ? '/canvas/' + chatflow.chatflowId + : chatflow.chatflowType === 'Custom Assistant' + ? '/assistants/custom/' + chatflow.chatflowId + : '/v2/agentcanvas/' + chatflow.chatflowId, + '_blank' + ) + } > ))} )} {outdated.errors.length > 0 && - outdated.errors.map((error, index) => {error})} + outdated.errors.map((error, index) => {error.message})} { {showCharts && ( {customEvalsDefined && ( - + { }} > - Chatflows Used: + Flows Used: {(evaluation.chatflowName || []).map((chatflowUsed, index) => ( { : '0 2px 14px 0 rgb(32 40 45 / 10%)' }} label={chatflowUsed} - onClick={() => navigate('/canvas/' + evaluation.chatflowId[index])} + onClick={() => openFlow(index)} > ))} diff --git a/packages/ui/src/views/evaluations/EvaluationResultSideDrawer.jsx b/packages/ui/src/views/evaluations/EvaluationResultSideDrawer.jsx index 9a3ada22b..c415fb983 100644 --- a/packages/ui/src/views/evaluations/EvaluationResultSideDrawer.jsx +++ b/packages/ui/src/views/evaluations/EvaluationResultSideDrawer.jsx @@ -1,8 +1,25 @@ import PropTypes from 'prop-types' -import { CardContent, Card, Box, SwipeableDrawer, Stack, Button, Chip, Divider, Typography } from '@mui/material' +import { + CardContent, + Card, + Box, + SwipeableDrawer, + Stack, + Button, + Chip, + Divider, + Typography, + Table, + TableHead, + TableRow, + TableBody +} from '@mui/material' +import { IconHierarchy, IconUsersGroup, IconRobot } from '@tabler/icons-react' + import { useSelector } from 'react-redux' -import { IconSquareRoundedChevronsRight } from '@tabler/icons-react' import { evaluators as evaluatorsOptions, numericOperators } from '../evaluators/evaluatorConstant' +import TableCell from '@mui/material/TableCell' +import { Close } from '@mui/icons-material' const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => { const onOpen = () => {} @@ -19,12 +36,32 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => { return '' } + const getFlowIcon = (index) => { + if (index === undefined) { + return + } + if (dialogProps.additionalConfig.chatflowTypes) { + switch (dialogProps.additionalConfig.chatflowTypes[index]) { + case 'Chatflow': + return + case 'Custom Assistant': + return + case 'Agentflow v2': + return + } + } + return + } + return ( onClickFunction()} onOpen={onOpen}> - - +
+ + Evaluation Details + +
+ Evaluation Id @@ -61,13 +98,19 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => { {dialogProps.evaluationChatflows?.length > 0 && ( <> - - - Chatflow +
+ {getFlowIcon(index)} + + {dialogProps.evaluationChatflows[index]} - {dialogProps.evaluationChatflows[index]} - -
+
)} @@ -153,79 +196,222 @@ const EvaluationResultSideDrawer = ({ show, dialogProps, onClickFunction }) => {

- - - Tokens - - - - - - - - - + {dialogProps.data.metrics[index]?.nested_metrics ? ( + + + Tokens + + + + + + Node + + + Provider & Model + + + Input + + + Output + + + Total + + + + + {dialogProps.data.metrics[index]?.nested_metrics?.map((metric, index) => ( + + + {metric.nodeLabel} + + + {metric.provider} +
+ {metric.model} + + + {metric.promptTokens} + + + {metric.completionTokens} + + + {metric.totalTokens} + + + ))} + + + Total + + + {dialogProps.data.metrics[index].promptTokens} + + + {dialogProps.data.metrics[index].completionTokens} + + + {dialogProps.data.metrics[index].totalTokens} + + + +
+ + ) : ( + + + Tokens + + + + + + + + + + )}
- - - Cost - - - - - - - - - + {dialogProps.data.metrics[index]?.nested_metrics ? ( + + + Cost + + + + + + Node + + + Provider & Model + + + Input + + + Output + + + Total + + + + + {dialogProps.data.metrics[index]?.nested_metrics?.map((metric, index) => ( + + + {metric.nodeLabel} + + + {metric.provider}
+ {metric.model} + + + {metric.promptCost} + + + {metric.completionCost} + + + {metric.totalCost} + + + ))} + + + Total + + + {dialogProps.data.metrics[index].promptCost} + + + {dialogProps.data.metrics[index].completionCost} + + + {dialogProps.data.metrics[index].totalCost} + + + +
+ + ) : ( + + + Cost + + + + + + + + + + )}

diff --git a/packages/ui/src/views/evaluations/MetricsItemCard.jsx b/packages/ui/src/views/evaluations/MetricsItemCard.jsx index a8bdca820..380f19c72 100644 --- a/packages/ui/src/views/evaluations/MetricsItemCard.jsx +++ b/packages/ui/src/views/evaluations/MetricsItemCard.jsx @@ -11,7 +11,7 @@ import SkeletonChatflowCard from '@/ui-component/cards/Skeleton/ChatflowCard' const CardWrapper = styled(MainCard)(({ theme }) => ({ background: theme.palette.card.main, color: theme.darkTextPrimary, - overflow: 'auto', + overflow: 'hidden', position: 'relative', boxShadow: '0 2px 14px 0 rgb(32 40 45 / 8%)', cursor: 'pointer', diff --git a/packages/ui/src/views/evaluations/index.jsx b/packages/ui/src/views/evaluations/index.jsx index 2f28e1b13..6f7c09316 100644 --- a/packages/ui/src/views/evaluations/index.jsx +++ b/packages/ui/src/views/evaluations/index.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from 'react' +import React, { useEffect, useState, useCallback } from 'react' import * as PropTypes from 'prop-types' import moment from 'moment/moment' import { useNavigate } from 'react-router-dom' @@ -20,7 +20,8 @@ import { TableBody, TableContainer, TableHead, - TableRow + TableRow, + ToggleButton } from '@mui/material' import { useTheme } from '@mui/material/styles' import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions' @@ -35,7 +36,6 @@ import useNotifier from '@/utils/useNotifier' // project import MainCard from '@/ui-component/cards/MainCard' -import { StyledButton } from '@/ui-component/button/StyledButton' import { BackdropLoader } from '@/ui-component/loading/BackdropLoader' import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog' import ErrorBoundary from '@/ErrorBoundary' @@ -53,7 +53,9 @@ import { IconTrash, IconX, IconChevronsUp, - IconChevronsDown + IconChevronsDown, + IconPlayerPlay, + IconPlayerPause } from '@tabler/icons-react' import empty_evalSVG from '@/assets/images/empty_evals.svg' @@ -79,6 +81,7 @@ const EvalsEvaluation = () => { const [loading, setLoading] = useState(false) const [isTableLoading, setTableLoading] = useState(false) const [selected, setSelected] = useState([]) + const [autoRefresh, setAutoRefresh] = useState(false) const onSelectAllClick = (event) => { if (event.target.checked) { @@ -240,14 +243,34 @@ const EvalsEvaluation = () => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [createNewEvaluation.error]) - const onRefresh = () => { + const onRefresh = useCallback(() => { getAllEvaluations.request() - } + }, [getAllEvaluations]) useEffect(() => { setTableLoading(getAllEvaluations.loading) }, [getAllEvaluations.loading]) + useEffect(() => { + let intervalId = null + + if (autoRefresh) { + intervalId = setInterval(() => { + onRefresh() + }, 5000) + } + + return () => { + if (intervalId) { + clearInterval(intervalId) + } + } + }, [autoRefresh, onRefresh]) + + const toggleAutoRefresh = () => { + setAutoRefresh(!autoRefresh) + } + return ( <> @@ -256,15 +279,52 @@ const EvalsEvaluation = () => { ) : ( - } + - Refresh - + {autoRefresh ? : } + + + + { Latest Version Average Metrics Last Evaluated - Chatflow(s) + Flow(s) Dataset @@ -438,7 +498,7 @@ function EvaluationRunRow(props) { } const goToDataset = (id) => { - navigate(`/dataset_rows/${id}`) + window.open(`/dataset_rows/${id}`, '_blank') } const onSelectAllChildClick = (event) => { @@ -513,10 +573,6 @@ function EvaluationRunRow(props) { } } - const goToCanvas = (id) => { - navigate(`/canvas/${id}`) - } - const getStatusColor = (status) => { switch (status) { case 'pending': @@ -619,16 +675,11 @@ function EvaluationRunRow(props) { {props.item?.usedFlows?.map((usedFlow, index) => ( goToCanvas(props.item.chatIds[index])} > ))} @@ -637,6 +688,7 @@ function EvaluationRunRow(props) {