'use strict'; var requestContext = require('@mastra/core/request-context'); var evals = require('@mastra/core/evals'); // src/scorers/utils.ts function getTextContentFromMastraDBMessage(message) { const content = message.content; if (typeof content === "string") { return content; } if (Array.isArray(content)) { const textParts = content.filter((p) => p.type === "text"); return textParts.length > 0 ? textParts[textParts.length - 1]?.text || "" : ""; } if (typeof content?.content === "string" && content.content !== "") { return content.content; } if (typeof content?.text === "string" && content.text !== "") { return content.text; } if (content?.parts && Array.isArray(content.parts)) { const textParts = content.parts.filter((p) => p.type === "text"); return textParts.length > 0 ? textParts[textParts.length - 1]?.text || "" : ""; } return ""; } var isRecord = (value) => { return typeof value === "object" && value !== null; }; var getTextFromValue = (value) => { if (typeof value === "string") return value === "" ? void 0 : value; if (Array.isArray(value)) { const textParts = value.filter((part) => isRecord(part) && part.type === "text" && typeof part.text === "string").map((part) => part.text); return textParts.length > 0 ? textParts[textParts.length - 1] : void 0; } if (!isRecord(value)) return void 0; const fromParts = Array.isArray(value.parts) ? getTextFromValue(value.parts) : void 0; return getTextFromValue(value.content) ?? (typeof value.text === "string" && value.text !== "" ? value.text : void 0) ?? (typeof value.body === "string" && value.body !== "" ? value.body : void 0) ?? fromParts; }; var isScorerRunInputForAgent = (input) => { return isRecord(input) && Array.isArray(input.inputMessages) && Array.isArray(input.rememberedMessages) && Array.isArray(input.systemMessages) && isRecord(input.taggedSystemMessages); }; var isMastraDBMessageLike = (message) => { return isRecord(message) && typeof message.id === "string" && typeof message.role === "string" && "content" in message && "createdAt" in message; }; var isScorerRunOutputForAgent = (output) => { return Array.isArray(output) && output.every(isMastraDBMessageLike); }; var getTextFromMessages = (messages, role) => { if (!Array.isArray(messages)) return void 0; const message = messages.find((message2) => isRecord(message2) && message2.role === role); return message ? getTextFromValue(message) : void 0; }; var roundToTwoDecimals = (num) => { return Math.round((num + Number.EPSILON) * 100) / 100; }; function isCloserTo(value, target1, target2) { return Math.abs(value - target1) < Math.abs(value - target2); } var createTestRun = (input, output, additionalContext, requestContext) => { return { input: [{ role: "user", content: input }], output: { role: "assistant", text: output }, additionalContext: additionalContext ?? {}, requestContext: requestContext ?? {} }; }; var getUserMessageFromRunInput = (input) => { if (typeof input === "string") return input; if (!isRecord(input)) return void 0; return getTextFromMessages(input.inputMessages, "user") ?? getTextFromMessages(input.messages, "user") ?? (typeof input.prompt === "string" ? input.prompt : void 0) ?? (typeof input.text === "string" ? input.text : void 0) ?? getTextFromValue(input.content) ?? getTextFromValue(input.input) ?? getTextFromValue(input.user); }; var getSystemMessagesFromRunInput = (input) => { const systemMessages = []; if (!isRecord(input)) return systemMessages; if (Array.isArray(input.systemMessages)) { systemMessages.push( ...input.systemMessages.map((msg) => { if (typeof msg.content === "string") { return msg.content; } else if (Array.isArray(msg.content)) { return msg.content.filter((part) => part.type === "text").map((part) => part.text || "").join(" "); } return ""; }).filter((content) => content) ); } const addSystemMessages = (messages) => { if (!Array.isArray(messages)) return; systemMessages.push( ...messages.filter((message) => isRecord(message) && message.role === "system").map((message) => getTextFromValue(message)).filter((content) => Boolean(content)) ); }; addSystemMessages(input.inputMessages); addSystemMessages(input.messages); if (isRecord(input.taggedSystemMessages)) { Object.values(input.taggedSystemMessages).forEach((messages) => { if (!Array.isArray(messages)) return; messages.forEach((msg) => { const content = getTextFromValue(msg); if (content) { systemMessages.push(content); } }); }); } return systemMessages; }; var getCombinedSystemPrompt = (input) => { const systemMessages = getSystemMessagesFromRunInput(input); return systemMessages.join("\n\n"); }; var getAssistantMessageFromRunOutput = (output) => { if (typeof output === "string") return output; if (Array.isArray(output)) return getTextFromMessages(output, "assistant"); if (!isRecord(output)) return void 0; const isAssistantOutput = output.role === void 0 || output.role === "assistant"; if (isAssistantOutput && typeof output.text === "string") return output.text; if (isAssistantOutput && typeof output.content === "string") return output.content; if (isAssistantOutput && (isRecord(output.content) || Array.isArray(output.content))) { return getTextContentFromMastraDBMessage(output) || getTextContentFromMastraDBMessage(output.content) || void 0; } if (output.role === "assistant") return getTextContentFromMastraDBMessage(output) || void 0; return void 0; }; var getReasoningFromRunOutput = (output) => { if (!output) return void 0; const message = output.find(({ role }) => role === "assistant"); if (!message) return void 0; if (message.content.reasoning) { return message.content.reasoning; } const reasoningParts = message.content.parts?.filter((p) => p.type === "reasoning"); if (reasoningParts && reasoningParts.length > 0) { const reasoningTexts = reasoningParts.map((p) => { if (p.details && Array.isArray(p.details)) { return p.details.filter((d) => d.type === "text").map((d) => d.text).join(""); } return p.reasoning || ""; }).filter(Boolean); return reasoningTexts.length > 0 ? reasoningTexts.join("\n") : void 0; } return void 0; }; var createToolInvocation = ({ toolCallId, toolName, args, result, state = "result" }) => { return { toolCallId, toolName, args, result, state }; }; function createTestMessage({ content, role, id = "test-message", toolInvocations = [] }) { return { id, role, content: { format: 2, parts: [{ type: "text", text: content }], content, ...toolInvocations.length > 0 && { toolInvocations: toolInvocations.map((ti) => ({ toolCallId: ti.toolCallId, toolName: ti.toolName, args: ti.args, result: ti.result, state: ti.state })) } }, createdAt: /* @__PURE__ */ new Date() }; } var createAgentTestRun = ({ inputMessages = [], output, rememberedMessages = [], systemMessages = [], taggedSystemMessages = {}, requestContext: requestContext$1 = new requestContext.RequestContext(), runId = crypto.randomUUID() }) => { return { input: { inputMessages, rememberedMessages, systemMessages, taggedSystemMessages }, output, requestContext: requestContext$1, runId }; }; var createTrajectoryTestRun = ({ inputMessages = [], trajectory, rememberedMessages = [], systemMessages = [], taggedSystemMessages = {}, requestContext: requestContext$1 = new requestContext.RequestContext(), runId = crypto.randomUUID(), expectedTrajectory }) => { return { input: { inputMessages, rememberedMessages, systemMessages, taggedSystemMessages }, output: trajectory, expectedTrajectory, requestContext: requestContext$1, runId }; }; function extractToolCalls(output) { const toolCalls = []; const toolCallInfos = []; for (let messageIndex = 0; messageIndex < output.length; messageIndex++) { const message = output[messageIndex]; const legacy = message?.content?.toolInvocations; const fromParts = legacy ? void 0 : message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation); const toolInvocations = legacy ?? fromParts; if (!toolInvocations?.length) continue; for (let invocationIndex = 0; invocationIndex < toolInvocations.length; invocationIndex++) { const invocation = toolInvocations[invocationIndex]; if (invocation && invocation.toolName && (invocation.state === "result" || invocation.state === "call")) { toolCalls.push(invocation.toolName); toolCallInfos.push({ toolName: invocation.toolName, toolCallId: invocation.toolCallId || `${messageIndex}-${invocationIndex}`, messageIndex, invocationIndex }); } } } return { tools: toolCalls, toolCallInfos }; } var extractInputMessages = (runInput) => { return runInput?.inputMessages?.map((msg) => getTextContentFromMastraDBMessage(msg)) || []; }; var extractAgentResponseMessages = (runOutput) => { return runOutput.filter((msg) => msg.role === "assistant").map((msg) => getTextContentFromMastraDBMessage(msg)); }; function extractToolResults(output) { const results = []; for (const message of output) { const legacy = message?.content?.toolInvocations; const fromParts = legacy ? void 0 : message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation); const toolInvocations = legacy ?? fromParts; if (!toolInvocations?.length) continue; for (const invocation of toolInvocations) { if (invocation.state === "result" && invocation.result !== void 0) { results.push({ toolName: invocation.toolName, toolCallId: invocation.toolCallId || "", args: invocation.args || {}, result: invocation.result }); } } } return results; } function compareTrajectories(actual, expected, options = {}) { const { allowRepeatedSteps = true, ordering = "relaxed" } = options; const normalizedExpected = { steps: expected.steps }; if (normalizedExpected.steps.length === 0) { return { score: actual.steps.length === 0 ? 1 : 0, matchedSteps: 0, totalExpectedSteps: 0, totalActualSteps: actual.steps.length, missingSteps: [], extraSteps: actual.steps.map((s) => s.name), outOfOrderSteps: [], repeatedSteps: [] }; } const actualNames = actual.steps.map((s) => s.name); const nameCounts = /* @__PURE__ */ new Map(); for (const name of actualNames) { nameCounts.set(name, (nameCounts.get(name) || 0) + 1); } const repeatedSteps = [...nameCounts.entries()].filter(([_, count]) => count > 1).map(([name]) => name); if (ordering === "strict") { return compareStrictOrder(actual, normalizedExpected, { allowRepeatedSteps, repeatedSteps }); } if (ordering === "unordered") { return compareUnorderedPresence(actual, normalizedExpected, { allowRepeatedSteps, repeatedSteps }); } return compareRelaxedOrder(actual, normalizedExpected, { allowRepeatedSteps, repeatedSteps }); } function compareStrictOrder(actual, expected, opts) { const actualNames = actual.steps.map((s) => s.name); const expectedNames = expected.steps.map((s) => s.name); let matchedSteps = 0; const outOfOrderSteps = []; const matchedExpectedIndices = /* @__PURE__ */ new Set(); const maxLen = Math.max(actualNames.length, expectedNames.length); for (let i = 0; i < maxLen; i++) { const actualName = actualNames[i]; const expectedName = expectedNames[i]; if (actualName === expectedName) { if (actual.steps[i] && expected.steps[i]) { if (expectedStepMatches(actual.steps[i], expected.steps[i])) { matchedSteps++; matchedExpectedIndices.add(i); } } else { matchedSteps++; matchedExpectedIndices.add(i); } } else if (actualName && expectedNames.includes(actualName)) { outOfOrderSteps.push(actualName); } } const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i)); const extraSteps = actualNames.filter((name) => !expectedNames.includes(name)); let score = matchedSteps / expected.steps.length; if (actualNames.length > expectedNames.length) { const extraPenalty = (actualNames.length - expectedNames.length) / expectedNames.length; score = Math.max(0, score - extraPenalty * 0.5); } if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) { score = Math.max(0, score - opts.repeatedSteps.length * 0.1); } return { score: roundToTwoDecimals(Math.max(0, Math.min(1, score))), matchedSteps, totalExpectedSteps: expected.steps.length, totalActualSteps: actual.steps.length, missingSteps, extraSteps, outOfOrderSteps, repeatedSteps: opts.repeatedSteps }; } function compareRelaxedOrder(actual, expected, opts) { const actualNames = actual.steps.map((s) => s.name); const expectedNames = expected.steps.map((s) => s.name); let matchedSteps = 0; let lastMatchedIndex = -1; const outOfOrderSteps = []; const matchedExpectedIndices = /* @__PURE__ */ new Set(); for (let i = 0; i < expectedNames.length; i++) { const expectedName = expectedNames[i]; let found = false; for (let j = lastMatchedIndex + 1; j < actualNames.length; j++) { if (actualNames[j] === expectedName) { if (actual.steps[j] && expected.steps[i]) { if (expectedStepMatches(actual.steps[j], expected.steps[i])) { matchedSteps++; lastMatchedIndex = j; matchedExpectedIndices.add(i); found = true; break; } } else { matchedSteps++; lastMatchedIndex = j; matchedExpectedIndices.add(i); found = true; break; } } } if (!found) { if (actualNames.includes(expectedName)) { outOfOrderSteps.push(expectedName); } } } const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i)); const expectedSet = new Set(expectedNames); const extraSteps = actualNames.filter((name) => !expectedSet.has(name)); let score = matchedSteps / expected.steps.length; if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) { score = Math.max(0, score - opts.repeatedSteps.length * 0.1); } return { score: roundToTwoDecimals(Math.max(0, Math.min(1, score))), matchedSteps, totalExpectedSteps: expected.steps.length, totalActualSteps: actual.steps.length, missingSteps, extraSteps, outOfOrderSteps, repeatedSteps: opts.repeatedSteps }; } var COMPARABLE_FIELDS_BY_TYPE = { tool_call: ["toolArgs", "toolResult", "success"], mcp_tool_call: ["toolArgs", "toolResult", "mcpServer", "success"], model_generation: ["modelId", "promptTokens", "completionTokens", "finishReason"], agent_run: ["agentId"], workflow_step: ["stepId", "status", "output"], workflow_run: ["workflowId", "status"], workflow_conditional: ["conditionCount", "selectedSteps"], workflow_parallel: ["branchCount", "parallelSteps"], workflow_loop: ["loopType", "totalIterations"], workflow_sleep: ["sleepDurationMs", "sleepType"], workflow_wait_event: ["eventName", "eventReceived"], processor_run: ["processorId"] }; function expectedStepMatches(actual, expected) { if (actual.name !== expected.name) return false; if (expected.stepType && actual.stepType !== expected.stepType) return false; if (expected.stepType) { const fields = COMPARABLE_FIELDS_BY_TYPE[expected.stepType] ?? []; for (const field of fields) { const expectedVal = expected[field]; if (expectedVal === void 0) continue; const actualVal = actual[field]; if (actualVal === void 0) return false; try { if (JSON.stringify(actualVal) !== JSON.stringify(expectedVal)) return false; } catch { return false; } } } return true; } function compareUnorderedPresence(actual, expected, opts) { const actualNames = actual.steps.map((s) => s.name); const expectedNames = expected.steps.map((s) => s.name); let matchedSteps = 0; const matchedExpectedIndices = /* @__PURE__ */ new Set(); const usedIndices = /* @__PURE__ */ new Set(); for (let i = 0; i < expected.steps.length; i++) { const expectedStep = expected.steps[i]; for (let j = 0; j < actual.steps.length; j++) { if (!usedIndices.has(j) && expectedStepMatches(actual.steps[j], expectedStep)) { matchedSteps++; matchedExpectedIndices.add(i); usedIndices.add(j); break; } } } const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i)); const expectedSet = new Set(expectedNames); const extraSteps = actualNames.filter((name) => !expectedSet.has(name)); let score = matchedSteps / expected.steps.length; if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) { score = Math.max(0, score - opts.repeatedSteps.length * 0.1); } return { score: roundToTwoDecimals(Math.max(0, Math.min(1, score))), matchedSteps, totalExpectedSteps: expected.steps.length, totalActualSteps: actual.steps.length, missingSteps, extraSteps, outOfOrderSteps: [], // ordering not checked in unordered mode repeatedSteps: opts.repeatedSteps }; } function checkTrajectoryEfficiency(trajectory, options = {}) { const { maxSteps, maxTotalTokens, maxTotalDurationMs, noRedundantCalls = true } = options; const totalSteps = trajectory.steps.length; let totalTokens = 0; for (const step of trajectory.steps) { if (step.stepType === "model_generation") { totalTokens += (step.promptTokens ?? 0) + (step.completionTokens ?? 0); } } const totalDurationMs = trajectory.totalDurationMs ?? trajectory.steps.reduce((sum, s) => sum + (s.durationMs ?? 0), 0); const redundantCalls = []; if (noRedundantCalls) { for (let i = 1; i < trajectory.steps.length; i++) { const prev = trajectory.steps[i - 1]; const curr = trajectory.steps[i]; if (prev.name === curr.name && prev.stepType === curr.stepType && (prev.stepType === "tool_call" || prev.stepType === "mcp_tool_call")) { const prevArgs = prev.toolArgs; const currArgs = curr.toolArgs; try { if (JSON.stringify(prevArgs) === JSON.stringify(currArgs)) { redundantCalls.push({ name: curr.name, index: i }); } } catch { } } } } const overStepBudget = maxSteps !== void 0 && totalSteps > maxSteps; const overTokenBudget = maxTotalTokens !== void 0 && totalTokens > maxTotalTokens; const overDurationBudget = maxTotalDurationMs !== void 0 && totalDurationMs > maxTotalDurationMs; const dimensions = []; if (maxSteps !== void 0) { dimensions.push(overStepBudget ? Math.max(0, 1 - (totalSteps - maxSteps) / maxSteps) : 1); } if (maxTotalTokens !== void 0) { dimensions.push(overTokenBudget ? Math.max(0, 1 - (totalTokens - maxTotalTokens) / maxTotalTokens) : 1); } if (maxTotalDurationMs !== void 0) { dimensions.push( overDurationBudget ? Math.max(0, 1 - (totalDurationMs - maxTotalDurationMs) / maxTotalDurationMs) : 1 ); } if (noRedundantCalls) { dimensions.push(redundantCalls.length === 0 ? 1 : Math.max(0, 1 - redundantCalls.length * 0.2)); } const score = dimensions.length > 0 ? dimensions.reduce((a, b) => a + b, 0) / dimensions.length : 1; return { score: roundToTwoDecimals(Math.max(0, Math.min(1, score))), totalSteps, overStepBudget, totalTokens, overTokenBudget, totalDurationMs, overDurationBudget, redundantCalls }; } function checkTrajectoryBlacklist(trajectory, options = {}) { const { blacklistedTools = [], blacklistedSequences = [] } = options; const violatedTools = []; const violatedSequences = []; const stepNames = trajectory.steps.map((s) => s.name); for (const forbidden of blacklistedTools) { if (stepNames.includes(forbidden)) { violatedTools.push(forbidden); } } for (const sequence of blacklistedSequences) { if (sequence.length === 0) continue; for (let i = 0; i <= stepNames.length - sequence.length; i++) { let match = true; for (let j = 0; j < sequence.length; j++) { if (stepNames[i + j] !== sequence[j]) { match = false; break; } } if (match) { violatedSequences.push(sequence); break; } } } const hasViolations = violatedTools.length > 0 || violatedSequences.length > 0; return { score: hasViolations ? 0 : 1, violatedTools, violatedSequences }; } function analyzeToolFailures(trajectory, options = {}) { const { maxRetriesPerTool = 2 } = options; const patterns = []; let totalRetries = 0; const toolCallSteps = trajectory.steps.filter((s) => s.stepType === "tool_call" || s.stepType === "mcp_tool_call"); if (toolCallSteps.length === 0) { return { score: 1, patterns: [], totalRetries: 0, excessiveRetryTools: [] }; } let i = 0; while (i < toolCallSteps.length) { const currentTool = toolCallSteps[i]; let retryCount = 0; let j = i + 1; while (j < toolCallSteps.length && toolCallSteps[j].name === currentTool.name) { const prevStep = toolCallSteps[j - 1]; if (prevStep.success === false) { retryCount++; } j++; } if (retryCount > 0) { const nextDifferentTool = j < toolCallSteps.length ? toolCallSteps[j] : void 0; const lastRetry = toolCallSteps[j - 1]; const lastSuccess = lastRetry.success !== false; patterns.push({ toolName: currentTool.name, retryCount, fellBackToAlternative: nextDifferentTool !== void 0 && !lastSuccess, alternativeTool: nextDifferentTool !== void 0 && !lastSuccess ? nextDifferentTool.name : void 0, eventuallySucceeded: lastSuccess }); totalRetries += retryCount; } i = j; } const excessiveRetryTools = patterns.filter((p) => p.retryCount > maxRetriesPerTool).map((p) => p.toolName); let score = 1; if (toolCallSteps.length > 0) { const excessRetries = patterns.reduce((sum, p) => sum + Math.max(0, p.retryCount - maxRetriesPerTool), 0); score = Math.max(0, 1 - excessRetries * 0.2); } return { score: roundToTwoDecimals(Math.max(0, Math.min(1, score))), patterns, totalRetries, excessiveRetryTools }; } Object.defineProperty(exports, "extractTrajectory", { enumerable: true, get: function () { return evals.extractTrajectory; } }); exports.analyzeToolFailures = analyzeToolFailures; exports.checkTrajectoryBlacklist = checkTrajectoryBlacklist; exports.checkTrajectoryEfficiency = checkTrajectoryEfficiency; exports.compareTrajectories = compareTrajectories; exports.createAgentTestRun = createAgentTestRun; exports.createTestMessage = createTestMessage; exports.createTestRun = createTestRun; exports.createToolInvocation = createToolInvocation; exports.createTrajectoryTestRun = createTrajectoryTestRun; exports.extractAgentResponseMessages = extractAgentResponseMessages; exports.extractInputMessages = extractInputMessages; exports.extractToolCalls = extractToolCalls; exports.extractToolResults = extractToolResults; exports.getAssistantMessageFromRunOutput = getAssistantMessageFromRunOutput; exports.getCombinedSystemPrompt = getCombinedSystemPrompt; exports.getReasoningFromRunOutput = getReasoningFromRunOutput; exports.getSystemMessagesFromRunInput = getSystemMessagesFromRunInput; exports.getTextContentFromMastraDBMessage = getTextContentFromMastraDBMessage; exports.getUserMessageFromRunInput = getUserMessageFromRunInput; exports.isCloserTo = isCloserTo; exports.isScorerRunInputForAgent = isScorerRunInputForAgent; exports.isScorerRunOutputForAgent = isScorerRunOutputForAgent; exports.roundToTwoDecimals = roundToTwoDecimals; //# sourceMappingURL=chunk-UNQXHPOD.cjs.map //# sourceMappingURL=chunk-UNQXHPOD.cjs.map