'use strict'; var chunkDIG2K5CV_cjs = require('./chunk-DIG2K5CV.cjs'); var v4 = require('zod/v4'); var jsonSchemaObject = v4.z.lazy(() => v4.z.record(v4.z.string(), v4.z.unknown())); var jsonSchemaField = v4.z.union([jsonSchemaObject, v4.z.null()]).optional(); var expectedStepBase = { name: v4.z.string().describe("Step name to match"), durationMs: v4.z.number().optional(), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), children: v4.z.any().optional().describe("Nested trajectory expectation (untyped at this depth)") }; var expectedToolCallStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("tool_call"), toolArgs: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), toolResult: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), success: v4.z.boolean().optional() }); var expectedMcpToolCallStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("mcp_tool_call"), toolArgs: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), toolResult: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), mcpServer: v4.z.string().optional(), success: v4.z.boolean().optional() }); var expectedModelGenerationStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("model_generation"), modelId: v4.z.string().optional(), promptTokens: v4.z.number().optional(), completionTokens: v4.z.number().optional(), finishReason: v4.z.string().optional() }); var expectedAgentRunStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("agent_run"), agentId: v4.z.string().optional() }); var expectedWorkflowStepStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("workflow_step"), stepId: v4.z.string().optional(), status: v4.z.string().optional(), output: v4.z.record(v4.z.string(), v4.z.unknown()).optional() }); var expectedWorkflowRunStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("workflow_run"), workflowId: v4.z.string().optional(), status: v4.z.string().optional() }); var expectedWorkflowConditionalStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("workflow_conditional"), conditionCount: v4.z.number().optional(), selectedSteps: v4.z.array(v4.z.string()).optional() }); var expectedWorkflowParallelStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("workflow_parallel"), branchCount: v4.z.number().optional(), parallelSteps: v4.z.array(v4.z.string()).optional() }); var expectedWorkflowLoopStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("workflow_loop"), loopType: v4.z.string().optional(), totalIterations: v4.z.number().optional() }); var expectedWorkflowSleepStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("workflow_sleep"), sleepDurationMs: v4.z.number().optional(), sleepType: v4.z.string().optional() }); var expectedWorkflowWaitEventStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("workflow_wait_event"), eventName: v4.z.string().optional(), eventReceived: v4.z.boolean().optional() }); var expectedProcessorRunStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.literal("processor_run"), processorId: v4.z.string().optional() }); var expectedGenericStepSchema = v4.z.object({ ...expectedStepBase, stepType: v4.z.undefined().optional() }); var expectedStepSchema = v4.z.union([ v4.z.discriminatedUnion("stepType", [ expectedToolCallStepSchema, expectedMcpToolCallStepSchema, expectedModelGenerationStepSchema, expectedAgentRunStepSchema, expectedWorkflowStepStepSchema, expectedWorkflowRunStepSchema, expectedWorkflowConditionalStepSchema, expectedWorkflowParallelStepSchema, expectedWorkflowLoopStepSchema, expectedWorkflowSleepStepSchema, expectedWorkflowWaitEventStepSchema, expectedProcessorRunStepSchema ]), expectedGenericStepSchema ]); var trajectoryExpectationSchema = v4.z.object({ // Accuracy steps: v4.z.array(expectedStepSchema).optional().describe("Expected steps for accuracy checking"), ordering: v4.z.enum(["strict", "relaxed", "unordered"]).optional().describe("How to compare step ordering (default: relaxed)"), allowRepeatedSteps: v4.z.boolean().optional().describe("Whether to allow repeated steps (default: true)"), // Efficiency maxSteps: v4.z.number().int().optional().describe("Maximum number of steps allowed"), maxTotalTokens: v4.z.number().int().optional().describe("Maximum total tokens across all model_generation steps"), maxTotalDurationMs: v4.z.number().optional().describe("Maximum total duration in milliseconds"), noRedundantCalls: v4.z.boolean().optional().describe("Whether to penalize redundant calls (same tool + same args consecutively, default: true)"), // Blacklist blacklistedTools: v4.z.array(v4.z.string()).optional().describe("Tool names that should never appear"), blacklistedSequences: v4.z.array(v4.z.array(v4.z.string())).optional().describe("Tool name sequences that should never appear"), // Tool failure tolerance maxRetriesPerTool: v4.z.number().int().optional().describe("Maximum retries per tool before penalizing (default: 2)") }).optional().nullable().describe("Expected trajectory configuration for trajectory scoring"); var datasetItemSourceSchema = v4.z.object({ type: v4.z.enum(["csv", "json", "trace", "llm", "experiment-result"]).describe("How this item was created"), referenceId: v4.z.string().optional().describe("Reference identifier (e.g., trace id, csv filename)") }).optional().describe("Source/provenance of this dataset item"); var datasetIdPathParams = v4.z.object({ datasetId: v4.z.string().describe("Unique identifier for the dataset") }); var experimentIdPathParams = v4.z.object({ experimentId: v4.z.string().describe("Unique identifier for the experiment") }); var itemIdPathParams = v4.z.object({ itemId: v4.z.string().describe("Unique identifier for the dataset item") }); var datasetAndExperimentIdPathParams = v4.z.object({ datasetId: v4.z.string().describe("Unique identifier for the dataset"), experimentId: v4.z.string().describe("Unique identifier for the experiment") }); var experimentResultIdPathParams = v4.z.object({ datasetId: v4.z.string().describe("Unique identifier for the dataset"), experimentId: v4.z.string().describe("Unique identifier for the experiment"), resultId: v4.z.string().describe("Unique identifier for the experiment result") }); var datasetAndItemIdPathParams = v4.z.object({ datasetId: v4.z.string().describe("Unique identifier for the dataset"), itemId: v4.z.string().describe("Unique identifier for the dataset item") }); var paginationQuerySchema = v4.z.object({ page: v4.z.coerce.number().optional().default(0), perPage: v4.z.coerce.number().optional().default(10) }); var listItemsQuerySchema = v4.z.object({ page: v4.z.coerce.number().optional().default(0), perPage: v4.z.coerce.number().optional().default(10), version: v4.z.coerce.number().int().optional(), // Optional version filter for snapshot semantics search: v4.z.string().optional() }); var createDatasetBodySchema = v4.z.object({ name: v4.z.string().describe("Name of the dataset"), description: v4.z.string().optional().describe("Description of the dataset"), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional().describe("Additional metadata"), inputSchema: jsonSchemaField.describe("JSON Schema for validating item input"), groundTruthSchema: jsonSchemaField.describe("JSON Schema for validating item groundTruth"), requestContextSchema: jsonSchemaField.describe("JSON Schema describing expected request context shape"), targetType: v4.z.string().optional().describe("Target entity type (e.g. agent, workflow, scorer)"), targetIds: v4.z.array(v4.z.string()).optional().describe("IDs of target entities this dataset is attached to"), scorerIds: v4.z.array(v4.z.string()).optional().describe("IDs of scorers attached to this dataset") }); var updateDatasetBodySchema = v4.z.object({ name: v4.z.string().optional().describe("Name of the dataset"), description: v4.z.string().optional().describe("Description of the dataset"), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional().describe("Additional metadata"), inputSchema: jsonSchemaField.describe("JSON Schema for validating item input"), groundTruthSchema: jsonSchemaField.describe("JSON Schema for validating item groundTruth"), requestContextSchema: jsonSchemaField.describe("JSON Schema describing expected request context shape"), tags: v4.z.array(v4.z.string()).optional().describe("Tag definitions for categorizing experiment results"), targetType: v4.z.string().optional().describe("Target entity type (e.g. agent, workflow, scorer)"), targetIds: v4.z.array(v4.z.string()).optional().describe("IDs of target entities this dataset is attached to"), scorerIds: v4.z.array(v4.z.string()).optional().nullable().describe("IDs of scorers attached to this dataset") }); var addItemBodySchema = v4.z.object({ input: v4.z.unknown().describe("Input data for the dataset item"), groundTruth: v4.z.unknown().optional().describe("Expected output for comparison"), expectedTrajectory: trajectoryExpectationSchema, requestContext: v4.z.record(v4.z.string(), v4.z.unknown()).optional().describe("Request context preset for this item"), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional().describe("Additional metadata"), source: datasetItemSourceSchema }); var updateItemBodySchema = v4.z.object({ input: v4.z.unknown().optional().describe("Input data for the dataset item"), groundTruth: v4.z.unknown().optional().describe("Expected output for comparison"), expectedTrajectory: trajectoryExpectationSchema, requestContext: v4.z.record(v4.z.string(), v4.z.unknown()).optional().describe("Request context preset for this item"), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional().describe("Additional metadata"), source: datasetItemSourceSchema }); var triggerExperimentBodySchema = v4.z.object({ targetType: v4.z.enum(["agent", "workflow", "scorer"]).describe("Type of target to run against"), targetId: v4.z.string().describe("ID of the target"), scorerIds: v4.z.array(v4.z.string()).optional().describe("IDs of scorers to apply"), version: v4.z.coerce.number().int().optional().describe("Pin to specific dataset version"), agentVersion: v4.z.string().optional().describe("Agent version ID to use for experiment"), maxConcurrency: v4.z.number().optional().describe("Maximum concurrent executions"), requestContext: v4.z.record(v4.z.string(), v4.z.unknown()).optional().describe("Global request context passed to the target"), versions: v4.z.object({ agents: v4.z.record( v4.z.string(), v4.z.union([v4.z.object({ versionId: v4.z.string() }), v4.z.object({ status: v4.z.enum(["draft", "published"]) })]) ).optional(), defaultStatus: v4.z.enum(["draft", "published"]).optional() }).optional().describe("Version overrides for sub-agent delegation during experiment execution") }); var compareExperimentsBodySchema = v4.z.object({ experimentIdA: v4.z.string().describe("ID of baseline experiment"), experimentIdB: v4.z.string().describe("ID of candidate experiment") }); var datasetResponseSchema = v4.z.object({ id: v4.z.string(), name: v4.z.string(), description: v4.z.string().optional().nullable(), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional().nullable(), inputSchema: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), groundTruthSchema: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), requestContextSchema: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), tags: v4.z.array(v4.z.string()).optional().nullable(), targetType: v4.z.string().optional().nullable(), targetIds: v4.z.array(v4.z.string()).optional().nullable(), scorerIds: v4.z.array(v4.z.string()).optional().nullable(), version: v4.z.number().int(), createdAt: v4.z.coerce.date(), updatedAt: v4.z.coerce.date() }); var datasetItemResponseSchema = v4.z.object({ id: v4.z.string(), datasetId: v4.z.string(), datasetVersion: v4.z.number().int(), input: v4.z.unknown(), groundTruth: v4.z.unknown().optional(), expectedTrajectory: v4.z.unknown().optional(), requestContext: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), source: datasetItemSourceSchema, createdAt: v4.z.coerce.date(), updatedAt: v4.z.coerce.date() }); var experimentResponseSchema = v4.z.object({ id: v4.z.string(), datasetId: v4.z.string().nullable(), datasetVersion: v4.z.number().int().nullable(), agentVersion: v4.z.string().nullable().optional(), targetType: v4.z.enum(["agent", "workflow", "scorer", "processor"]), targetId: v4.z.string(), name: v4.z.string().optional(), description: v4.z.string().optional(), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), status: v4.z.enum(["pending", "running", "completed", "failed"]), totalItems: v4.z.number(), succeededCount: v4.z.number(), failedCount: v4.z.number(), skippedCount: v4.z.number(), startedAt: v4.z.coerce.date().nullable(), completedAt: v4.z.coerce.date().nullable(), createdAt: v4.z.coerce.date(), updatedAt: v4.z.coerce.date() }); var scorerResultSchema = v4.z.object({ scorerId: v4.z.string(), scorerName: v4.z.string(), score: v4.z.number().nullable(), reason: v4.z.string().nullable(), error: v4.z.string().nullable() }); var experimentResultResponseSchema = v4.z.object({ id: v4.z.string(), experimentId: v4.z.string(), itemId: v4.z.string(), itemDatasetVersion: v4.z.number().int().nullable(), input: v4.z.unknown(), output: v4.z.unknown().nullable(), groundTruth: v4.z.unknown().nullable(), expectedTrajectory: v4.z.unknown().optional(), error: v4.z.object({ message: v4.z.string(), stack: v4.z.string().optional(), code: v4.z.string().optional() }).nullable(), startedAt: v4.z.coerce.date(), completedAt: v4.z.coerce.date(), retryCount: v4.z.number(), traceId: v4.z.string().nullable(), status: v4.z.enum(["needs-review", "reviewed", "complete"]).nullable().optional(), tags: v4.z.array(v4.z.string()).nullable().optional(), createdAt: v4.z.coerce.date() }); var updateExperimentResultBodySchema = v4.z.object({ status: v4.z.enum(["needs-review", "reviewed", "complete"]).nullable().optional(), tags: v4.z.array(v4.z.string()).optional() }); var comparisonItemSchema = v4.z.object({ itemId: v4.z.string(), input: v4.z.unknown().nullable(), groundTruth: v4.z.unknown().nullable(), results: v4.z.record( v4.z.string(), v4.z.object({ output: v4.z.unknown().nullable(), scores: v4.z.record(v4.z.string(), v4.z.number().nullable()) }).nullable() ) }); var comparisonResponseSchema = v4.z.object({ baselineId: v4.z.string(), items: v4.z.array(comparisonItemSchema) }); var experimentSummaryResponseSchema = v4.z.object({ experimentId: v4.z.string(), status: v4.z.enum(["pending", "running", "completed", "failed"]), totalItems: v4.z.number(), succeededCount: v4.z.number(), failedCount: v4.z.number(), startedAt: v4.z.coerce.date(), completedAt: v4.z.coerce.date().nullable(), results: v4.z.array( v4.z.object({ itemId: v4.z.string(), itemDatasetVersion: v4.z.number().int().nullable(), input: v4.z.unknown(), output: v4.z.unknown().nullable(), groundTruth: v4.z.unknown().nullable(), error: v4.z.string().nullable(), startedAt: v4.z.coerce.date(), completedAt: v4.z.coerce.date(), retryCount: v4.z.number(), scores: v4.z.array( v4.z.object({ scorerId: v4.z.string(), scorerName: v4.z.string(), score: v4.z.number().nullable(), reason: v4.z.string().nullable(), error: v4.z.string().nullable() }) ) }) ) }); var listDatasetsResponseSchema = v4.z.object({ datasets: v4.z.array(datasetResponseSchema), pagination: chunkDIG2K5CV_cjs.paginationInfoSchema }); var listItemsResponseSchema = v4.z.object({ items: v4.z.array(datasetItemResponseSchema), pagination: chunkDIG2K5CV_cjs.paginationInfoSchema }); var listExperimentsResponseSchema = v4.z.object({ experiments: v4.z.array(experimentResponseSchema), pagination: chunkDIG2K5CV_cjs.paginationInfoSchema }); var listExperimentResultsResponseSchema = v4.z.object({ results: v4.z.array(experimentResultResponseSchema), pagination: chunkDIG2K5CV_cjs.paginationInfoSchema }); var experimentReviewCountsSchema = v4.z.object({ experimentId: v4.z.string(), total: v4.z.number().int(), needsReview: v4.z.number().int(), reviewed: v4.z.number().int(), complete: v4.z.number().int() }); var reviewSummaryResponseSchema = v4.z.object({ counts: v4.z.array(experimentReviewCountsSchema) }); var datasetItemVersionPathParams = v4.z.object({ datasetId: v4.z.string().describe("Unique identifier for the dataset"), itemId: v4.z.string().describe("Unique identifier for the dataset item"), datasetVersion: v4.z.coerce.number().int().describe("Dataset version number") }); var itemVersionResponseSchema = v4.z.object({ id: v4.z.string(), datasetId: v4.z.string(), datasetVersion: v4.z.number().int(), input: v4.z.unknown(), groundTruth: v4.z.unknown().optional(), expectedTrajectory: v4.z.unknown().optional(), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), validTo: v4.z.number().int().nullable(), isDeleted: v4.z.boolean(), createdAt: v4.z.coerce.date(), updatedAt: v4.z.coerce.date() }); var listItemVersionsResponseSchema = v4.z.object({ history: v4.z.array(itemVersionResponseSchema) }); var datasetVersionResponseSchema = v4.z.object({ id: v4.z.string(), datasetId: v4.z.string(), version: v4.z.number().int(), createdAt: v4.z.coerce.date() }); var listDatasetVersionsResponseSchema = v4.z.object({ versions: v4.z.array(datasetVersionResponseSchema), pagination: chunkDIG2K5CV_cjs.paginationInfoSchema }); var batchInsertItemsBodySchema = v4.z.object({ items: v4.z.array( v4.z.object({ input: v4.z.unknown(), groundTruth: v4.z.unknown().optional(), expectedTrajectory: trajectoryExpectationSchema, requestContext: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), metadata: v4.z.record(v4.z.string(), v4.z.unknown()).optional(), source: datasetItemSourceSchema }) ) }); var batchInsertItemsResponseSchema = v4.z.object({ items: v4.z.array(datasetItemResponseSchema), count: v4.z.number() }); var batchDeleteItemsBodySchema = v4.z.object({ itemIds: v4.z.array(v4.z.string()) }); var batchDeleteItemsResponseSchema = v4.z.object({ success: v4.z.boolean(), deletedCount: v4.z.number() }); var generateItemsBodySchema = v4.z.object({ modelId: v4.z.string().describe('Model identifier in "provider/model" format (e.g., "openai/gpt-4o")'), prompt: v4.z.string().describe("Description of the kind of test data to generate"), count: v4.z.number().int().min(1).max(50).default(5).describe("Number of items to generate"), agentContext: v4.z.object({ description: v4.z.string().optional(), instructions: v4.z.string().optional(), tools: v4.z.array(v4.z.string()).optional() }).optional().describe("Context about the agent to generate relevant test data") }); var generatedItemSchema = v4.z.object({ input: v4.z.unknown(), groundTruth: v4.z.unknown().optional() }); var generateItemsResponseSchema = v4.z.object({ items: v4.z.array(generatedItemSchema) }); var clusterFailuresBodySchema = v4.z.object({ modelId: v4.z.string().describe('Model identifier in "provider/model" format (e.g., "openai/gpt-4o")'), items: v4.z.array( v4.z.object({ id: v4.z.string(), input: v4.z.unknown(), output: v4.z.unknown().optional(), error: v4.z.string().optional(), scores: v4.z.record(v4.z.string(), v4.z.number()).optional(), existingTags: v4.z.array(v4.z.string()).optional().describe("Tags already applied to this item") }) ).min(1).max(200).describe("Failure items to cluster"), availableTags: v4.z.array(v4.z.string()).optional().describe("Existing tag vocabulary from the dataset. The LLM should prefer reusing these tags when applicable."), prompt: v4.z.string().optional().describe('Optional user instructions to guide the analysis (e.g., "focus on tool usage failures")') }); var failureClusterSchema = v4.z.object({ id: v4.z.string().describe("A unique cluster identifier"), label: v4.z.string().describe("Short label for this failure pattern"), description: v4.z.string().describe("Description of the common pattern"), itemIds: v4.z.array(v4.z.string()).describe("IDs of items belonging to this cluster") }); var clusterFailuresResponseSchema = v4.z.object({ clusters: v4.z.array(failureClusterSchema), /** Per-item proposed tag assignments. Each entry maps an item ID to the tags the LLM suggests adding. */ proposedTags: v4.z.array( v4.z.object({ itemId: v4.z.string(), tags: v4.z.array(v4.z.string()), reason: v4.z.string().describe("Brief explanation of why these tags were assigned to this item") }) ).optional() }); exports.addItemBodySchema = addItemBodySchema; exports.batchDeleteItemsBodySchema = batchDeleteItemsBodySchema; exports.batchDeleteItemsResponseSchema = batchDeleteItemsResponseSchema; exports.batchInsertItemsBodySchema = batchInsertItemsBodySchema; exports.batchInsertItemsResponseSchema = batchInsertItemsResponseSchema; exports.clusterFailuresBodySchema = clusterFailuresBodySchema; exports.clusterFailuresResponseSchema = clusterFailuresResponseSchema; exports.compareExperimentsBodySchema = compareExperimentsBodySchema; exports.comparisonResponseSchema = comparisonResponseSchema; exports.createDatasetBodySchema = createDatasetBodySchema; exports.datasetAndExperimentIdPathParams = datasetAndExperimentIdPathParams; exports.datasetAndItemIdPathParams = datasetAndItemIdPathParams; exports.datasetIdPathParams = datasetIdPathParams; exports.datasetItemResponseSchema = datasetItemResponseSchema; exports.datasetItemVersionPathParams = datasetItemVersionPathParams; exports.datasetResponseSchema = datasetResponseSchema; exports.datasetVersionResponseSchema = datasetVersionResponseSchema; exports.experimentIdPathParams = experimentIdPathParams; exports.experimentResponseSchema = experimentResponseSchema; exports.experimentResultIdPathParams = experimentResultIdPathParams; exports.experimentResultResponseSchema = experimentResultResponseSchema; exports.experimentReviewCountsSchema = experimentReviewCountsSchema; exports.experimentSummaryResponseSchema = experimentSummaryResponseSchema; exports.generateItemsBodySchema = generateItemsBodySchema; exports.generateItemsResponseSchema = generateItemsResponseSchema; exports.itemIdPathParams = itemIdPathParams; exports.itemVersionResponseSchema = itemVersionResponseSchema; exports.listDatasetVersionsResponseSchema = listDatasetVersionsResponseSchema; exports.listDatasetsResponseSchema = listDatasetsResponseSchema; exports.listExperimentResultsResponseSchema = listExperimentResultsResponseSchema; exports.listExperimentsResponseSchema = listExperimentsResponseSchema; exports.listItemVersionsResponseSchema = listItemVersionsResponseSchema; exports.listItemsQuerySchema = listItemsQuerySchema; exports.listItemsResponseSchema = listItemsResponseSchema; exports.paginationQuerySchema = paginationQuerySchema; exports.reviewSummaryResponseSchema = reviewSummaryResponseSchema; exports.scorerResultSchema = scorerResultSchema; exports.triggerExperimentBodySchema = triggerExperimentBodySchema; exports.updateDatasetBodySchema = updateDatasetBodySchema; exports.updateExperimentResultBodySchema = updateExperimentResultBodySchema; exports.updateItemBodySchema = updateItemBodySchema; //# sourceMappingURL=chunk-EVIEMZPA.cjs.map //# sourceMappingURL=chunk-EVIEMZPA.cjs.map