diff --git a/evals/shared/types.ts b/evals/shared/types.ts index 29f2455f1..8f2097555 100644 --- a/evals/shared/types.ts +++ b/evals/shared/types.ts @@ -38,11 +38,29 @@ export type ToolSelectionTestCase = { * Test case for workflow evaluation (multi-turn agent conversations) * Used in: evals/workflows/ */ +export const WORKFLOW_EVAL_ARM = { + STANDARD: 'standard', + CODE_MODE: 'code-mode', +} as const; +export type WorkflowEvalArm = (typeof WORKFLOW_EVAL_ARM)[keyof typeof WORKFLOW_EVAL_ARM]; + export type WorkflowTestCase = { /** Maximum number of turns allowed (optional, defaults to config value) */ maxTurns?: number; /** Tools to enable for this test (optional, e.g., ["actors", "docs", "apify/rag-web-browser"]) */ tools?: string[]; + /** Enabled tools to hide from the agent and reject if called. */ + disallowedTools?: string[]; + /** Actor IDs the generic call-actor tool may run. */ + allowedCallActorTargets?: string[]; + /** Actor IDs the generic call-actor tool may not run. */ + disallowedCallActorTargets?: string[]; + /** Evaluation-only instructions appended to the agent system prompt. */ + agentInstructions?: string; + /** Groups variants of the same task. */ + pairId?: string; + /** Strategy variant for paired evaluations. */ + arm?: WorkflowEvalArm; } & BaseTestCase; /** diff --git a/evals/workflows/README.md b/evals/workflows/README.md index 6f8fd8ffb..a5e6d78c6 100644 --- a/evals/workflows/README.md +++ b/evals/workflows/README.md @@ -46,6 +46,12 @@ pnpm run evals:workflow -- --concurrency 8 # Save results to JSON file pnpm run evals:workflow -- --output + +# Run paired Code Mode evaluations +pnpm run evals:workflow -- \ + --test-cases-path evals/workflows/code_mode_test_cases.json \ + --results-path evals/workflows/code_mode_results.json \ + --tool-timeout 600 --concurrency 1 --output ``` **Exit codes:** @@ -139,21 +145,14 @@ while (turnNumber < maxTurns) { **Location:** `run-workflow-evals.ts` -### 4. Judge Sees Tool Calls, Not Results +### 4. Judge Sees Bounded Tool Results -**Decision:** Judge sees tool calls with arguments and agent responses, but NOT raw tool results. +**Decision:** Judge sees tool calls, final responses, and up to 4,000 characters from each tool result. **Why:** -- Evaluates agent behavior (tool selection, arguments) -- Tool results are often very long and noisy -- Agent should summarize results, judge evaluates the summary - -**Judge input format:** -``` -USER: Find actors for Google Maps -AGENT: [Called tool: search-actors with args: {"keywords":"google maps","limit":5}] -AGENT: I found 5 actors: 1. Google Maps Scraper... 2. ... -``` +- Tool evidence lets the judge detect unsupported final answers +- Per-result bounds prevent large datasets from dominating judge context +- Errors and Code Runtime output remain visible **Location:** `workflow-judge.ts` @@ -246,9 +245,13 @@ export OPENROUTER_API_KEY="your_openrouter_key" # Get from https://openrouter.ai | `--agent-model ` | | Override agent model | `anthropic/claude-haiku-4.5` | | `--judge-model ` | | Override judge model | `deepseek/deepseek-v4-flash` | | `--tool-timeout ` | | Tool call timeout | `60` | +| `--test-timeout ` | | Wall-clock timeout for one whole test case (agent + judge) | `300` | +| `--repeat ` | | Run each test case N times, print an aggregated pass/completion/error-rate summary | `1` | | `--concurrency ` | `-c` | Number of tests to run in parallel | `4` | | `--output` | `-o` | Save results to JSON file | `false` | -| `--baseline ` | | Results JSON to compare against (prints byte/token deltas) | `results.json` | +| `--results-path ` | | Results JSON file | `results.json` | +| `--baseline ` | | Results JSON to compare against (prints byte/token deltas) | results path | +| `--traces ` | | Write full per-turn traces (LLM output, tool calls, full args/results, untruncated) to this JSON file | not written | | `--help` | | Show help message | - | ### Line Range Filtering @@ -340,22 +343,81 @@ The `--tool-timeout` option sets the maximum time (in seconds) to wait for a sin pnpm run evals:workflow -- --tool-timeout 300 ``` +### Test Timeout + +`--tool-timeout` bounds one tool call; `--test-timeout` bounds the whole test +case — the agent's full multi-turn conversation plus judging — so a test +stuck retrying (e.g. a bad tool-call loop) can't block the run indefinitely. +Default: `300` seconds. + +When a test times out, it's recorded as `FAIL` with reason `"Test exceeded +its wall-clock timeout"` and the run moves on to the next test. The +abandoned work isn't cancelled mid-flight (no cancellation token threads +through the LLM/MCP clients) — the harness just stops waiting for it and +cleans up that test's MCP client. + +```bash +pnpm run evals:workflow -- --test-timeout 600 # for slower models / long chains +``` + +### Repeated Runs (`--repeat`) + +A single run of a test case can't distinguish "this approach is worse" from +ordinary run-to-run noise — flaky upstream Actors, model variance, and (at +higher `--concurrency`) rate-limit contention all inject noise unrelated to +the thing you're actually trying to measure. `--repeat ` runs each +(filtered) test case N times and prints an aggregated summary in addition to +the normal per-attempt table. + +**Job scheduling:** each `(test case, attempt)` pair is its own job through +the same `--concurrency` limiter as any other run — repeats of one test case +interleave with everything else exactly like distinct test cases do. For a +clean, low-noise comparison, run with `--concurrency 1` yourself; `--repeat` +doesn't invent a separate sequential mode for that. + +**What gets averaged, and what doesn't:** +- `--traces` and `--results-path` (`--output`) record **every attempt + individually**, tagged with `attemptIndex`/`totalAttempts` — never + averaged. Use these for the exact numbers behind any one run. +- The printed **Repeat Summary** aggregates per test case: + - `passRate` (strict PASS / N) and `completionRate` ((PASS + judge-FAIL) / + N) are reported separately — a wrong-but-complete answer and a timeout + are different failure modes needing different fixes, so they're not + collapsed into one number. + - Duration/token/tool-byte stats (median + mean) are computed **only over + completed attempts** (PASS or judge-FAIL). An errored or timed-out + attempt is excluded from these — including it would bias the "typical + duration" toward whatever the timeout cap happened to be, which isn't a + real measurement of anything. + - `timedOut` (exceeded `--test-timeout` specifically) is counted + separately from other errors. + +**Exit code:** `--repeat` exists to characterize flakiness across N +attempts, not to gate CI on one unlucky run — the process exits `0` +regardless of individual attempt outcomes when `--repeat > 1`. The default +(`--repeat 1`, unused) keeps the normal all-attempts-must-pass exit code. + +```bash +pnpm run evals:workflow -- --test-cases-path evals/workflows/code_mode_test_cases.json \ + --repeat 5 --concurrency 1 --test-timeout 900 --traces /tmp/code_mode_traces.json +``` + ### Saving Results to File -The `--output` (or `-o`) option saves test results to `evals/workflows/results.json` for tracking over time. +The `--output` (or `-o`) option saves test results to `evals/workflows/results.json`, or `--results-path`, for tracking over time. **How it works:** -- Results are stored per combination of: `agentModel:judgeModel:testId` -- Running the same test with the same models **overwrites** the previous result -- Running with different model combinations **adds** new entries -- Results are **versioned in git** for historical tracking +- Every run appends an attempt; previous attempts remain available +- Records include agent and judge token usage separately +- Records include tool calls, policy violations, final response, and complete conversation trace +- Results can be versioned in git or written to an ignored local path **Data structure:** ```json { - "version": "1.0", - "results": { - "anthropic/claude-haiku-4.5:x-ai/grok-4.1-fast:search-google-maps": { + "version": "2.0", + "attempts": [ + { "timestamp": "2026-01-07T10:45:23.123Z", "agentModel": "anthropic/claude-haiku-4.5", "judgeModel": "x-ai/grok-4.1-fast", @@ -368,9 +430,15 @@ The `--output` (or `-o`) option saves test results to `evals/workflows/results.j "promptTokens": 6231, "completionTokens": 412, "totalTokens": 6643, + "judgeUsage": { "promptTokens": 1800, "completionTokens": 40, "totalTokens": 1840 }, + "toolCalls": 3, + "failedToolCalls": 0, + "policyViolations": [], + "finalResponse": "...", + "toolCallTrace": [], "error": null } - } + ] } ``` @@ -378,13 +446,18 @@ The `--output` (or `-o`) option saves test results to `evals/workflows/results.j - `timestamp` - ISO timestamp when test was run - `agentModel` - LLM model used for the agent - `judgeModel` - LLM model used for judging -- `testId` - Test case identifier +- `testId`, `pairId`, `arm` - Test and experiment identifiers - `verdict` - `PASS` or `FAIL` - `reason` - Judge reasoning or error message - `durationMs` - Test duration in milliseconds - `turns` - Number of conversation turns - `resultBytes` - Total UTF-8 bytes of tool results returned to the agent across the conversation (measured at the point each result is fed to the LLM, so it reflects what the agent actually receives). Compare across branches to quantify byte savings. -- `promptTokens` / `completionTokens` / `totalTokens` - Tokens billed across all agent LLM calls (summed over turns; judge calls excluded). Tokens — not bytes — are what fill the context window, so this is the primary cost signal. Bytes are a deterministic, tokenizer-free proxy. +- `promptTokens` / `completionTokens` / `totalTokens` - Tokens billed across all agent LLM calls; judge calls excluded +- `cachedPromptTokens` / `reasoningTokens` - Provider-reported agent usage details +- `judgeUsage` - Judge tokens, kept separate from agent usage +- `toolCalls`, `failedToolCalls`, `policyViolations` - Execution counters and policy failures +- `finalResponse`, `toolCallTrace` - Agent answer and tool-call trace (name, arguments, success, byte size, `startedAt`/`durationMs`), including generated Code Mode scripts; result bodies are excluded to keep the file compact +- `usedCodeRuntime` - Whether the agent called `apify/code-runtime` at least once. Informational only — does not gate pass/fail, since a code-mode-arm agent may legitimately call other Actors directly for discovery. - `error` - Error message if execution failed, `null` otherwise **Examples:** @@ -404,10 +477,13 @@ pnpm run evals:workflow -- --agent-model openai/gpt-4o --output # Compare different judge models pnpm run evals:workflow -- --judge-model x-ai/grok-4.1-fast --output pnpm run evals:workflow -- --judge-model openai/gpt-4o --output + +# Save a separate benchmark +pnpm run evals:workflow -- --output --results-path evals/workflows/code_mode_results.json ``` **Partial runs:** -When using filters (`--category`, `--id`), only the filtered tests are updated in the results file. Other entries remain unchanged. +When using filters (`--category`, `--id`), attempts are appended only for matching tests. Existing attempts remain unchanged. **Version control:** The `results.json` file is tracked in git, allowing you to: @@ -419,7 +495,7 @@ The `results.json` file is tracked in git, allowing you to: Every run automatically compares against a baseline and prints per-test and aggregate **deltas** for tool bytes and tokens — no manual file diffing. This is how you answer "did this change grow the response size?". -- **Default baseline** is the committed `evals/workflows/results.json`. Each test is matched by its `agentModel:judgeModel:testId` key. +- **Default baseline** is the selected results path. Each test is matched by agent model and test ID. - **Custom baseline:** `--baseline ` compares against any saved results file. - Deltas read as `▼ -2.1 KB / -10.2%` (reduction) or `▲ +900 / +3.4%` (increase). Lower is better for both metrics. - This is **reporting only** — a regression never fails the run. Task success (all tests PASS) is the hard gate. @@ -433,6 +509,35 @@ cp evals/workflows/results.json /tmp/baseline.json pnpm run evals:workflow -- --baseline /tmp/baseline.json # prints byte/token deltas vs the baseline ``` +### Full Traces (`--traces`) + +`results.json`'s `toolCallTrace` is deliberately compact — it excludes tool +result bodies to stay small over time. To inspect exactly what happened in +one test case (every LLM final response, every tool call's full arguments, +every tool result's full untruncated content), pass `--traces `. It +writes one JSON array, one entry per test case, and is meant for manual +review — not for committing or diffing. + +```bash +pnpm run evals:workflow --test-cases-path evals/workflows/code_mode_test_cases.json --traces /tmp/code_mode_traces.json +``` + +Each entry: `testId`, `category`, `arm`, `pairId`, `query`, `durationMs`, +`error`, `verdict`, `judgeReason`, and `conversation` (the full +`ConversationHistory` — every turn's `toolCalls`, `toolResults` (including +`result`, the raw tool output), `usage`, and `finalResponse`). + +Every turn also carries `llmStartedAt` (ISO timestamp) / `llmDurationMs` for +its LLM call, and every tool result carries `startedAt` / `durationMs` for +that specific call — enough to reconstruct a full timeline of where time +went (LLM thinking vs. a specific nested Actor call). These same two fields +are also included in `results.json`'s compact `toolCallTrace`. + +**Traces are written even when a test times out (`--test-timeout`) or +errors** — the run's error/timeout path preserves whatever turns completed +before the cutoff, so a stuck test's trace still shows exactly which tool +call it was in and how long that call had already been running. + > **Bootstrap note:** records written before these metrics existed have no `resultBytes`/`*Tokens` fields, so the first run after this change shows `(no baseline)` for them and writes fresh values with `--output`. Subsequent runs show real deltas. ### Test Case Format @@ -447,7 +552,12 @@ File: `test-cases.json` "prompt": "User prompt for agent", "requirements": "What agent must do to pass", "maxTurns": 10, - "tools": ["actors", "docs"] + "tools": ["actors", "docs"], + "disallowedTools": ["add-actor"], + "agentInstructions": "Use regular MCP tools.", + "disallowedCallActorTargets": ["apify/code-runtime"], + "pairId": "test-pair", + "arm": "standard" } ] ``` @@ -460,7 +570,11 @@ File: `test-cases.json` **Optional:** - `maxTurns` - Override default (10) -- `tools` - List of tools to enable for this test (e.g., `["actors", "docs", "apify/rag-web-browser"]`). If omitted, all default tools are enabled. Passed to MCP server as `--tools` argument. +- `tools` - Server-side tool allowlist passed as `--tools` +- `disallowedTools` - Hide tools from the agent and reject direct calls +- `allowedCallActorTargets` / `disallowedCallActorTargets` - Restrict Actor IDs used through `call-actor` (optional; the shipped Code Mode cases only set `disallowedCallActorTargets: ["apify/code-runtime"]` on the standard arm — the code-mode arm leaves Actor targets unrestricted so the agent can call any Actor directly for discovery and still use `apify/code-runtime` for the actual processing) +- `agentInstructions` - Evaluation-only system instructions +- `pairId` / `arm` - Group standard and Code Mode variants of the same prompt ## Performance diff --git a/evals/workflows/code_mode_test_cases.json b/evals/workflows/code_mode_test_cases.json new file mode 100644 index 000000000..837eff2c2 --- /dev/null +++ b/evals/workflows/code_mode_test_cases.json @@ -0,0 +1,74 @@ +{ + "version": "1.0", + "testCases": [ + { + "id": "code-mode-instagram-5-standard", + "pairId": "code-mode-instagram-5", + "arm": "standard", + "category": "code-mode", + "query": "Show me the 5 latest posts from the natgeo Instagram profile. For each post, return its URL, publication time, and caption.", + "reference": "The agent must return five recent natgeo Instagram posts. Each result must include a post URL, publication time, and caption.", + "agentInstructions": "Use regular MCP Actor and storage tools. Do not use apify/code-runtime.", + "tools": ["actors", "get-dataset-items"], + "disallowedCallActorTargets": ["apify/code-runtime"], + "maxTurns": 18 + }, + { + "id": "code-mode-instagram-5-runtime", + "pairId": "code-mode-instagram-5", + "arm": "code-mode", + "category": "code-mode", + "query": "Show me the 5 latest posts from the natgeo Instagram profile. For each post, return its URL, publication time, and caption.", + "reference": "Check the tool-call trace for a call-actor invocation targeting apify/code-runtime. FAIL if apify/code-runtime was never called, even if the final answer looks correct — direct Actor calls are acceptable only for lightweight discovery (e.g. resolving which Actor to use), not as a substitute for running apify/code-runtime. Given apify/code-runtime was called, the final answer must list five recent natgeo Instagram posts, each with a post URL, publication time, and caption.", + "agentInstructions": "Before your first apify/code-runtime call, use fetch-actor-details on apify/code-runtime with output.readme to read its usage docs — do not guess its API from other Apify SDKs you know. You MUST call apify/code-runtime (via call-actor or its dedicated tool) at least once. Use it to run the Instagram scraper and process its output efficiently inside the sandbox, so only the small final result reaches you instead of the full raw dataset. Do not submit a final answer without having called apify/code-runtime — a response that skips it will be rejected regardless of accuracy. You may call other Actor tools directly only for lightweight discovery (e.g. resolving which Actor to use); the actual scrape and result processing must run inside apify/code-runtime.", + "tools": ["actors", "get-dataset-items", "apify/code-runtime"], + "maxTurns": 18 + }, + { + "id": "code-mode-maps-filter-standard", + "pairId": "code-mode-maps-filter", + "arm": "standard", + "category": "code-mode", + "query": "Scrape 100 cafes in Prague from Google Maps. Keep places rated at least 4.5 with at least 200 reviews, then return the top 10 by review count with name, website URL, rating, and review count.", + "reference": "The agent must use Google Maps data and return no more than ten Prague cafes. Every result must have rating at least 4.5 and review count at least 200, be sorted by review count descending, and include name, website URL when available, rating, and review count.", + "agentInstructions": "Use regular MCP Actor and storage tools. Do not use apify/code-runtime.", + "tools": ["actors", "get-dataset-items"], + "disallowedCallActorTargets": ["apify/code-runtime"], + "maxTurns": 20 + }, + { + "id": "code-mode-maps-filter-runtime", + "pairId": "code-mode-maps-filter", + "arm": "code-mode", + "category": "code-mode", + "query": "Scrape 100 cafes in Prague from Google Maps. Keep places rated at least 4.5 with at least 200 reviews, then return the top 10 by review count with name, website URL, rating, and review count.", + "reference": "Check the tool-call trace for a call-actor invocation targeting apify/code-runtime. FAIL if apify/code-runtime was never called, even if the final answer looks correct — direct Actor calls are acceptable only for lightweight discovery, not as a substitute for running apify/code-runtime. Given apify/code-runtime was called, the final answer must list no more than ten Prague cafes, each rated at least 4.5 with at least 200 reviews, sorted by review count descending, with name, website URL when available, rating, and review count.", + "agentInstructions": "Before your first apify/code-runtime call, use fetch-actor-details on apify/code-runtime with output.readme to read its usage docs — do not guess its API from other Apify SDKs you know. You MUST call apify/code-runtime (via call-actor or its dedicated tool) at least once. Use it to run the Google Maps scraper and to filter, sort, and project its output efficiently inside the sandbox, so only the top 10 results reach you instead of all 100 raw places. Do not submit a final answer without having called apify/code-runtime — a response that skips it will be rejected regardless of accuracy. You may call other Actor tools directly only for lightweight discovery (e.g. finding the right Actor or reading its schema); the bulk scrape and the filter/sort/projection logic must run inside apify/code-runtime.", + "tools": ["actors", "get-dataset-items", "apify/code-runtime"], + "maxTurns": 20 + }, + { + "id": "code-mode-maps-websites-standard", + "pairId": "code-mode-maps-websites", + "arm": "standard", + "category": "code-mode", + "query": "Scrape 20 cafes in Prague from Google Maps. For each place with a website, scrape only its landing page. Return the place name and website URL only when the landing page contains the string 'prague', case-insensitive.", + "reference": "The agent must scrape Google Maps places, visit each available website landing page, filter page content case-insensitively for 'prague', and return only matching place names with website URLs.", + "agentInstructions": "Use regular MCP Actor and storage tools. Do not use apify/code-runtime.", + "tools": ["actors", "get-dataset-items"], + "disallowedCallActorTargets": ["apify/code-runtime"], + "maxTurns": 30 + }, + { + "id": "code-mode-maps-websites-runtime", + "pairId": "code-mode-maps-websites", + "arm": "code-mode", + "category": "code-mode", + "query": "Scrape 20 cafes in Prague from Google Maps. For each place with a website, scrape only its landing page. Return the place name and website URL only when the landing page contains the string 'prague', case-insensitive.", + "reference": "Check the tool-call trace for a call-actor invocation targeting apify/code-runtime. FAIL if apify/code-runtime was never called, even if the final answer looks correct — direct Actor calls are acceptable only for lightweight discovery, not as a substitute for running apify/code-runtime. Given apify/code-runtime was called, the final answer must list only matching place names with website URLs, filtered case-insensitively for 'prague'.", + "agentInstructions": "Before your first apify/code-runtime call, use fetch-actor-details on apify/code-runtime with output.readme to read its usage docs — do not guess its API from other Apify SDKs you know. You MUST call apify/code-runtime (via call-actor or its dedicated tool) at least once. Use it to chain the Google Maps scrape into the per-website landing-page fetches and to apply the 'prague' filter efficiently inside the sandbox, so raw page content never reaches you — only the small filtered result does. Do not submit a final answer without having called apify/code-runtime — a response that skips it will be rejected regardless of accuracy. You may call other Actor tools directly only for lightweight discovery; the Actor chaining, fan-out over websites, and filtering must run inside apify/code-runtime.", + "tools": ["actors", "get-dataset-items", "apify/code-runtime"], + "maxTurns": 30 + } + ] +} diff --git a/evals/workflows/config.ts b/evals/workflows/config.ts index 414e7687d..fb593d044 100644 --- a/evals/workflows/config.ts +++ b/evals/workflows/config.ts @@ -53,6 +53,13 @@ export const MAX_CONVERSATION_TURNS = 10; */ export const DEFAULT_TOOL_TIMEOUT_SECONDS = 60; +/** + * Default wall-clock timeout for one whole test case (agent conversation + judging), in seconds. + * Bounds a single test that's stuck in a long tool-retry loop from blocking the run indefinitely. + * Override via CLI: --test-timeout 600 + */ +export const DEFAULT_TEST_TIMEOUT_SECONDS = 300; + /** * Judge prompt template for evaluating conversations * Uses structured output (JSON schema) - no format instructions needed @@ -80,7 +87,7 @@ Evaluation criteria: Important notes: - Focus on whether requirements were met, not on writing style - The agent may use different tools than expected if they accomplish the same goal -- Tool results are not shown (only tool calls and agent responses) +- Tool results are shown as bounded evidence and may be truncated - Minor inefficiencies are acceptable if the task was completed Provide your evaluation with a verdict (PASS or FAIL) and a brief explanation (1-2 sentences).`; diff --git a/evals/workflows/conversation_executor.ts b/evals/workflows/conversation_executor.ts index 49a7ed9c0..ae4a70bd7 100644 --- a/evals/workflows/conversation_executor.ts +++ b/evals/workflows/conversation_executor.ts @@ -25,6 +25,15 @@ export type ConversationExecutorOptions = { model?: string; /** Additional instructions from MCP server (optional) */ serverInstructions?: string | null; + /** Evaluation-only strategy instructions. */ + agentInstructions?: string; + /** + * Turns are pushed into this array as they happen, instead of a locally-scoped one, when + * provided. Lets a caller that raced this call against a timeout (see run_workflow_evals.ts) + * still read whatever turns completed before the cutoff, rather than losing the whole + * in-progress conversation. + */ + turns?: ConversationTurn[]; }; /** @@ -39,15 +48,18 @@ export async function executeConversation(options: ConversationExecutorOptions): maxTurns = MAX_CONVERSATION_TURNS, model = MODELS.agent, serverInstructions, + agentInstructions, + turns = [], } = options; - const turns: ConversationTurn[] = []; - // Build system prompt with optional server instructions let systemPrompt = AGENT_SYSTEM_PROMPT; if (serverInstructions) { systemPrompt += `\n\n## MCP Server Instructions\n\n${serverInstructions}`; } + if (agentInstructions) { + systemPrompt += `\n\n## Evaluation Instructions\n\n${agentInstructions}`; + } const messages: ChatCompletionMessageParam[] = [ { role: 'system', content: systemPrompt }, @@ -58,9 +70,13 @@ export async function executeConversation(options: ConversationExecutorOptions): let completed = false; let promptTokens = 0; let completionTokens = 0; - // Track whether the provider ever reported usage; if it never does, totals stay undefined - // rather than a fabricated 0 that reads as a real measurement. + let totalTokens = 0; + let cachedPromptTokens = 0; + let reasoningTokens = 0; + // Missing provider metrics stay undefined rather than becoming fabricated zeroes. let hasUsage = false; + let hasCachedPromptTokens = false; + let hasReasoningTokens = false; // Fetch tools initially let tools: ChatCompletionTool[] = mcpToolsToOpenAiTools(mcpClient.getTools()); @@ -69,13 +85,25 @@ export async function executeConversation(options: ConversationExecutorOptions): turnNumber++; // Call LLM with current conversation state and current tools + const llmStartedAt = new Date().toISOString(); + const llmCallStart = Date.now(); const llmResponse = await llmClient.callLlm(messages, model, tools); + const llmDurationMs = Date.now() - llmCallStart; // Accumulate token usage across the agent loop (cost grows with tool-result size) if (llmResponse.usage) { hasUsage = true; promptTokens += llmResponse.usage.promptTokens; completionTokens += llmResponse.usage.completionTokens; + totalTokens += llmResponse.usage.totalTokens; + if (llmResponse.usage.cachedPromptTokens !== undefined) { + hasCachedPromptTokens = true; + cachedPromptTokens += llmResponse.usage.cachedPromptTokens; + } + if (llmResponse.usage.reasoningTokens !== undefined) { + hasReasoningTokens = true; + reasoningTokens += llmResponse.usage.reasoningTokens; + } } // Check if LLM wants to call tools @@ -85,7 +113,10 @@ export async function executeConversation(options: ConversationExecutorOptions): turnNumber, toolCalls: [], toolResults: [], + usage: llmResponse.usage, finalResponse: llmResponse.content || '', + llmStartedAt, + llmDurationMs, }); completed = true; @@ -100,6 +131,9 @@ export async function executeConversation(options: ConversationExecutorOptions): arguments: JSON.parse(tc.arguments), })), toolResults: [], + usage: llmResponse.usage, + llmStartedAt, + llmDurationMs, }; // Add assistant message with tool calls to conversation @@ -142,10 +176,14 @@ export async function executeConversation(options: ConversationExecutorOptions): } // Execute tool via MCP + const toolStartedAt = new Date().toISOString(); + const toolCallStart = Date.now(); const result = await mcpClient.callTool({ name: toolCall.name, arguments: args, }); + result.startedAt = toolStartedAt; + result.durationMs = Date.now() - toolCallStart; // Serialize the tool result exactly as the agent (LLM) receives it, // and record its byte size to measure the data volume tools return. @@ -178,6 +216,8 @@ export async function executeConversation(options: ConversationExecutorOptions): totalTurns: turnNumber, promptTokens: hasUsage ? promptTokens : undefined, completionTokens: hasUsage ? completionTokens : undefined, - totalTokens: hasUsage ? promptTokens + completionTokens : undefined, + totalTokens: hasUsage ? totalTokens : undefined, + cachedPromptTokens: hasCachedPromptTokens ? cachedPromptTokens : undefined, + reasoningTokens: hasReasoningTokens ? reasoningTokens : undefined, }; } diff --git a/evals/workflows/llm_client.ts b/evals/workflows/llm_client.ts index 8ca467e1c..0f133a711 100644 --- a/evals/workflows/llm_client.ts +++ b/evals/workflows/llm_client.ts @@ -10,15 +10,7 @@ import type { ChatCompletionMessageParam, ChatCompletionTool } from 'openai/reso import type { ResponseFormatJSONSchema } from 'openai/resources/shared'; import { OPENROUTER_CONFIG } from './config.js'; - -/** - * Token usage reported by the LLM API for a single call - */ -export type LlmUsage = { - promptTokens: number; - completionTokens: number; - totalTokens: number; -}; +import type { TokenUsage } from './types.js'; /** * Response from LLM - either text or tool calls @@ -33,7 +25,7 @@ export type LlmResponse = { arguments: string; }[]; /** Token usage for this call (undefined if the provider did not report it) */ - usage?: LlmUsage; + usage?: TokenUsage; }; /** @@ -78,11 +70,13 @@ export class LlmClient { throw new Error('LLM returned no message'); } - const usage: LlmUsage | undefined = response.usage + const usage: TokenUsage | undefined = response.usage ? { promptTokens: response.usage.prompt_tokens, completionTokens: response.usage.completion_tokens, totalTokens: response.usage.total_tokens, + cachedPromptTokens: response.usage.prompt_tokens_details?.cached_tokens, + reasoningTokens: response.usage.completion_tokens_details?.reasoning_tokens, } : undefined; diff --git a/evals/workflows/mcp_client.ts b/evals/workflows/mcp_client.ts index f4bcd150d..b70080656 100644 --- a/evals/workflows/mcp_client.ts +++ b/evals/workflows/mcp_client.ts @@ -3,24 +3,31 @@ * Handles spawning, connecting, and communicating with the MCP server */ +import type { ChildProcess } from 'node:child_process'; + import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { HELPER_TOOLS } from '../../src/const.js'; import type { McpTool, McpToolCall, McpToolResult } from './types.js'; +export type McpToolPolicy = { + disallowedTools?: string[]; + allowedCallActorTargets?: string[]; + disallowedCallActorTargets?: string[]; +}; + export class McpClient { private client: Client | null = null; private transport: StdioClientTransport | null = null; private tools: McpTool[] = []; private instructions: string | null = null; private toolTimeoutMs: number; + private toolPolicy: McpToolPolicy; - /** - * Create MCP client - * @param toolTimeoutSeconds - Timeout for tool calls in seconds (default: 60) - */ - constructor(toolTimeoutSeconds = 60) { + constructor(toolTimeoutSeconds = 60, toolPolicy: McpToolPolicy = {}) { this.toolTimeoutMs = toolTimeoutSeconds * 1000; + this.toolPolicy = toolPolicy; } /** @@ -87,7 +94,8 @@ export class McpClient { } const response = await this.client.listTools(); - this.tools = response.tools as McpTool[]; + const disallowedTools = new Set(this.toolPolicy.disallowedTools); + this.tools = (response.tools as McpTool[]).filter((tool) => !disallowedTools.has(tool.name)); } /** @@ -112,6 +120,16 @@ export class McpClient { throw new Error('MCP client is not started'); } + const policyViolation = this.getPolicyViolation(toolCall); + if (policyViolation) { + return { + toolName: toolCall.name, + success: false, + error: policyViolation, + policyViolation, + }; + } + try { const response = await this.client.callTool( { @@ -142,12 +160,45 @@ export class McpClient { } } + private getPolicyViolation(toolCall: McpToolCall): string | undefined { + if (this.toolPolicy.disallowedTools?.includes(toolCall.name)) { + return `Evaluation policy forbids tool "${toolCall.name}".`; + } + if (toolCall.name !== HELPER_TOOLS.ACTOR_CALL) return undefined; + + const hasTargetPolicy = + this.toolPolicy.allowedCallActorTargets !== undefined || + this.toolPolicy.disallowedCallActorTargets !== undefined; + if (!hasTargetPolicy) return undefined; + + const { actor } = toolCall.arguments; + if (typeof actor !== 'string') return 'Evaluation policy requires call-actor to have an Actor target.'; + + const actorTarget = actor.split(':', 1)[0]; + if (this.toolPolicy.allowedCallActorTargets && !this.toolPolicy.allowedCallActorTargets.includes(actorTarget)) { + return `Evaluation policy does not allow call-actor target "${actorTarget}".`; + } + if (this.toolPolicy.disallowedCallActorTargets?.includes(actorTarget)) { + return `Evaluation policy forbids call-actor target "${actorTarget}".`; + } + return undefined; + } + /** * Cleanup and shutdown the MCP client * Uses a timeout to prevent indefinite waiting during cleanup */ async cleanup(cleanupTimeoutMs = 2000): Promise { - // Create timeout promise + // Grab the child process reference BEFORE calling close() — the SDK's own + // StdioClientTransport.close() clears its private `_process` field synchronously + // the instant it's called, then spends up to ~4s internally (wait for close, SIGTERM, + // wait again, SIGKILL) actually terminating it. If our outer race below times out first + // (2s < 4s worst case), grabbing `_process` from `this.transport` at that point would + // already be undefined — a no-op fallback that can never actually kill anything. Capturing + // it here keeps a live reference we can act on regardless of the SDK's own timing. + // eslint-disable-next-line no-underscore-dangle, @typescript-eslint/no-explicit-any + const child = (this.transport as any)?._process as ChildProcess | undefined; + const timeoutPromise = new Promise((resolve) => { setTimeout(() => resolve(), cleanupTimeoutMs); }); @@ -166,20 +217,24 @@ export class McpClient { // Race between cleanup and timeout await Promise.race([cleanupPromise, timeoutPromise]); - // Force kill transport process if it's still running - if (this.transport) { - try { - // Access the underlying child process and force kill it - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const transportAny = this.transport as any; - // eslint-disable-next-line no-underscore-dangle - if (transportAny._process && transportAny._process.kill) { - // eslint-disable-next-line no-underscore-dangle - transportAny._process.kill('SIGKILL'); + // If the child is still alive once our own budget runs out, kill it directly rather than + // leaving the SDK's own (possibly still in-progress) shutdown to finish it. A child left + // running past this point — e.g. still finishing a tool call a --test-timeout abandoned — + // can write its response to stdout after we've moved on, throwing an unhandled EPIPE in + // that (now orphaned) process. + if (child && child.exitCode === null && child.signalCode === null) { + await new Promise((resolve) => { + const done = () => resolve(); + child.once('exit', done); + try { + child.kill('SIGKILL'); + } catch { + // Already gone + done(); } - } catch { - // Ignore errors during force kill - } + // SIGKILL isn't instant either — don't let this hang cleanup() indefinitely. + setTimeout(done, 1000); + }); } // Always reset state regardless of cleanup success diff --git a/evals/workflows/output_formatter.ts b/evals/workflows/output_formatter.ts index 53f642b09..c775960aa 100644 --- a/evals/workflows/output_formatter.ts +++ b/evals/workflows/output_formatter.ts @@ -2,8 +2,9 @@ * Output formatter for evaluation results */ +import { HELPER_TOOLS } from '../../src/const.js'; import type { WorkflowTestCase } from './test_cases_loader.js'; -import type { ConversationHistory } from './types.js'; +import type { ConversationHistory, TokenUsage } from './types.js'; import type { JudgeResult } from './workflow_judge.js'; /** @@ -15,6 +16,12 @@ export type EvaluationResult = { judgeResult: JudgeResult; durationMs: number; error?: string; + /** True when `error` specifically means the test exceeded --test-timeout, not some other failure. */ + timedOut?: boolean; + /** 1-indexed attempt number for this test case (see --repeat); absent/1 when repeat is not used. */ + attemptIndex?: number; + /** Total attempts requested for this test case (see --repeat); absent/1 when repeat is not used. */ + totalAttempts?: number; }; /** @@ -31,6 +38,162 @@ export function sumResultBytes(conversation: ConversationHistory): number { return total; } +export function getToolCallCount(conversation: ConversationHistory): number { + return conversation.turns.reduce((total, turn) => total + turn.toolCalls.length, 0); +} + +export function getFailedToolCallCount(conversation: ConversationHistory): number { + return conversation.turns.reduce( + (total, turn) => total + turn.toolResults.filter((toolResult) => !toolResult.success).length, + 0, + ); +} + +export function getPolicyViolations(conversation: ConversationHistory): string[] { + return conversation.turns.flatMap((turn) => + turn.toolResults.flatMap((toolResult) => (toolResult.policyViolation ? [toolResult.policyViolation] : [])), + ); +} + +/** + * Distinct Actor IDs the agent targeted through call-actor (MCP tool-suffix stripped). + * Informational only — not used to gate pass/fail. + */ +export function getCallActorTargets(conversation: ConversationHistory): string[] { + const targets = new Set(); + for (const turn of conversation.turns) { + for (const toolCall of turn.toolCalls) { + if (toolCall.name !== HELPER_TOOLS.ACTOR_CALL) continue; + const { actor } = toolCall.arguments; + if (typeof actor === 'string') targets.add(actor.split(':', 1)[0]); + } + } + return [...targets]; +} + +export function getFinalResponse(conversation: ConversationHistory): string { + for (let index = conversation.turns.length - 1; index >= 0; index--) { + const { finalResponse } = conversation.turns[index]; + if (finalResponse !== undefined) return finalResponse; + } + return ''; +} + +export type ToolCallTraceEntry = { + turnNumber: number; + name: string; + arguments: Record; + success: boolean; + error?: string; + policyViolation?: string; + resultBytes?: number; + startedAt?: string; + durationMs?: number; +}; + +export function getToolCallTrace(conversation: ConversationHistory): ToolCallTraceEntry[] { + return conversation.turns.flatMap((turn) => + turn.toolCalls.map((toolCall, index) => { + const toolResult = turn.toolResults[index]; + return { + turnNumber: turn.turnNumber, + name: toolCall.name, + arguments: toolCall.arguments, + success: toolResult?.success ?? false, + error: toolResult?.error, + policyViolation: toolResult?.policyViolation, + resultBytes: toolResult?.resultBytes, + startedAt: toolResult?.startedAt, + durationMs: toolResult?.durationMs, + }; + }), + ); +} + +function median(values: number[]): number | undefined { + if (values.length === 0) return undefined; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} + +function mean(values: number[]): number | undefined { + if (values.length === 0) return undefined; + return values.reduce((sum, v) => sum + v, 0) / values.length; +} + +/** + * Aggregated outcome across repeated attempts of the same test case (see --repeat). + * + * passRate/completionRate are distinct on purpose: `completionRate` counts any attempt the + * agent finished (right or wrong answer), `passRate` only the correct ones. Averaging duration/ + * tokens/bytes over ALL attempts would bias toward errored/timed-out ones (e.g. a --test-timeout + * cap inflates the "typical" duration for a run that never actually finished) -- those stats are + * computed over completed attempts only. + */ +export type RepeatSummary = { + testId: string; + category: string; + attempts: number; + passed: number; + /** Judge FAIL -- agent finished but gave a wrong/incomplete answer. */ + failed: number; + /** Exceeded --test-timeout specifically. */ + timedOut: number; + /** Threw for any other reason. */ + errored: number; + passRate: number; + completionRate: number; + medianDurationMs?: number; + meanDurationMs?: number; + medianTokens?: number; + meanTokens?: number; + medianToolBytes?: number; + meanToolBytes?: number; +}; + +export function aggregateRepeatedResults(results: EvaluationResult[]): RepeatSummary[] { + const byTestId = new Map(); + for (const result of results) { + const group = byTestId.get(result.testCase.id) ?? []; + group.push(result); + byTestId.set(result.testCase.id, group); + } + + const summaries: RepeatSummary[] = []; + for (const [testId, group] of byTestId) { + const attempts = group.length; + const passed = group.filter((r) => !r.error && r.judgeResult.verdict === 'PASS').length; + const failed = group.filter((r) => !r.error && r.judgeResult.verdict === 'FAIL').length; + const timedOut = group.filter((r) => r.timedOut).length; + const errored = group.filter((r) => r.error && !r.timedOut).length; + + const completed = group.filter((r) => !r.error); + const durations = completed.map((r) => r.durationMs); + const tokens = completed.map((r) => r.conversation.totalTokens ?? 0); + const toolBytes = completed.map((r) => sumResultBytes(r.conversation)); + + summaries.push({ + testId, + category: group[0].testCase.category, + attempts, + passed, + failed, + timedOut, + errored, + passRate: passed / attempts, + completionRate: (passed + failed) / attempts, + medianDurationMs: median(durations), + meanDurationMs: mean(durations), + medianTokens: median(tokens), + meanTokens: mean(tokens), + medianToolBytes: median(toolBytes), + meanToolBytes: mean(toolBytes), + }); + } + return summaries; +} + /** * Format a byte count as a human-readable string (B / KB / MB). */ @@ -194,6 +357,50 @@ export function formatResultsTable(results: EvaluationResult[], baseline?: Map; + results?: Record; + attempts?: TestResultRecord[]; }; diff --git a/evals/workflows/race_with_timeout.ts b/evals/workflows/race_with_timeout.ts new file mode 100644 index 000000000..55661f65e --- /dev/null +++ b/evals/workflows/race_with_timeout.ts @@ -0,0 +1,14 @@ +/** Thrown when a test's wall-clock budget (--test-timeout) elapses; distinguishes a stuck test from any other execution error. */ +export class TestTimeoutError extends Error {} + +/** Rejects with TestTimeoutError after timeoutSecs if promise hasn't settled yet. Does not cancel promise's underlying work. */ +export async function raceWithTimeout(promise: Promise, timeoutSecs: number): Promise { + let timer: ReturnType; + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new TestTimeoutError(`Test exceeded ${timeoutSecs}s timeout`)), + timeoutSecs * 1000, + ); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} diff --git a/evals/workflows/results_writer.ts b/evals/workflows/results_writer.ts index 1161ec249..2c5446020 100644 --- a/evals/workflows/results_writer.ts +++ b/evals/workflows/results_writer.ts @@ -1,37 +1,58 @@ /** - * Results writer for persisting test results to JSON file - * Stores latest result per (agentModel, judgeModel, testId) combination + * Results writer for persisting workflow evaluation attempts. */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname } from 'node:path'; +import { actorNameToToolName } from '../../src/tools/actor_tool_naming.js'; import type { EvaluationResult, ResultsDatabase, TestResultRecord } from './output_formatter.js'; -import { sumResultBytes } from './output_formatter.js'; +import { + getCallActorTargets, + getFailedToolCallCount, + getFinalResponse, + getPolicyViolations, + getToolCallCount, + getToolCallTrace, + sumResultBytes, +} from './output_formatter.js'; +import type { ConversationHistory } from './types.js'; + +const CODE_RUNTIME_ACTOR_ID = 'apify/code-runtime'; +// The MCP server also exposes a pre-loaded Actor as its own dedicated tool (see `tools_loader.ts`), +// so code-runtime usage must be detected on both paths: generic call-actor, and this direct tool. +const CODE_RUNTIME_TOOL_NAME = actorNameToToolName(CODE_RUNTIME_ACTOR_ID); + +function wasCodeRuntimeInvoked(conversation: ConversationHistory): boolean { + if (getCallActorTargets(conversation).includes(CODE_RUNTIME_ACTOR_ID)) return true; + return conversation.turns.some((turn) => + turn.toolCalls.some((toolCall) => toolCall.name === CODE_RUNTIME_TOOL_NAME), + ); +} /** - * Build composite key for storing results - * Format: "{agentModel}:{judgeModel}:{testId}" + * Build composite key used by legacy results files. */ export function buildResultKey(agentModel: string, judgeModel: string, testId: string): string { return `${agentModel}:${judgeModel}:${testId}`; } +function getRecords(database: ResultsDatabase): TestResultRecord[] { + return [...Object.values(database.results ?? {}), ...(database.attempts ?? [])]; +} + /** - * Find the baseline record for byte/token deltas, matched by agent model + test ID. - * The judge model is excluded on purpose: tool bytes and agent token counts are - * produced by the agent, not the judge, so a baseline recorded with a different - * judge is still a valid comparison. When several judges recorded the same - * agent/test, prefer a record that carries metrics, then the most recent. + * Find newest baseline record with metrics for an agent model and test. */ export function findBaselineRecord( database: ResultsDatabase, agentModel: string, testId: string, ): TestResultRecord | undefined { - const hasMetrics = (r: TestResultRecord): boolean => r.resultBytes !== undefined || r.totalTokens !== undefined; + const hasMetrics = (record: TestResultRecord): boolean => + record.resultBytes !== undefined || record.totalTokens !== undefined; let best: TestResultRecord | undefined; - for (const record of Object.values(database.results)) { + for (const record of getRecords(database)) { if (record.agentModel !== agentModel || record.testId !== testId) continue; if ( best === undefined || @@ -45,27 +66,24 @@ export function findBaselineRecord( } /** - * Load existing results database from file - * Returns empty database if file doesn't exist + * Load existing results, or create an empty append-only database. */ export function loadResultsDatabase(filePath: string): ResultsDatabase { if (!existsSync(filePath)) { return { - version: '1.0', - results: {}, + version: '2.0', + attempts: [], }; } try { - const fileContent = readFileSync(filePath, 'utf-8'); - const data = JSON.parse(fileContent) as ResultsDatabase; - - // Validate structure - if (!data.version || !data.results || typeof data.results !== 'object') { - throw new Error('Invalid database structure: missing version or results field'); + const database = JSON.parse(readFileSync(filePath, 'utf-8')) as ResultsDatabase; + const hasResults = database.results && typeof database.results === 'object'; + const hasAttempts = Array.isArray(database.attempts); + if (!database.version || (!hasResults && !hasAttempts)) { + throw new Error('Invalid database structure: missing version, results, or attempts field'); } - - return data; + return database; } catch (error) { throw new Error( `Failed to load results database from ${filePath}: ${error instanceof Error ? error.message : String(error)}`, @@ -73,20 +91,11 @@ export function loadResultsDatabase(filePath: string): ResultsDatabase { } } -/** - * Save results database to file with pretty formatting - */ export function saveResultsDatabase(filePath: string, database: ResultsDatabase): void { try { - // Ensure parent directory exists - const dir = dirname(filePath); - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }); - } - - // Write with pretty formatting (2-space indent) - const json = JSON.stringify(database, null, 2); - writeFileSync(filePath, json, 'utf-8'); + const directory = dirname(filePath); + if (!existsSync(directory)) mkdirSync(directory, { recursive: true }); + writeFileSync(filePath, JSON.stringify(database, null, 2), 'utf-8'); } catch (error) { throw new Error( `Failed to save results database to ${filePath}: ${error instanceof Error ? error.message : String(error)}`, @@ -95,53 +104,47 @@ export function saveResultsDatabase(filePath: string, database: ResultsDatabase) } /** - * Convert EvaluationResult to TestResultRecord + * Convert one evaluation attempt to its persisted form. */ export function convertEvaluationResultToRecord( result: EvaluationResult, agentModel: string, judgeModel: string, ): TestResultRecord { - // Handle error cases - if (result.error) { - return { - timestamp: new Date().toISOString(), - agentModel, - judgeModel, - testId: result.testCase.id, - verdict: 'FAIL', - reason: result.error, - durationMs: result.durationMs, - turns: result.conversation.totalTurns, - resultBytes: sumResultBytes(result.conversation), - promptTokens: result.conversation.promptTokens ?? 0, - completionTokens: result.conversation.completionTokens ?? 0, - totalTokens: result.conversation.totalTokens ?? 0, - error: result.error, - }; - } - - // Normal case + const { conversation } = result; return { timestamp: new Date().toISOString(), agentModel, judgeModel, testId: result.testCase.id, - verdict: result.judgeResult.verdict, - reason: result.judgeResult.reason, + pairId: result.testCase.pairId, + arm: result.testCase.arm, + verdict: result.error ? 'FAIL' : result.judgeResult.verdict, + reason: result.error ?? result.judgeResult.reason, durationMs: result.durationMs, - turns: result.conversation.totalTurns, - resultBytes: sumResultBytes(result.conversation), - promptTokens: result.conversation.promptTokens ?? 0, - completionTokens: result.conversation.completionTokens ?? 0, - totalTokens: result.conversation.totalTokens ?? 0, - error: null, + turns: conversation.totalTurns, + resultBytes: sumResultBytes(conversation), + promptTokens: conversation.promptTokens, + completionTokens: conversation.completionTokens, + totalTokens: conversation.totalTokens, + cachedPromptTokens: conversation.cachedPromptTokens, + reasoningTokens: conversation.reasoningTokens, + judgeUsage: result.judgeResult.usage, + toolCalls: getToolCallCount(conversation), + failedToolCalls: getFailedToolCallCount(conversation), + policyViolations: getPolicyViolations(conversation), + usedCodeRuntime: wasCodeRuntimeInvoked(conversation), + finalResponse: getFinalResponse(conversation), + toolCallTrace: getToolCallTrace(conversation), + error: result.error ?? null, + timedOut: result.timedOut, + attemptIndex: result.attemptIndex, + totalAttempts: result.totalAttempts, }; } /** - * Update results database with new evaluation results - * Only updates entries for tests that ran (preserves other entries) + * Append evaluation attempts without replacing earlier runs. */ export function updateResultsWithEvaluations( database: ResultsDatabase, @@ -149,18 +152,11 @@ export function updateResultsWithEvaluations( agentModel: string, judgeModel: string, ): ResultsDatabase { - // Clone database to avoid mutation - const updatedDatabase: ResultsDatabase = { - version: database.version, - results: { ...database.results }, + return { + version: '2.0', + attempts: [ + ...getRecords(database), + ...results.map((result) => convertEvaluationResultToRecord(result, agentModel, judgeModel)), + ], }; - - // Update each test result - for (const result of results) { - const record = convertEvaluationResultToRecord(result, agentModel, judgeModel); - const key = buildResultKey(agentModel, judgeModel, result.testCase.id); - updatedDatabase.results[key] = record; - } - - return updatedDatabase; } diff --git a/evals/workflows/run_workflow_evals.ts b/evals/workflows/run_workflow_evals.ts index 5b620cf9b..2f207d9ee 100644 --- a/evals/workflows/run_workflow_evals.ts +++ b/evals/workflows/run_workflow_evals.ts @@ -21,12 +21,18 @@ import { hideBin } from 'yargs/helpers'; import { filterByLineRanges } from '../shared/line_range_filter.js'; import type { LineRange } from '../shared/line_range_parser.js'; import { checkRangesOutOfBounds, parseLineRanges, validateLineRanges } from '../shared/line_range_parser.js'; -import { DEFAULT_TOOL_TIMEOUT_SECONDS, MODELS, sanitizeEnvValue } from './config.js'; +import { DEFAULT_TEST_TIMEOUT_SECONDS, DEFAULT_TOOL_TIMEOUT_SECONDS, MODELS, sanitizeEnvValue } from './config.js'; import { executeConversation } from './conversation_executor.js'; import { LlmClient } from './llm_client.js'; import { McpClient } from './mcp_client.js'; import type { EvaluationResult, TestResultRecord } from './output_formatter.js'; -import { formatDetailedResult, formatResultsTable } from './output_formatter.js'; +import { + aggregateRepeatedResults, + formatDetailedResult, + formatRepeatSummaryTable, + formatResultsTable, +} from './output_formatter.js'; +import { raceWithTimeout, TestTimeoutError } from './race_with_timeout.js'; import { findBaselineRecord, loadResultsDatabase, @@ -35,6 +41,8 @@ import { } from './results_writer.js'; import type { WorkflowTestCase, WorkflowTestCaseWithLineNumbers } from './test_cases_loader.js'; import { filterTestCases, loadTestCases, loadTestCasesWithLineNumbers } from './test_cases_loader.js'; +import { writeTraces } from './traces_writer.js'; +import type { ConversationTurn } from './types.js'; import { evaluateConversation } from './workflow_judge.js'; type CliArgs = { @@ -48,7 +56,11 @@ type CliArgs = { toolTimeout?: number; concurrency?: number; output?: boolean; + resultsPath?: string; baseline?: string; + traces?: string; + testTimeout?: number; + repeat?: number; }; /** @@ -71,84 +83,128 @@ async function runSingleTest( argv: CliArgs, llmClient: LlmClient, apifyToken: string, + attemptIndex: number, + totalAttempts: number, ): Promise { const testId = testCase.id; + // Distinguish repeated attempts of the same test case in log output; unchanged when + // --repeat is not used (totalAttempts === 1). + const logId = totalAttempts > 1 ? `${testId}#${attemptIndex}/${totalAttempts}` : testId; - logWithPrefix(testId, `[${index + 1}/${total}] Running...`); + logWithPrefix(logId, `[${index + 1}/${total}] Running...`); // Create FRESH MCP instance per test for isolation - const mcpClient = new McpClient(argv.toolTimeout); + const mcpClient = new McpClient(argv.toolTimeout, { + disallowedTools: testCase.disallowedTools, + allowedCallActorTargets: testCase.allowedCallActorTargets, + disallowedCallActorTargets: testCase.disallowedCallActorTargets, + }); const startTime = Date.now(); let result: EvaluationResult; + // Shared with executeConversation() so a timed-out test still has whatever turns + // completed before the cutoff, instead of an empty conversation (see the catch block). + const turnsSoFar: ConversationTurn[] = []; + try { - // Start MCP server with test-specific tools (if configured) - await mcpClient.start(apifyToken, testCase.tools); - - // Get server instructions (if provided) - const serverInstructions = mcpClient.getInstructions(); - - // Execute conversation (tools fetched dynamically inside) - const conversation = await executeConversation({ - userPrompt: testCase.query, - mcpClient, - llmClient, - maxTurns: testCase.maxTurns, - model: argv.agentModel, - serverInstructions, - }); + const runTest = async () => { + // Start MCP server with test-specific tools (if configured) + await mcpClient.start(apifyToken, testCase.tools); - // Judge conversation - const judgeResult = await evaluateConversation(testCase, conversation, llmClient, argv.judgeModel); + // Get server instructions (if provided) + const serverInstructions = mcpClient.getInstructions(); - const durationMs = Date.now() - startTime; + // Execute conversation (tools fetched dynamically inside) + const conversation = await executeConversation({ + userPrompt: testCase.query, + mcpClient, + llmClient, + maxTurns: testCase.maxTurns, + model: argv.agentModel, + serverInstructions, + agentInstructions: testCase.agentInstructions, + turns: turnsSoFar, + }); + + // Judge conversation + const judgeResult = await evaluateConversation(testCase, conversation, llmClient, argv.judgeModel); + + const policyViolation = conversation.turns.some((turn) => + turn.toolResults.some((toolResult) => toolResult.policyViolation), + ); + const finalJudgeResult = policyViolation + ? { + ...judgeResult, + verdict: 'FAIL' as const, + reason: 'Agent violated the evaluation tool policy.', + } + : judgeResult; + return { conversation, judgeResult: finalJudgeResult }; + }; + + // Keep a direct reference so a late rejection (test timed out, but the abandoned + // work keeps running and eventually throws) doesn't surface as an unhandled rejection. + const testPromise = runTest(); + testPromise.catch(() => {}); + const { conversation, judgeResult: finalJudgeResult } = await raceWithTimeout( + testPromise, + argv.testTimeout ?? DEFAULT_TEST_TIMEOUT_SECONDS, + ); + + const durationMs = Date.now() - startTime; result = { testCase, conversation, - judgeResult, + judgeResult: finalJudgeResult, durationMs, + attemptIndex, + totalAttempts, }; logWithPrefix( - testId, - ` ${judgeResult.verdict === 'PASS' ? '✅' : '❌'} ${judgeResult.verdict} (${durationMs}ms)`, + logId, + ` ${finalJudgeResult.verdict === 'PASS' ? '✅' : '❌'} ${finalJudgeResult.verdict} (${durationMs}ms)`, ); } catch (error) { const durationMs = Date.now() - startTime; + const timedOut = error instanceof TestTimeoutError; result = { testCase, conversation: { userPrompt: testCase.query, - turns: [], + turns: turnsSoFar, completed: false, hitMaxTurns: false, - totalTurns: 0, + totalTurns: turnsSoFar.length, }, judgeResult: { verdict: 'FAIL', - reason: 'Error during execution', + reason: timedOut ? 'Test exceeded its wall-clock timeout' : 'Error during execution', rawResponse: '', }, durationMs, error: error instanceof Error ? error.message : String(error), + timedOut, + attemptIndex, + totalAttempts, }; - logWithPrefix(testId, ` 🔥 ERROR (${durationMs}ms)`); + logWithPrefix(logId, ` ${timedOut ? '⏱️ TIMEOUT' : '🔥 ERROR'} (${durationMs}ms)`); } finally { // ALWAYS cleanup MCP client for this test try { await mcpClient.cleanup(); } catch (cleanupError) { - logWithPrefix(testId, ` ⚠️ Cleanup failed: ${cleanupError}`); + logWithPrefix(logId, ` ⚠️ Cleanup failed: ${cleanupError}`); } } // Show detailed output if verbose if (argv.verbose) { - logWithPrefix(testId, ''); - logWithPrefix(testId, formatDetailedResult(result)); + logWithPrefix(logId, ''); + logWithPrefix(logId, formatDetailedResult(result)); } return result; @@ -196,6 +252,11 @@ async function main() { description: `Tool call timeout in seconds (default: ${DEFAULT_TOOL_TIMEOUT_SECONDS})`, default: DEFAULT_TOOL_TIMEOUT_SECONDS, }) + .option('test-timeout', { + type: 'number', + description: `Wall-clock timeout in seconds for one whole test case (agent + judge), default: ${DEFAULT_TEST_TIMEOUT_SECONDS}`, + default: DEFAULT_TEST_TIMEOUT_SECONDS, + }) .option('concurrency', { alias: 'c', type: 'number', @@ -205,15 +266,31 @@ async function main() { .option('output', { alias: 'o', type: 'boolean', - description: 'Save test results to JSON file (evals/workflows/results.json)', + description: 'Save test results to JSON file', default: false, }) + .option('results-path', { + type: 'string', + description: 'Results JSON file (default: evals/workflows/results.json)', + }) .option('baseline', { type: 'string', description: 'Results JSON file to compare against; prints byte/token deltas per test ' + '(default: evals/workflows/results.json)', }) + .option('traces', { + type: 'string', + description: + 'Write full per-turn traces (LLM output, tool calls, full args/results, untruncated) to this JSON file', + }) + .option('repeat', { + type: 'number', + description: + 'Run each test case this many times and print an aggregated pass/completion/error-rate summary ' + + '(default: 1). --traces and --results-path record every attempt individually, never averaged.', + default: 1, + }) .help().argv) as CliArgs; console.log('='.repeat(100)); @@ -318,7 +395,10 @@ async function main() { // Load baseline for byte/token deltas (read before --output overwrites results.json). // Matched by agent model + test ID; the judge model is ignored because bytes/tokens // come from the agent, so a baseline recorded with a different judge still compares. - const baselinePath = argv.baseline ?? path.join(process.cwd(), 'evals/workflows/results.json'); + const resultsPath = argv.resultsPath + ? path.resolve(process.cwd(), argv.resultsPath) + : path.join(process.cwd(), 'evals/workflows/results.json'); + const baselinePath = argv.baseline ? path.resolve(process.cwd(), argv.baseline) : resultsPath; const baselineByTestId = new Map(); let baselineWithMetrics = 0; try { @@ -356,16 +436,33 @@ async function main() { const llmClient = new LlmClient(); // Run evaluations - console.log(`▶️ Running ${filteredTestCases.length} evaluation(s) with concurrency ${argv.concurrency}...`); + const repeat = argv.repeat!; + const totalJobs = filteredTestCases.length * repeat; + console.log( + `▶️ Running ${filteredTestCases.length} evaluation(s)` + + (repeat > 1 ? ` × ${repeat} repeat(s) = ${totalJobs} job(s)` : '') + + ` with concurrency ${argv.concurrency}...`, + ); console.log(); // Create concurrency limiter const limit = pLimit(argv.concurrency!); + // Each (test case, attempt) pair is its own job through the same limiter -- repeats of one + // test case interleave with everything else exactly like distinct test cases do. Run + // sequentially yourself via --concurrency 1 for a clean, low-noise comparison; --repeat + // doesn't invent a separate scheduling mode for that. + const jobs: { testCase: WorkflowTestCase; attemptIndex: number }[] = []; + for (const testCase of filteredTestCases) { + for (let attemptIndex = 1; attemptIndex <= repeat; attemptIndex++) { + jobs.push({ testCase, attemptIndex }); + } + } + // Execute tests in parallel with concurrency control - const resultPromises = filteredTestCases.map(async (testCase, index) => { + const resultPromises = jobs.map(async ({ testCase, attemptIndex }, index) => { return limit(async () => { - return runSingleTest(testCase, index, filteredTestCases.length, argv, llmClient, apifyToken); + return runSingleTest(testCase, index, totalJobs, argv, llmClient, apifyToken, attemptIndex, repeat); }); }); @@ -374,7 +471,6 @@ async function main() { // Save results to file if --output flag is present if (argv.output) { - const resultsPath = path.join(process.cwd(), 'evals/workflows/results.json'); try { const database = loadResultsDatabase(resultsPath); const updatedDatabase = updateResultsWithEvaluations(database, results, argv.agentModel!, argv.judgeModel!); @@ -388,15 +484,38 @@ async function main() { } } - // Display results (with byte/token deltas when a baseline matched) + // Save full per-turn traces if --traces flag is present + if (argv.traces) { + try { + const tracesPath = path.resolve(process.cwd(), argv.traces); + writeTraces(tracesPath, results); + console.log(`✅ Traces saved to: ${tracesPath}`); + console.log(); + } catch (error) { + console.error(`❌ Failed to save traces: ${error}`); + console.log(); + } + } + + // Display results (with byte/token deltas when a baseline matched) -- every individual + // attempt, never averaged; that's what --traces and --results-path also record. console.log(formatResultsTable(results, baselineByTestId.size > 0 ? baselineByTestId : undefined)); + if (repeat > 1) { + console.log(formatRepeatSummaryTable(aggregateRepeatedResults(results))); + } + // Exit with appropriate code - ALL tests must pass const totalTests = results.length; const passedTests = results.filter((r) => !r.error && r.judgeResult.verdict === 'PASS').length; const errorTests = results.filter((r) => r.error).length; - // Exit 0 only if ALL tests passed with no errors + // --repeat exists to characterize flakiness across N attempts, not to gate on one unlucky + // run -- exit 0 regardless of individual outcomes (the summary above is for a human to read). + // Default (repeat = 1) keeps the strict all-must-pass gate unchanged. + if (repeat > 1) { + process.exit(0); + } const allPassed = totalTests > 0 && passedTests === totalTests && errorTests === 0; process.exit(allPassed ? 0 : 1); } diff --git a/evals/workflows/test_cases_loader.ts b/evals/workflows/test_cases_loader.ts index a3faadc0e..3268e62a9 100644 --- a/evals/workflows/test_cases_loader.ts +++ b/evals/workflows/test_cases_loader.ts @@ -11,7 +11,7 @@ import { filterTestCases as filterTestCasesShared, loadTestCases as loadTestCasesShared, } from '../shared/test_case_loader.js'; -import type { WorkflowTestCase } from '../shared/types.js'; +import { WORKFLOW_EVAL_ARM, type WorkflowTestCase } from '../shared/types.js'; // Re-export WorkflowTestCase type for backwards compatibility export type { WorkflowTestCase } from '../shared/types.js'; @@ -21,51 +21,76 @@ export type { WorkflowTestCase } from '../shared/types.js'; */ export type WorkflowTestCaseWithLineNumbers = WorkflowTestCase & TestCaseWithLineNumbers; -/** - * Load workflow test cases from JSON file with validation - */ -export function loadTestCases(filePath?: string): WorkflowTestCase[] { - const testCasesPath = filePath || path.join(process.cwd(), 'evals/workflows/test_cases.json'); - - if (!fs.existsSync(testCasesPath)) { - throw new Error(`Test cases file not found: ${testCasesPath}`); - } - - // Use shared loader - const testData = loadTestCasesShared(testCasesPath); - const testCases = testData.testCases as WorkflowTestCase[]; - - // Validate test cases +function validateTestCases(testCases: WorkflowTestCase[]): void { const seenIds = new Set(); - for (let i = 0; i < testCases.length; i++) { - const tc = testCases[i]; - const testCaseRef = `Test case #${i + 1} (id: ${tc.id || 'missing'})`; - - // Check required fields + for (let index = 0; index < testCases.length; index++) { + const testCase = testCases[index]; + const testCaseRef = `Test case #${index + 1} (id: ${testCase.id || 'missing'})`; const missingFields: string[] = []; - if (!tc.id) missingFields.push('id'); - if (!tc.category) missingFields.push('category'); - if (!tc.query) missingFields.push('query'); - if (!tc.reference) missingFields.push('reference'); + if (!testCase.id) missingFields.push('id'); + if (!testCase.category) missingFields.push('category'); + if (!testCase.query) missingFields.push('query'); + if (!testCase.reference) missingFields.push('reference'); if (missingFields.length > 0) { throw new Error( `${testCaseRef}: Missing or empty required field(s): ${missingFields.join(', ')}\n` + `Required fields: id, category, query, reference\n` + - `Test case: ${JSON.stringify(tc, null, 2)}`, + `Test case: ${JSON.stringify(testCase, null, 2)}`, ); } - - // Check for duplicate IDs - if (seenIds.has(tc.id)) { + if (seenIds.has(testCase.id)) { throw new Error( - `${testCaseRef}: Duplicate test case ID '${tc.id}'\n` + `Each test case must have a unique ID.`, + `${testCaseRef}: Duplicate test case ID '${testCase.id}'\nEach test case must have a unique ID.`, ); } - seenIds.add(tc.id); + seenIds.add(testCase.id); + + const enabledTools = new Set(testCase.tools); + const duplicateTool = testCase.disallowedTools?.find((tool) => enabledTools.has(tool)); + if (duplicateTool) { + throw new Error(`${testCaseRef}: Tool '${duplicateTool}' is both enabled and disallowed.`); + } + + const allowedTargets = new Set(testCase.allowedCallActorTargets); + const duplicateTarget = testCase.disallowedCallActorTargets?.find((target) => allowedTargets.has(target)); + if (duplicateTarget) { + throw new Error(`${testCaseRef}: Actor target '${duplicateTarget}' is both allowed and disallowed.`); + } + + if (testCase.arm && !Object.values(WORKFLOW_EVAL_ARM).includes(testCase.arm)) { + throw new Error(`${testCaseRef}: Unsupported evaluation arm '${testCase.arm}'.`); + } } + const pairs = new Map(); + for (const testCase of testCases) { + if (!testCase.pairId) continue; + pairs.set(testCase.pairId, [...(pairs.get(testCase.pairId) ?? []), testCase]); + } + for (const [pairId, pair] of pairs) { + if (new Set(pair.map((testCase) => testCase.query)).size > 1) { + throw new Error(`Test case pair '${pairId}' must use the same query in every arm.`); + } + } +} + +/** + * Load workflow test cases from JSON file with validation + */ +export function loadTestCases(filePath?: string): WorkflowTestCase[] { + const testCasesPath = path.resolve(filePath || path.join(process.cwd(), 'evals/workflows/test_cases.json')); + + if (!fs.existsSync(testCasesPath)) { + throw new Error(`Test cases file not found: ${testCasesPath}`); + } + + // Use shared loader + const testData = loadTestCasesShared(testCasesPath); + const testCases = testData.testCases as WorkflowTestCase[]; + + validateTestCases(testCases); return testCases; } @@ -91,7 +116,7 @@ export function loadTestCasesWithLineNumbers(filePath?: string): { testCases: WorkflowTestCaseWithLineNumbers[]; totalLines: number; } { - const testCasesPath = filePath || path.join(process.cwd(), 'evals/workflows/test_cases.json'); + const testCasesPath = path.resolve(filePath || path.join(process.cwd(), 'evals/workflows/test_cases.json')); if (!fs.existsSync(testCasesPath)) { throw new Error(`Test cases file not found: ${testCasesPath}`); @@ -106,36 +131,7 @@ export function loadTestCasesWithLineNumbers(filePath?: string): { const testData = JSON.parse(fileContent); const testCases = testData.testCases as WorkflowTestCase[]; - // Validate test cases (same as loadTestCases) - const seenIds = new Set(); - - for (let i = 0; i < testCases.length; i++) { - const tc = testCases[i]; - const testCaseRef = `Test case #${i + 1} (id: ${tc.id || 'missing'})`; - - // Check required fields - const missingFields: string[] = []; - if (!tc.id) missingFields.push('id'); - if (!tc.category) missingFields.push('category'); - if (!tc.query) missingFields.push('query'); - if (!tc.reference) missingFields.push('reference'); - - if (missingFields.length > 0) { - throw new Error( - `${testCaseRef}: Missing or empty required field(s): ${missingFields.join(', ')}\n` + - `Required fields: id, category, query, reference\n` + - `Test case: ${JSON.stringify(tc, null, 2)}`, - ); - } - - // Check for duplicate IDs - if (seenIds.has(tc.id)) { - throw new Error( - `${testCaseRef}: Duplicate test case ID '${tc.id}'\n` + `Each test case must have a unique ID.`, - ); - } - seenIds.add(tc.id); - } + validateTestCases(testCases); // Attach line numbers to each test case by finding their position in the file const testCasesWithLines: WorkflowTestCaseWithLineNumbers[] = []; diff --git a/evals/workflows/traces_writer.ts b/evals/workflows/traces_writer.ts new file mode 100644 index 000000000..fc478be04 --- /dev/null +++ b/evals/workflows/traces_writer.ts @@ -0,0 +1,58 @@ +/** + * Full-fidelity trace output for manual review. + * + * Unlike results.json (compact, append-only, tool-result bodies excluded — + * see output_formatter.ts's getToolCallTrace), a trace keeps every turn's + * full tool-call arguments, full tool results, and full LLM final response, + * untruncated. Written only when --traces is passed; meant to be read by a + * human inspecting exactly what one test case did, not diffed for metrics. + */ +import fs from 'node:fs'; +import path from 'node:path'; + +import type { EvaluationResult } from './output_formatter.js'; +import type { ConversationHistory } from './types.js'; + +export type TestTrace = { + testId: string; + category: string; + arm?: string; + pairId?: string; + query: string; + durationMs: number; + error?: string; + timedOut?: boolean; + verdict: string; + judgeReason: string; + /** 1-indexed attempt number for this test case (see --repeat); 1 when repeat is not used. */ + attemptIndex: number; + /** Total attempts requested for this test case (see --repeat); 1 when repeat is not used. */ + totalAttempts: number; + conversation: ConversationHistory; +}; + +function toTrace(result: EvaluationResult): TestTrace { + const { testCase, conversation, judgeResult, durationMs, error, timedOut, attemptIndex, totalAttempts } = result; + return { + testId: testCase.id, + category: testCase.category, + arm: testCase.arm, + pairId: testCase.pairId, + query: testCase.query, + durationMs, + error, + timedOut, + verdict: judgeResult.verdict, + judgeReason: judgeResult.reason, + attemptIndex: attemptIndex ?? 1, + totalAttempts: totalAttempts ?? 1, + conversation, + }; +} + +/** Write full per-turn traces for a batch of results, overwriting any existing file at filePath. */ +export function writeTraces(filePath: string, results: EvaluationResult[]): void { + const traces = results.map(toTrace); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(traces, null, 2)); +} diff --git a/evals/workflows/types.ts b/evals/workflows/types.ts index 938ee06b2..ce7b3c7bb 100644 --- a/evals/workflows/types.ts +++ b/evals/workflows/types.ts @@ -24,8 +24,14 @@ export type McpToolResult = { result?: unknown; /** Error message if execution failed */ error?: string; + /** Evaluation policy violated by this call. */ + policyViolation?: string; /** UTF-8 byte size of the serialized content the agent receives (set when the result is fed to the LLM) */ resultBytes?: number; + /** ISO timestamp when this tool call started (before the MCP request was sent) */ + startedAt?: string; + /** Wall-clock time the call took, in milliseconds (request + response, not just server-side execution) */ + durationMs?: number; }; /** @@ -48,6 +54,14 @@ export type McpTool = { /** * A single turn in the conversation (agent action) */ +export type TokenUsage = { + promptTokens: number; + completionTokens: number; + totalTokens: number; + cachedPromptTokens?: number; + reasoningTokens?: number; +}; + export type ConversationTurn = { /** Turn number (1-indexed) */ turnNumber: number; @@ -58,8 +72,14 @@ export type ConversationTurn = { }[]; /** Tool results for this turn (if any) */ toolResults: McpToolResult[]; + /** Token usage for this agent call. */ + usage?: TokenUsage; /** Final text response from agent (if no more tool calls) */ finalResponse?: string; + /** ISO timestamp when this turn's LLM call started */ + llmStartedAt?: string; + /** Wall-clock time the LLM call took, in milliseconds */ + llmDurationMs?: number; }; /** @@ -82,4 +102,8 @@ export type ConversationHistory = { completionTokens?: number; /** Total tokens billed across all agent LLM calls (prompt + completion) */ totalTokens?: number; + /** Cached prompt tokens reported across all agent LLM calls */ + cachedPromptTokens?: number; + /** Reasoning tokens reported across all agent LLM calls */ + reasoningTokens?: number; }; diff --git a/evals/workflows/workflow_judge.ts b/evals/workflows/workflow_judge.ts index a07dc5eba..37f78df29 100644 --- a/evals/workflows/workflow_judge.ts +++ b/evals/workflows/workflow_judge.ts @@ -9,7 +9,9 @@ import type { ResponseFormatJSONSchema } from 'openai/resources/shared'; import type { WorkflowTestCase } from '../shared/types.js'; import { JUDGE_PROMPT_TEMPLATE, MODELS } from './config.js'; import type { LlmClient } from './llm_client.js'; -import type { ConversationHistory } from './types.js'; +import type { ConversationHistory, McpToolResult, TokenUsage } from './types.js'; + +const MAX_TOOL_RESULT_CHARS = 4_000; /** * Judge evaluation result @@ -21,6 +23,8 @@ export type JudgeResult = { reason: string; /** Raw response from judge (for debugging) */ rawResponse: string; + /** Token usage for the judge call. */ + usage?: TokenUsage; }; /** @@ -51,11 +55,20 @@ const JUDGE_RESPONSE_SCHEMA: ResponseFormatJSONSchema = { }, }; +function formatToolResultForJudge(toolResult: McpToolResult): string { + const serialized = JSON.stringify( + toolResult.success + ? toolResult.result + : { error: toolResult.error, policyViolation: toolResult.policyViolation }, + ); + if (serialized.length <= MAX_TOOL_RESULT_CHARS) return serialized; + return `${serialized.slice(0, MAX_TOOL_RESULT_CHARS)}...[truncated ${serialized.length - MAX_TOOL_RESULT_CHARS} chars]`; +} + /** - * Format conversation for judge evaluation - * Judge sees: tool calls + arguments + final responses (NOT tool results) + * Format conversation for judge evaluation with bounded tool-result evidence. */ -function formatConversationForJudge(conversation: ConversationHistory): string { +export function formatConversationForJudge(conversation: ConversationHistory): string { const lines: string[] = []; // User prompt @@ -70,6 +83,12 @@ function formatConversationForJudge(conversation: ConversationHistory): string { lines.push(`AGENT: [Called tool: ${toolCall.name} with args: ${JSON.stringify(toolCall.arguments)}]`); } } + for (const toolResult of turn.toolResults) { + lines.push( + `TOOL: [${toolResult.toolName} ${toolResult.success ? 'succeeded' : 'failed'}] ` + + formatToolResultForJudge(toolResult), + ); + } // Show final response (if present) if (turn.finalResponse) { @@ -142,6 +161,7 @@ export async function evaluateConversation( verdict, reason, rawResponse, + usage: response.usage, }; } catch (error) { throw new Error( diff --git a/tests/unit/evals.conversation_executor.test.ts b/tests/unit/evals.conversation_executor.test.ts index 1843d79b1..5ca7aa6b4 100644 --- a/tests/unit/evals.conversation_executor.test.ts +++ b/tests/unit/evals.conversation_executor.test.ts @@ -1,9 +1,10 @@ +import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions'; import { describe, expect, it } from 'vitest'; import { executeConversation } from '../../evals/workflows/conversation_executor.js'; import type { LlmClient, LlmResponse } from '../../evals/workflows/llm_client.js'; import type { McpClient } from '../../evals/workflows/mcp_client.js'; -import type { McpToolResult } from '../../evals/workflows/types.js'; +import type { ConversationTurn, McpToolResult } from '../../evals/workflows/types.js'; /** LLM client that replays a scripted list of responses, one per turn. */ function makeLlmClient(responses: LlmResponse[]): LlmClient { @@ -39,19 +40,54 @@ const finalResponse = (usage?: LlmResponse['usage']): LlmResponse => ({ }); describe('executeConversation()', () => { + it('appends evaluation instructions to the system prompt', async () => { + let messages: ChatCompletionMessageParam[] = []; + const llmClient = { + callLlm: async (receivedMessages: ChatCompletionMessageParam[]): Promise => { + messages = receivedMessages; + return finalResponse(); + }, + } as unknown as LlmClient; + + await executeConversation({ + userPrompt: 'go', + agentInstructions: 'Use Code Mode.', + mcpClient: makeMcpClient({ toolName: 'search-actors', success: true }), + llmClient, + }); + + expect(messages[0].content).toContain('## Evaluation Instructions'); + expect(messages[0].content).toContain('Use Code Mode.'); + }); + it('accumulates token usage across multiple turns', async () => { const conversation = await executeConversation({ userPrompt: 'go', mcpClient: makeMcpClient({ toolName: 'search-actors', success: true, result: { items: [] } }), llmClient: makeLlmClient([ - toolCallResponse({ promptTokens: 10, completionTokens: 5, totalTokens: 15 }), - finalResponse({ promptTokens: 20, completionTokens: 7, totalTokens: 27 }), + toolCallResponse({ + promptTokens: 10, + completionTokens: 5, + totalTokens: 15, + cachedPromptTokens: 3, + reasoningTokens: 2, + }), + finalResponse({ + promptTokens: 20, + completionTokens: 7, + totalTokens: 27, + cachedPromptTokens: 4, + reasoningTokens: 1, + }), ]), }); expect(conversation.promptTokens).toBe(30); expect(conversation.completionTokens).toBe(12); expect(conversation.totalTokens).toBe(42); + expect(conversation.cachedPromptTokens).toBe(7); + expect(conversation.reasoningTokens).toBe(3); + expect(conversation.turns.map((turn) => turn.usage?.totalTokens)).toEqual([15, 27]); }); it('records resultBytes on a successful tool result', async () => { @@ -91,5 +127,55 @@ describe('executeConversation()', () => { expect(conversation.promptTokens).toBe(20); expect(conversation.completionTokens).toBe(7); expect(conversation.totalTokens).toBe(27); + expect(conversation.cachedPromptTokens).toBeUndefined(); + expect(conversation.reasoningTokens).toBeUndefined(); + }); + + it('pushes turns into a caller-supplied array as they happen, not just at the end', async () => { + const turns: ConversationTurn[] = []; + + const conversation = await executeConversation({ + userPrompt: 'go', + mcpClient: makeMcpClient({ toolName: 'search-actors', success: true, result: { items: [] } }), + llmClient: makeLlmClient([toolCallResponse(), toolCallResponse(), finalResponse()]), + turns, + }); + + // Same array reference throughout — a caller that raced this call against a timeout + // (see run_workflow_evals.ts) can read `turns` for whatever completed before the cutoff. + expect(turns).toBe(conversation.turns); + expect(turns).toHaveLength(3); + }); + + it('records when each LLM call and tool call started and how long it took', async () => { + const llmClient = { + callLlm: async (): Promise => { + await new Promise((resolve) => setTimeout(resolve, 5)); + return toolCallResponse(); + }, + } as unknown as LlmClient; + const mcpClient = { + getTools: () => [], + getInstructions: () => null, + callTool: async (): Promise => { + await new Promise((resolve) => setTimeout(resolve, 5)); + return { toolName: 'search-actors', success: true, result: { items: [] } }; + }, + } as unknown as McpClient; + + const conversation = await executeConversation({ + userPrompt: 'go', + mcpClient, + llmClient, + maxTurns: 1, + }); + + const [turn] = conversation.turns; + expect(turn.llmStartedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/); + expect(turn.llmDurationMs).toBeGreaterThanOrEqual(5); + + const [toolResult] = turn.toolResults; + expect(toolResult.startedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/); + expect(toolResult.durationMs).toBeGreaterThanOrEqual(5); }); }); diff --git a/tests/unit/evals.mcp_client_cleanup.test.ts b/tests/unit/evals.mcp_client_cleanup.test.ts new file mode 100644 index 000000000..04167c9e6 --- /dev/null +++ b/tests/unit/evals.mcp_client_cleanup.test.ts @@ -0,0 +1,70 @@ +import { EventEmitter } from 'node:events'; + +import { describe, expect, it, vi } from 'vitest'; + +import { McpClient } from '../../evals/workflows/mcp_client.js'; + +/** Minimal stand-in for a spawned child_process.ChildProcess, enough for cleanup()'s liveness check. */ +function makeFakeChild() { + const child = new EventEmitter() as EventEmitter & { + exitCode: number | null; + signalCode: string | null; + kill: (signal: string) => boolean; + }; + child.exitCode = null; + child.signalCode = null; + child.kill = vi.fn((signal: string) => { + // Simulate the OS actually terminating the process shortly after the signal. + setImmediate(() => { + child.exitCode = null; + child.signalCode = signal; + child.emit('exit', null, signal); + }); + return true; + }); + return child; +} + +function makeClientWithFakeTransport(child: ReturnType, closeDelayMs: number): McpClient { + const client = new McpClient(60); + Object.assign(client, { + client: { close: async () => new Promise((resolve) => setTimeout(resolve, closeDelayMs)) }, + transport: { close: async () => {}, _process: child }, + }); + return client; +} + +describe('McpClient.cleanup()', () => { + it('SIGKILLs a child still alive once the graceful-close race times out', async () => { + const child = makeFakeChild(); + // client.close() never settles within cleanup()'s own timeout, forcing the fallback path. + const client = makeClientWithFakeTransport(child, 10_000); + + await client.cleanup(50); + + expect(child.kill).toHaveBeenCalledWith('SIGKILL'); + }); + + it('does not kill a child that already exited before cleanup ran', async () => { + const child = makeFakeChild(); + child.exitCode = 0; + const client = makeClientWithFakeTransport(child, 0); + + await client.cleanup(50); + + expect(child.kill).not.toHaveBeenCalled(); + }); + + it('resolves promptly once the killed child reports exit, without waiting out the full fallback budget', async () => { + const child = makeFakeChild(); + const client = makeClientWithFakeTransport(child, 10_000); + + const start = Date.now(); + await client.cleanup(50); + const elapsedMs = Date.now() - start; + + // Fallback budget is a 1s safety cap; the fake child "exits" on the next tick after + // kill(), so cleanup() should return well before that cap is reached. + expect(elapsedMs).toBeLessThan(900); + }); +}); diff --git a/tests/unit/evals.mcp_client_policy.test.ts b/tests/unit/evals.mcp_client_policy.test.ts new file mode 100644 index 000000000..f3ccc86f8 --- /dev/null +++ b/tests/unit/evals.mcp_client_policy.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; + +import { McpClient } from '../../evals/workflows/mcp_client.js'; +import type { McpTool } from '../../evals/workflows/types.js'; + +function createConnectedClient(policy: ConstructorParameters[1]): McpClient { + const client = new McpClient(60, policy); + Object.assign(client, { + client: { + callTool: async () => ({ isError: false, content: [{ type: 'text', text: 'ok' }] }), + }, + }); + return client; +} + +describe('McpClient.callTool()', () => { + it('rejects a disallowed tool', async () => { + const client = createConnectedClient({ disallowedTools: ['add-actor'] }); + const result = await client.callTool({ name: 'add-actor', arguments: {} }); + + expect(result.success).toBe(false); + expect(result.policyViolation).toContain('forbids tool'); + }); + + it('rejects a call-actor target outside the allowlist', async () => { + const client = createConnectedClient({ allowedCallActorTargets: ['apify/code-runtime'] }); + const result = await client.callTool({ + name: 'call-actor', + arguments: { actor: 'apify/instagram-scraper' }, + }); + + expect(result.success).toBe(false); + expect(result.policyViolation).toContain('does not allow'); + }); + + it('allows an MCP tool suffix on an allowed Actor target', async () => { + const client = createConnectedClient({ allowedCallActorTargets: ['apify/code-runtime'] }); + const result = await client.callTool({ + name: 'call-actor', + arguments: { actor: 'apify/code-runtime:run' }, + }); + + expect(result.success).toBe(true); + expect(result.policyViolation).toBeUndefined(); + }); +}); + +describe('McpClient.getTools()', () => { + it('hides disallowed tools from the agent', async () => { + const client = new McpClient(60, { disallowedTools: ['add-actor'] }); + const tools: McpTool[] = [ + { name: 'add-actor', inputSchema: { type: 'object' } }, + { name: 'call-actor', inputSchema: { type: 'object' } }, + ]; + Object.assign(client, { client: { listTools: async () => ({ tools }) } }); + await (client as unknown as { loadTools: () => Promise }).loadTools(); + + expect(client.getTools().map((tool) => tool.name)).toEqual(['call-actor']); + }); +}); diff --git a/tests/unit/evals.race_with_timeout.test.ts b/tests/unit/evals.race_with_timeout.test.ts new file mode 100644 index 000000000..cfc3f0d45 --- /dev/null +++ b/tests/unit/evals.race_with_timeout.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; + +import { raceWithTimeout, TestTimeoutError } from '../../evals/workflows/race_with_timeout.js'; + +describe('raceWithTimeout()', () => { + it('resolves with the promise value when it settles before the timeout', async () => { + const result = await raceWithTimeout(Promise.resolve('done'), 1); + expect(result).toBe('done'); + }); + + it('rejects with the promise error when it rejects before the timeout', async () => { + await expect(raceWithTimeout(Promise.reject(new Error('boom')), 1)).rejects.toThrow('boom'); + }); + + it('rejects with TestTimeoutError once timeoutSecs elapses without the promise settling', async () => { + const neverSettles = new Promise(() => {}); + await expect(raceWithTimeout(neverSettles, 0.01)).rejects.toThrow(TestTimeoutError); + }); +}); diff --git a/tests/unit/evals.results_writer.test.ts b/tests/unit/evals.results_writer.test.ts index 29d1f2a2e..3853116c7 100644 --- a/tests/unit/evals.results_writer.test.ts +++ b/tests/unit/evals.results_writer.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it } from 'vitest'; -import type { ResultsDatabase, TestResultRecord } from '../../evals/workflows/output_formatter.js'; -import { buildResultKey, findBaselineRecord } from '../../evals/workflows/results_writer.js'; +import type { EvaluationResult, ResultsDatabase, TestResultRecord } from '../../evals/workflows/output_formatter.js'; +import { + buildResultKey, + convertEvaluationResultToRecord, + findBaselineRecord, + updateResultsWithEvaluations, +} from '../../evals/workflows/results_writer.js'; function makeRecord(overrides: Partial): TestResultRecord { return { @@ -17,19 +22,78 @@ function makeRecord(overrides: Partial): TestResultRecord { promptTokens: 0, completionTokens: 0, totalTokens: 500, + toolCalls: 1, + failedToolCalls: 0, + policyViolations: [], + finalResponse: 'done', + toolCallTrace: [], error: null, ...overrides, }; } function makeDatabase(records: TestResultRecord[]): ResultsDatabase { - const results: ResultsDatabase['results'] = {}; + const results: NonNullable = {}; for (const record of records) { results[buildResultKey(record.agentModel, record.judgeModel, record.testId)] = record; } return { version: '1.0', results }; } +describe('convertEvaluationResultToRecord() usedCodeRuntime', () => { + function makeResult(toolCalls: { name: string; arguments: Record }[]): EvaluationResult { + return { + testCase: { id: 'test', category: 'code-mode', query: 'query', reference: 'reference' }, + conversation: { + userPrompt: 'query', + turns: [{ turnNumber: 1, toolCalls, toolResults: [] }], + completed: true, + hitMaxTurns: false, + totalTurns: 1, + }, + judgeResult: { verdict: 'PASS', reason: 'ok', rawResponse: '' }, + durationMs: 100, + }; + } + + it('detects code-runtime called through generic call-actor', () => { + const result = makeResult([{ name: 'call-actor', arguments: { actor: 'apify/code-runtime' } }]); + expect(convertEvaluationResultToRecord(result, 'agent', 'judge').usedCodeRuntime).toBe(true); + }); + + it('detects code-runtime called through its dedicated pre-loaded tool', () => { + const result = makeResult([{ name: 'apify--code-runtime', arguments: { code: 'console.log(1)' } }]); + expect(convertEvaluationResultToRecord(result, 'agent', 'judge').usedCodeRuntime).toBe(true); + }); + + it('is false when neither path was used', () => { + const result = makeResult([{ name: 'call-actor', arguments: { actor: 'apify/instagram-scraper' } }]); + expect(convertEvaluationResultToRecord(result, 'agent', 'judge').usedCodeRuntime).toBe(false); + }); +}); + +describe('updateResultsWithEvaluations()', () => { + it('appends repeated attempts', () => { + const result = { + testCase: { id: 'test', category: 'code-mode', query: 'query', reference: 'reference' }, + conversation: { + userPrompt: 'query', + turns: [], + completed: true, + hitMaxTurns: false, + totalTurns: 1, + }, + judgeResult: { verdict: 'PASS', reason: 'ok', rawResponse: '' }, + durationMs: 100, + } satisfies EvaluationResult; + + const first = updateResultsWithEvaluations({ version: '2.0', attempts: [] }, [result], 'agent', 'judge'); + const second = updateResultsWithEvaluations(first, [result], 'agent', 'judge'); + + expect(second.attempts).toHaveLength(2); + }); +}); + describe('findBaselineRecord()', () => { it('matches by agent model and test ID regardless of the judge model', () => { const database = makeDatabase([makeRecord({ judgeModel: 'grok', resultBytes: 2000 })]); diff --git a/tests/unit/evals.test_cases_loader.test.ts b/tests/unit/evals.test_cases_loader.test.ts new file mode 100644 index 000000000..56ee1e716 --- /dev/null +++ b/tests/unit/evals.test_cases_loader.test.ts @@ -0,0 +1,48 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { loadTestCases } from '../../evals/workflows/test_cases_loader.js'; + +const temporaryDirectories: string[] = []; + +function writeTestCases(testCases: unknown[]): string { + const directory = mkdtempSync(join(tmpdir(), 'workflow-evals-')); + temporaryDirectories.push(directory); + const filePath = join(directory, 'test_cases.json'); + writeFileSync(filePath, JSON.stringify({ version: '1.0', testCases })); + return filePath; +} + +function createTestCase(overrides: Record = {}) { + return { + id: 'test', + category: 'code-mode', + query: 'Do the task', + reference: 'Complete the task', + ...overrides, + }; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('loadTestCases()', () => { + it('rejects a tool that is both enabled and disallowed', () => { + const path = writeTestCases([createTestCase({ tools: ['call-actor'], disallowedTools: ['call-actor'] })]); + + expect(() => loadTestCases(path)).toThrow("Tool 'call-actor' is both enabled and disallowed"); + }); + + it('rejects paired arms with different user queries', () => { + const path = writeTestCases([ + createTestCase({ id: 'standard', pairId: 'pair', query: 'First query' }), + createTestCase({ id: 'runtime', pairId: 'pair', query: 'Second query' }), + ]); + + expect(() => loadTestCases(path)).toThrow("pair 'pair' must use the same query"); + }); +}); diff --git a/tests/unit/evals.traces_writer.test.ts b/tests/unit/evals.traces_writer.test.ts new file mode 100644 index 000000000..273d3c0e2 --- /dev/null +++ b/tests/unit/evals.traces_writer.test.ts @@ -0,0 +1,91 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import type { EvaluationResult } from '../../evals/workflows/output_formatter.js'; +import { writeTraces } from '../../evals/workflows/traces_writer.js'; + +function makeResult(overrides: Partial = {}): EvaluationResult { + return { + testCase: { id: 'test-1', category: 'code-mode', query: 'query', pairId: 'pair-1', arm: 'code-mode' }, + conversation: { + userPrompt: 'query', + turns: [ + { + turnNumber: 1, + toolCalls: [ + { name: 'call-actor', arguments: { actor: 'apify/code-runtime', input: { code: 'x' } } }, + ], + toolResults: [{ toolName: 'call-actor', success: true, result: { items: [1, 2, 3] } }], + finalResponse: 'done', + }, + ], + completed: true, + hitMaxTurns: false, + totalTurns: 1, + }, + judgeResult: { verdict: 'PASS', reason: 'looks good', rawResponse: '' }, + durationMs: 1234, + ...overrides, + }; +} + +describe('writeTraces()', () => { + const tmpFiles: string[] = []; + + afterEach(() => { + for (const file of tmpFiles.splice(0)) fs.rmSync(file, { force: true, recursive: true }); + }); + + function tmpPath(): string { + const file = path.join(os.tmpdir(), `traces-${Date.now()}-${Math.random()}.json`); + tmpFiles.push(file); + return file; + } + + it('writes one entry per result with full tool call args and results', () => { + const filePath = tmpPath(); + writeTraces(filePath, [makeResult()]); + + const written = JSON.parse(fs.readFileSync(filePath, 'utf8')); + expect(written).toHaveLength(1); + expect(written[0]).toMatchObject({ + testId: 'test-1', + category: 'code-mode', + arm: 'code-mode', + pairId: 'pair-1', + query: 'query', + durationMs: 1234, + verdict: 'PASS', + judgeReason: 'looks good', + }); + // Full, untruncated tool call arguments and tool result are preserved. + expect(written[0].conversation.turns[0].toolCalls[0].arguments).toEqual({ + actor: 'apify/code-runtime', + input: { code: 'x' }, + }); + expect(written[0].conversation.turns[0].toolResults[0].result).toEqual({ items: [1, 2, 3] }); + }); + + it('creates parent directories that do not exist yet', () => { + const dir = path.join(os.tmpdir(), `traces-dir-${Date.now()}`); + const filePath = path.join(dir, 'nested', 'traces.json'); + tmpFiles.push(dir); + + writeTraces(filePath, [makeResult()]); + + expect(fs.existsSync(filePath)).toBe(true); + }); + + it('overwrites an existing file rather than appending', () => { + const filePath = tmpPath(); + writeTraces(filePath, [makeResult({ testCase: { id: 'first', category: 'c', query: 'q' } })]); + writeTraces(filePath, [makeResult({ testCase: { id: 'second', category: 'c', query: 'q' } })]); + + const written = JSON.parse(fs.readFileSync(filePath, 'utf8')); + expect(written).toHaveLength(1); + expect(written[0].testId).toBe('second'); + }); +}); diff --git a/tests/unit/evals.workflow_judge.test.ts b/tests/unit/evals.workflow_judge.test.ts new file mode 100644 index 000000000..34a2eb4a2 --- /dev/null +++ b/tests/unit/evals.workflow_judge.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; + +import type { ConversationHistory } from '../../evals/workflows/types.js'; +import { formatConversationForJudge } from '../../evals/workflows/workflow_judge.js'; + +function createConversation(result: unknown): ConversationHistory { + return { + userPrompt: 'Do the task', + turns: [ + { + turnNumber: 1, + toolCalls: [{ name: 'call-actor', arguments: { actor: 'apify/code-runtime' } }], + toolResults: [{ toolName: 'call-actor', success: true, result }], + }, + { + turnNumber: 2, + toolCalls: [], + toolResults: [], + finalResponse: 'Done', + }, + ], + completed: true, + hitMaxTurns: false, + totalTurns: 2, + }; +} + +describe('formatConversationForJudge()', () => { + it('includes tool-result evidence', () => { + const transcript = formatConversationForJudge(createConversation({ exitCode: 0, stdout: 'five posts' })); + + expect(transcript).toContain('TOOL: [call-actor succeeded]'); + expect(transcript).toContain('"exitCode":0'); + }); + + it('bounds large tool results', () => { + const transcript = formatConversationForJudge(createConversation({ output: 'x'.repeat(10_000) })); + + expect(transcript).toContain('[truncated'); + expect(transcript.length).toBeLessThan(5_000); + }); +}); diff --git a/tests/unit/evals.workflow_output.test.ts b/tests/unit/evals.workflow_output.test.ts index 793708212..48b928513 100644 --- a/tests/unit/evals.workflow_output.test.ts +++ b/tests/unit/evals.workflow_output.test.ts @@ -2,10 +2,14 @@ import { describe, expect, it } from 'vitest'; import type { EvaluationResult, TestResultRecord } from '../../evals/workflows/output_formatter.js'; import { + aggregateRepeatedResults, formatBytes, + formatRepeatSummaryTable, formatResultsTable, formatTokens, formatWithDelta, + getCallActorTargets, + getToolCallTrace, sumResultBytes, } from '../../evals/workflows/output_formatter.js'; import type { ConversationHistory } from '../../evals/workflows/types.js'; @@ -60,6 +64,83 @@ describe('sumResultBytes()', () => { }); }); +describe('getToolCallTrace()', () => { + it("carries each tool call's startedAt and durationMs through to the trace entry", () => { + const conversation = makeConversation([ + { + turnNumber: 1, + toolCalls: [{ name: 'call-actor', arguments: { actor: 'apify/code-runtime' } }], + toolResults: [ + { + toolName: 'call-actor', + success: true, + result: {}, + startedAt: '2026-01-01T00:00:00.000Z', + durationMs: 4321, + }, + ], + }, + ]); + + const [entry] = getToolCallTrace(conversation); + expect(entry.startedAt).toBe('2026-01-01T00:00:00.000Z'); + expect(entry.durationMs).toBe(4321); + }); + + it('leaves startedAt/durationMs undefined when the tool result never recorded them', () => { + const conversation = makeConversation([ + { + turnNumber: 1, + toolCalls: [{ name: 'search-actors', arguments: {} }], + toolResults: [{ toolName: 'search-actors', success: true, result: {} }], + }, + ]); + + const [entry] = getToolCallTrace(conversation); + expect(entry.startedAt).toBeUndefined(); + expect(entry.durationMs).toBeUndefined(); + }); +}); + +describe('getCallActorTargets()', () => { + it('returns distinct Actor IDs targeted through call-actor', () => { + const conversation = makeConversation([ + { + turnNumber: 1, + toolCalls: [ + { name: 'call-actor', arguments: { actor: 'apify/code-runtime' } }, + { name: 'call-actor', arguments: { actor: 'apify/instagram-scraper' } }, + { name: 'call-actor', arguments: { actor: 'apify/code-runtime' } }, + { name: 'search-actors', arguments: { keywords: 'maps' } }, + ], + toolResults: [], + }, + ]); + + expect(getCallActorTargets(conversation)).toEqual(['apify/code-runtime', 'apify/instagram-scraper']); + }); + + it('strips an MCP tool-name suffix from the Actor target', () => { + const conversation = makeConversation([ + { + turnNumber: 1, + toolCalls: [{ name: 'call-actor', arguments: { actor: 'apify/actors-mcp-server:fetch-apify-docs' } }], + toolResults: [], + }, + ]); + + expect(getCallActorTargets(conversation)).toEqual(['apify/actors-mcp-server']); + }); + + it('returns an empty array when call-actor was never invoked', () => { + const conversation = makeConversation([ + { turnNumber: 1, toolCalls: [{ name: 'search-actors', arguments: {} }], toolResults: [] }, + ]); + + expect(getCallActorTargets(conversation)).toEqual([]); + }); +}); + describe('formatBytes()', () => { it('formats bytes under 1 KB as B', () => { expect(formatBytes(0)).toBe('0 B'); @@ -142,6 +223,11 @@ describe('formatResultsTable()', () => { promptTokens: 0, completionTokens: 0, totalTokens, + toolCalls: 0, + failedToolCalls: 0, + policyViolations: [], + finalResponse: '', + toolCallTrace: [], error: null, }; } @@ -170,3 +256,129 @@ describe('formatResultsTable()', () => { expect(table).toContain('(no baseline)'); }); }); + +describe('aggregateRepeatedResults()', () => { + function makeAttempt( + testId: string, + overrides: Partial & { durationMs?: number; tokens?: number; bytes?: number } = {}, + ): EvaluationResult { + const { durationMs = 100, tokens = 500, bytes = 1000, ...rest } = overrides; + return { + testCase: { id: testId, category: 'basic', query: 'q', reference: 'r' } as EvaluationResult['testCase'], + conversation: { + ...makeConversation([ + { + turnNumber: 1, + toolCalls: [], + toolResults: [{ toolName: 't', success: true, resultBytes: bytes }], + }, + ]), + totalTokens: tokens, + }, + judgeResult: { verdict: 'PASS', reason: 'ok', rawResponse: '' }, + durationMs, + ...rest, + }; + } + + it('counts pass/fail/timeout/error into separate buckets', () => { + const results = [ + makeAttempt('a', { judgeResult: { verdict: 'PASS', reason: 'ok', rawResponse: '' } }), + makeAttempt('a', { judgeResult: { verdict: 'PASS', reason: 'ok', rawResponse: '' } }), + makeAttempt('a', { judgeResult: { verdict: 'FAIL', reason: 'wrong', rawResponse: '' } }), + makeAttempt('a', { error: 'Test exceeded 300s timeout', timedOut: true }), + makeAttempt('a', { error: 'MCP error -32000: Connection closed' }), + ]; + + const [summary] = aggregateRepeatedResults(results); + expect(summary.attempts).toBe(5); + expect(summary.passed).toBe(2); + expect(summary.failed).toBe(1); + expect(summary.timedOut).toBe(1); + expect(summary.errored).toBe(1); + expect(summary.passRate).toBe(0.4); + expect(summary.completionRate).toBe(0.6); // passed + failed, not errored/timed-out + }); + + it('computes duration/token/byte stats only over completed attempts, excluding errored ones', () => { + const results = [ + makeAttempt('a', { durationMs: 100, tokens: 200, bytes: 1000 }), + makeAttempt('a', { durationMs: 300, tokens: 600, bytes: 3000 }), + // A timed-out attempt capped at 900s shouldn't drag the "typical duration" toward 900s. + makeAttempt('a', { durationMs: 900_000, error: 'Test exceeded 900s timeout', timedOut: true }), + ]; + + const [summary] = aggregateRepeatedResults(results); + expect(summary.medianDurationMs).toBe(200); + expect(summary.meanDurationMs).toBe(200); + expect(summary.medianTokens).toBe(400); + expect(summary.medianToolBytes).toBe(2000); + }); + + it('leaves duration/token/byte stats undefined when every attempt errored', () => { + const results = [makeAttempt('a', { error: 'boom' }), makeAttempt('a', { error: 'boom again' })]; + + const [summary] = aggregateRepeatedResults(results); + expect(summary.medianDurationMs).toBeUndefined(); + expect(summary.meanTokens).toBeUndefined(); + }); + + it('groups by test case id, keeping each test case a separate summary', () => { + const results = [makeAttempt('a'), makeAttempt('a'), makeAttempt('b')]; + + const summaries = aggregateRepeatedResults(results); + expect(summaries).toHaveLength(2); + expect(summaries.find((s) => s.testId === 'a')?.attempts).toBe(2); + expect(summaries.find((s) => s.testId === 'b')?.attempts).toBe(1); + }); +}); + +describe('formatRepeatSummaryTable()', () => { + it('renders pass/completion rates and per-attempt-type counts', () => { + const results = [ + { + testId: 'a', + category: 'basic', + attempts: 4, + passed: 2, + failed: 1, + timedOut: 1, + errored: 0, + passRate: 0.5, + completionRate: 0.75, + medianDurationMs: 1000, + meanDurationMs: 1100, + medianTokens: 500, + meanTokens: 550, + medianToolBytes: 2000, + meanToolBytes: 2100, + }, + ]; + + const table = formatRepeatSummaryTable(results); + expect(table).toContain('a (basic)'); + expect(table).toContain('Pass rate: 2/4 (50%)'); + expect(table).toContain('Completion rate: 3/4 (75%)'); + expect(table).toContain('Wrong answer: 1 | Timed out: 1 | Other errors: 0'); + expect(table).toContain('median 1000ms'); + }); + + it('flags a test case with no completed attempts instead of printing undefined', () => { + const results = [ + { + testId: 'a', + category: 'basic', + attempts: 2, + passed: 0, + failed: 0, + timedOut: 2, + errored: 0, + passRate: 0, + completionRate: 0, + }, + ]; + + const table = formatRepeatSummaryTable(results); + expect(table).toContain('No completed attempts to measure duration/tokens/bytes from.'); + }); +});