apify · MQ37 · Jul 16, 2026 · Jul 16, 2026 · Jul 16, 2026 · Jul 16, 2026
diff --git a/evals/shared/types.ts b/evals/shared/types.ts
@@ -38,11 +38,29 @@ export type ToolSelectionTestCase = {
  * Test case for workflow evaluation (multi-turn agent conversations)
  * Used in: evals/workflows/
  */
+export const WORKFLOW_EVAL_ARM = {
+    STANDARD: 'standard',
+    CODE_MODE: 'code-mode',
+} as const;
+export type WorkflowEvalArm = (typeof WORKFLOW_EVAL_ARM)[keyof typeof WORKFLOW_EVAL_ARM];
+
 export type WorkflowTestCase = {
     /** Maximum number of turns allowed (optional, defaults to config value) */
     maxTurns?: number;
     /** Tools to enable for this test (optional, e.g., ["actors", "docs", "apify/rag-web-browser"]) */
     tools?: string[];
+    /** Enabled tools to hide from the agent and reject if called. */
+    disallowedTools?: string[];
+    /** Actor IDs the generic call-actor tool may run. */
+    allowedCallActorTargets?: string[];
+    /** Actor IDs the generic call-actor tool may not run. */
+    disallowedCallActorTargets?: string[];
+    /** Evaluation-only instructions appended to the agent system prompt. */
+    agentInstructions?: string;
+    /** Groups variants of the same task. */
+    pairId?: string;
+    /** Strategy variant for paired evaluations. */
+    arm?: WorkflowEvalArm;
 } & BaseTestCase;
 
 /**

diff --git a/evals/workflows/README.md b/evals/workflows/README.md
@@ -46,6 +46,12 @@ pnpm run evals:workflow -- --concurrency 8
 
 # Save results to JSON file
 pnpm run evals:workflow -- --output
+
+# Run paired Code Mode evaluations
+pnpm run evals:workflow -- \
+  --test-cases-path evals/workflows/code_mode_test_cases.json \
+  --results-path evals/workflows/code_mode_results.json \
+  --tool-timeout 600 --concurrency 1 --output
 ```
 
 **Exit codes:**
@@ -139,21 +145,14 @@ while (turnNumber < maxTurns) {
 
 **Location:** `run-workflow-evals.ts`
 
-### 4. Judge Sees Tool Calls, Not Results
+### 4. Judge Sees Bounded Tool Results
 
-**Decision:** Judge sees tool calls with arguments and agent responses, but NOT raw tool results.
+**Decision:** Judge sees tool calls, final responses, and up to 4,000 characters from each tool result.
 
 **Why:**
-- Evaluates agent behavior (tool selection, arguments)
-- Tool results are often very long and noisy
-- Agent should summarize results, judge evaluates the summary
-
-**Judge input format:**
-```
-USER: Find actors for Google Maps
-AGENT: [Called tool: search-actors with args: {"keywords":"google maps","limit":5}]
-AGENT: I found 5 actors: 1. Google Maps Scraper... 2. ...
-```
+- Tool evidence lets the judge detect unsupported final answers
+- Per-result bounds prevent large datasets from dominating judge context
+- Errors and Code Runtime output remain visible
 
 **Location:** `workflow-judge.ts`
 
@@ -246,9 +245,13 @@ export OPENROUTER_API_KEY="your_openrouter_key" # Get from https://openrouter.ai
 | `--agent-model <model>` | | Override agent model | `anthropic/claude-haiku-4.5` |
 | `--judge-model <model>` | | Override judge model | `deepseek/deepseek-v4-flash` |
 | `--tool-timeout <seconds>` | | Tool call timeout | `60` |
+| `--test-timeout <seconds>` | | Wall-clock timeout for one whole test case (agent + judge) | `300` |
+| `--repeat <n>` | | Run each test case N times, print an aggregated pass/completion/error-rate summary | `1` |
 | `--concurrency <number>` | `-c` | Number of tests to run in parallel | `4` |
 | `--output` | `-o` | Save results to JSON file | `false` |
-| `--baseline <path>` | | Results JSON to compare against (prints byte/token deltas) | `results.json` |
+| `--results-path <path>` | | Results JSON file | `results.json` |
+| `--baseline <path>` | | Results JSON to compare against (prints byte/token deltas) | results path |
+| `--traces <path>` | | Write full per-turn traces (LLM output, tool calls, full args/results, untruncated) to this JSON file | not written |
 | `--help` | | Show help message | - |
 
 ### Line Range Filtering
@@ -340,22 +343,81 @@ The `--tool-timeout` option sets the maximum time (in seconds) to wait for a sin
 pnpm run evals:workflow -- --tool-timeout 300
 ```
 
+### Test Timeout
+
+`--tool-timeout` bounds one tool call; `--test-timeout` bounds the whole test
+case — the agent's full multi-turn conversation plus judging — so a test
+stuck retrying (e.g. a bad tool-call loop) can't block the run indefinitely.
+Default: `300` seconds.
+
+When a test times out, it's recorded as `FAIL` with reason `"Test exceeded
+its wall-clock timeout"` and the run moves on to the next test. The
+abandoned work isn't cancelled mid-flight (no cancellation token threads
+through the LLM/MCP clients) — the harness just stops waiting for it and
+cleans up that test's MCP client.
+
+```bash
+pnpm run evals:workflow -- --test-timeout 600   # for slower models / long chains
+```
+
+### Repeated Runs (`--repeat`)
+
+A single run of a test case can't distinguish "this approach is worse" from
+ordinary run-to-run noise — flaky upstream Actors, model variance, and (at
+higher `--concurrency`) rate-limit contention all inject noise unrelated to
+the thing you're actually trying to measure. `--repeat <n>` runs each
+(filtered) test case N times and prints an aggregated summary in addition to
+the normal per-attempt table.
+
+**Job scheduling:** each `(test case, attempt)` pair is its own job through
+the same `--concurrency` limiter as any other run — repeats of one test case
+interleave with everything else exactly like distinct test cases do. For a
+clean, low-noise comparison, run with `--concurrency 1` yourself; `--repeat`
+doesn't invent a separate sequential mode for that.
+
+**What gets averaged, and what doesn't:**
+- `--traces` and `--results-path` (`--output`) record **every attempt
+  individually**, tagged with `attemptIndex`/`totalAttempts` — never
+  averaged. Use these for the exact numbers behind any one run.
+- The printed **Repeat Summary** aggregates per test case:
+  - `passRate` (strict PASS / N) and `completionRate` ((PASS + judge-FAIL) /
+    N) are reported separately — a wrong-but-complete answer and a timeout
+    are different failure modes needing different fixes, so they're not
+    collapsed into one number.
+  - Duration/token/tool-byte stats (median + mean) are computed **only over
+    completed attempts** (PASS or judge-FAIL). An errored or timed-out
+    attempt is excluded from these — including it would bias the "typical
+    duration" toward whatever the timeout cap happened to be, which isn't a
+    real measurement of anything.
+  - `timedOut` (exceeded `--test-timeout` specifically) is counted
+    separately from other errors.
+
+**Exit code:** `--repeat` exists to characterize flakiness across N
+attempts, not to gate CI on one unlucky run — the process exits `0`
+regardless of individual attempt outcomes when `--repeat > 1`. The default
+(`--repeat 1`, unused) keeps the normal all-attempts-must-pass exit code.
+
+```bash
+pnpm run evals:workflow -- --test-cases-path evals/workflows/code_mode_test_cases.json \
+    --repeat 5 --concurrency 1 --test-timeout 900 --traces /tmp/code_mode_traces.json
+```
+
 ### Saving Results to File
 
-The `--output` (or `-o`) option saves test results to `evals/workflows/results.json` for tracking over time.
+The `--output` (or `-o`) option saves test results to `evals/workflows/results.json`, or `--results-path`, for tracking over time.
 
 **How it works:**
-- Results are stored per combination of: `agentModel:judgeModel:testId`
-- Running the same test with the same models **overwrites** the previous result
-- Running with different model combinations **adds** new entries
-- Results are **versioned in git** for historical tracking
+- Every run appends an attempt; previous attempts remain available
+- Records include agent and judge token usage separately
+- Records include tool calls, policy violations, final response, and complete conversation trace
+- Results can be versioned in git or written to an ignored local path
 
 **Data structure:**
 ```json
 {
-  "version": "1.0",
-  "results": {
-    "anthropic/claude-haiku-4.5:x-ai/grok-4.1-fast:search-google-maps": {
+  "version": "2.0",
+  "attempts": [
+    {
       "timestamp": "2026-01-07T10:45:23.123Z",
       "agentModel": "anthropic/claude-haiku-4.5",
       "judgeModel": "x-ai/grok-4.1-fast",
@@ -368,23 +430,34 @@ The `--output` (or `-o`) option saves test results to `evals/workflows/results.j
       "promptTokens": 6231,
       "completionTokens": 412,
       "totalTokens": 6643,
+      "judgeUsage": { "promptTokens": 1800, "completionTokens": 40, "totalTokens": 1840 },
+      "toolCalls": 3,
+      "failedToolCalls": 0,
+      "policyViolations": [],
+      "finalResponse": "...",
+      "toolCallTrace": [],
       "error": null
     }
-  }
+  ]
 }
 ```
 
 **Each result contains:**
 - `timestamp` - ISO timestamp when test was run
 - `agentModel` - LLM model used for the agent
 - `judgeModel` - LLM model used for judging
-- `testId` - Test case identifier
+- `testId`, `pairId`, `arm` - Test and experiment identifiers
 - `verdict` - `PASS` or `FAIL`
 - `reason` - Judge reasoning or error message
 - `durationMs` - Test duration in milliseconds
 - `turns` - Number of conversation turns
 - `resultBytes` - Total UTF-8 bytes of tool results returned to the agent across the conversation (measured at the point each result is fed to the LLM, so it reflects what the agent actually receives). Compare across branches to quantify byte savings.
-- `promptTokens` / `completionTokens` / `totalTokens` - Tokens billed across all agent LLM calls (summed over turns; judge calls excluded). Tokens — not bytes — are what fill the context window, so this is the primary cost signal. Bytes are a deterministic, tokenizer-free proxy.
+- `promptTokens` / `completionTokens` / `totalTokens` - Tokens billed across all agent LLM calls; judge calls excluded
+- `cachedPromptTokens` / `reasoningTokens` - Provider-reported agent usage details
+- `judgeUsage` - Judge tokens, kept separate from agent usage
+- `toolCalls`, `failedToolCalls`, `policyViolations` - Execution counters and policy failures
+- `finalResponse`, `toolCallTrace` - Agent answer and tool-call trace (name, arguments, success, byte size, `startedAt`/`durationMs`), including generated Code Mode scripts; result bodies are excluded to keep the file compact
+- `usedCodeRuntime` - Whether the agent called `apify/code-runtime` at least once. Informational only — does not gate pass/fail, since a code-mode-arm agent may legitimately call other Actors directly for discovery.
 - `error` - Error message if execution failed, `null` otherwise
 
 **Examples:**
@@ -404,10 +477,13 @@ pnpm run evals:workflow -- --agent-model openai/gpt-4o --output
 # Compare different judge models
 pnpm run evals:workflow -- --judge-model x-ai/grok-4.1-fast --output
 pnpm run evals:workflow -- --judge-model openai/gpt-4o --output
+
+# Save a separate benchmark
+pnpm run evals:workflow -- --output --results-path evals/workflows/code_mode_results.json
 ```
 
 **Partial runs:**
-When using filters (`--category`, `--id`), only the filtered tests are updated in the results file. Other entries remain unchanged.
+When using filters (`--category`, `--id`), attempts are appended only for matching tests. Existing attempts remain unchanged.
 
 **Version control:**
 The `results.json` file is tracked in git, allowing you to:
@@ -419,7 +495,7 @@ The `results.json` file is tracked in git, allowing you to:
 
 Every run automatically compares against a baseline and prints per-test and aggregate **deltas** for tool bytes and tokens — no manual file diffing. This is how you answer "did this change grow the response size?".
 
-- **Default baseline** is the committed `evals/workflows/results.json`. Each test is matched by its `agentModel:judgeModel:testId` key.
+- **Default baseline** is the selected results path. Each test is matched by agent model and test ID.
 - **Custom baseline:** `--baseline <path>` compares against any saved results file.
 - Deltas read as `▼ -2.1 KB / -10.2%` (reduction) or `▲ +900 / +3.4%` (increase). Lower is better for both metrics.
 - This is **reporting only** — a regression never fails the run. Task success (all tests PASS) is the hard gate.
@@ -433,6 +509,35 @@ cp evals/workflows/results.json /tmp/baseline.json
 pnpm run evals:workflow -- --baseline /tmp/baseline.json   # prints byte/token deltas vs the baseline
 ```
 
+### Full Traces (`--traces`)
+
+`results.json`'s `toolCallTrace` is deliberately compact — it excludes tool
+result bodies to stay small over time. To inspect exactly what happened in
+one test case (every LLM final response, every tool call's full arguments,
+every tool result's full untruncated content), pass `--traces <path>`. It
+writes one JSON array, one entry per test case, and is meant for manual
+review — not for committing or diffing.
+
+```bash
+pnpm run evals:workflow --test-cases-path evals/workflows/code_mode_test_cases.json --traces /tmp/code_mode_traces.json
+```
+
+Each entry: `testId`, `category`, `arm`, `pairId`, `query`, `durationMs`,
+`error`, `verdict`, `judgeReason`, and `conversation` (the full
+`ConversationHistory` — every turn's `toolCalls`, `toolResults` (including
+`result`, the raw tool output), `usage`, and `finalResponse`).
+
+Every turn also carries `llmStartedAt` (ISO timestamp) / `llmDurationMs` for
+its LLM call, and every tool result carries `startedAt` / `durationMs` for
+that specific call — enough to reconstruct a full timeline of where time
+went (LLM thinking vs. a specific nested Actor call). These same two fields
+are also included in `results.json`'s compact `toolCallTrace`.
+
+**Traces are written even when a test times out (`--test-timeout`) or
+errors** — the run's error/timeout path preserves whatever turns completed
+before the cutoff, so a stuck test's trace still shows exactly which tool
+call it was in and how long that call had already been running.
+
 > **Bootstrap note:** records written before these metrics existed have no `resultBytes`/`*Tokens` fields, so the first run after this change shows `(no baseline)` for them and writes fresh values with `--output`. Subsequent runs show real deltas.
 
 ### Test Case Format
@@ -447,7 +552,12 @@ File: `test-cases.json`
     "prompt": "User prompt for agent",
     "requirements": "What agent must do to pass",
     "maxTurns": 10,
-    "tools": ["actors", "docs"]
+    "tools": ["actors", "docs"],
+    "disallowedTools": ["add-actor"],
+    "agentInstructions": "Use regular MCP tools.",
+    "disallowedCallActorTargets": ["apify/code-runtime"],
+    "pairId": "test-pair",
+    "arm": "standard"
   }
 ]
 ```
@@ -460,7 +570,11 @@ File: `test-cases.json`
 
 **Optional:**
 - `maxTurns` - Override default (10)
-- `tools` - List of tools to enable for this test (e.g., `["actors", "docs", "apify/rag-web-browser"]`). If omitted, all default tools are enabled. Passed to MCP server as `--tools` argument.
+- `tools` - Server-side tool allowlist passed as `--tools`
+- `disallowedTools` - Hide tools from the agent and reject direct calls
+- `allowedCallActorTargets` / `disallowedCallActorTargets` - Restrict Actor IDs used through `call-actor` (optional; the shipped Code Mode cases only set `disallowedCallActorTargets: ["apify/code-runtime"]` on the standard arm — the code-mode arm leaves Actor targets unrestricted so the agent can call any Actor directly for discovery and still use `apify/code-runtime` for the actual processing)
+- `agentInstructions` - Evaluation-only system instructions
+- `pairId` / `arm` - Group standard and Code Mode variants of the same prompt
 
 ## Performance