From 4e4c4887c3d5d71bac50945ea25f60cb0b61b47e Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Fri, 31 Jul 2026 17:04:25 -0700 Subject: [PATCH] Fix auto model --- .../evaluator/evaluator-handler.test.ts | 150 ++++++++++++++++-- .../handlers/evaluator/evaluator-handler.ts | 73 +++++++-- .../handlers/router/router-handler.test.ts | 129 ++++++++++++++- .../handlers/router/router-handler.ts | 113 +++++++++++-- apps/sim/lib/model-router/resolve.test.ts | 49 +++++- apps/sim/lib/model-router/resolve.ts | 29 +++- 6 files changed, 481 insertions(+), 62 deletions(-) diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts index b992730f0cc..0cefcf821ff 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts @@ -3,6 +3,10 @@ import '@sim/testing/mocks/executor' import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest' +const { mockResolveAutoModel } = vi.hoisted(() => ({ + mockResolveAutoModel: vi.fn(), +})) + vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) vi.mock('@/lib/credentials/access', () => ({ @@ -20,6 +24,13 @@ vi.mock('@/lib/credentials/access', () => ({ }), })) +vi.mock('@/lib/model-router/resolve', () => ({ + addAutoRoutingCost: (cost: Record, routingCost: number) => + routingCost > 0 ? { ...cost, routing: routingCost, total: cost.total + routingCost } : cost, + resolveAutoModel: mockResolveAutoModel, + SIM_AUTO_SYSTEM_PREAMBLE: 'Sim auto system preamble', +})) + import { BlockType } from '@/executor/constants' import { EvaluatorBlockHandler } from '@/executor/handlers/evaluator/evaluator-handler' import type { ExecutionContext } from '@/executor/types' @@ -82,6 +93,12 @@ describe('EvaluatorBlockHandler', () => { refreshed: false, }) mockGetProviderFromModel.mockReturnValue('openai') + mockResolveAutoModel.mockResolvedValue({ + model: 'fireworks/glm-5.2', + tier: '2', + decidedBy: 'llm', + billableRoutingCost: 0.002, + }) // Set up fetch mock to return a successful response mockFetch.mockImplementation(() => { @@ -101,7 +118,10 @@ describe('EvaluatorBlockHandler', () => { it('should handle evaluator blocks', () => { expect(handler.canHandle(mockBlock)).toBe(true) - const nonEvalBlock: SerializedBlock = { ...mockBlock, metadata: { id: 'other' } } + const nonEvalBlock: SerializedBlock = { + ...mockBlock, + metadata: { id: 'other' }, + } expect(handler.canHandle(nonEvalBlock)).toBe(false) }) @@ -109,8 +129,16 @@ describe('EvaluatorBlockHandler', () => { const inputs = { content: 'This is the content to evaluate.', metrics: [ - { name: 'score1', description: 'First score', range: { min: 0, max: 10 } }, - { name: 'score2', description: 'Second score', range: { min: 0, max: 10 } }, + { + name: 'score1', + description: 'First score', + range: { min: 0, max: 10 }, + }, + { + name: 'score2', + description: 'Second score', + range: { min: 0, max: 10 }, + }, ], model: 'gpt-4o', apiKey: 'test-api-key', @@ -163,6 +191,64 @@ describe('EvaluatorBlockHandler', () => { }) }) + it('resolves sim-auto before executing evaluator and preserves its public identity', async () => { + const inputs = { + content: 'A clear and accurate answer.', + metrics: [ + { + name: 'quality', + description: 'Overall answer quality', + range: { min: 1, max: 5 }, + }, + ], + model: 'sim-auto', + } + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => + Promise.resolve({ + content: JSON.stringify({ quality: 5 }), + model: 'fireworks/glm-5.2', + tokens: { input: 80, output: 10, total: 90 }, + cost: { input: 0.001, output: 0.0005, total: 0.0015 }, + }), + }) + + const result = await handler.execute(mockContext, mockBlock, inputs) + + expect(mockResolveAutoModel).toHaveBeenCalledWith({ + ctx: mockContext, + blockId: mockBlock.id, + signals: expect.objectContaining({ + lastMessage: inputs.content, + messageCount: 1, + toolNames: [], + mediaKind: 'none', + hasResponseFormat: true, + }), + fallbackModel: 'claude-sonnet-5', + }) + expect(mockGetProviderFromModel).toHaveBeenCalledWith('fireworks/glm-5.2') + + const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body) + expect(requestBody).toMatchObject({ + provider: 'openai', + model: 'fireworks/glm-5.2', + systemPrompt: expect.stringMatching(/^Sim auto system preamble\n\n/), + }) + expect(result).toMatchObject({ + model: 'sim-auto', + quality: 5, + cost: { + input: 0.001, + output: 0.0005, + routing: 0.002, + total: 0.0035, + }, + }) + }) + it('bills the cost the provider proxy decided rather than recomputing it', async () => { // The proxy already resolved key provenance and the margin; recomputing // here would re-charge a BYOK caller the proxy correctly zeroed. @@ -195,7 +281,13 @@ describe('EvaluatorBlockHandler', () => { const contentObj = { text: 'Evaluate this JSON.', value: 42 } const inputs = { content: JSON.stringify(contentObj), - metrics: [{ name: 'clarity', description: 'Clarity score', range: { min: 1, max: 5 } }], + metrics: [ + { + name: 'clarity', + description: 'Clarity score', + range: { min: 1, max: 5 }, + }, + ], apiKey: 'test-api-key', } @@ -227,7 +319,11 @@ describe('EvaluatorBlockHandler', () => { const inputs = { content: contentObj, metrics: [ - { name: 'completeness', description: 'Data completeness', range: { min: 0, max: 1 } }, + { + name: 'completeness', + description: 'Data completeness', + range: { min: 0, max: 1 }, + }, ], apiKey: 'test-api-key', } @@ -258,7 +354,13 @@ describe('EvaluatorBlockHandler', () => { it('should parse valid JSON response correctly', async () => { const inputs = { content: 'Test content', - metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }], + metrics: [ + { + name: 'quality', + description: 'Quality score', + range: { min: 1, max: 10 }, + }, + ], apiKey: 'test-api-key', } @@ -339,7 +441,13 @@ describe('EvaluatorBlockHandler', () => { it('should extract metric scores ignoring case', async () => { const inputs = { content: 'Test', - metrics: [{ name: 'CamelCaseScore', description: 'Desc', range: { min: 0, max: 10 } }], + metrics: [ + { + name: 'CamelCaseScore', + description: 'Desc', + range: { min: 0, max: 10 }, + }, + ], apiKey: 'test-api-key', } @@ -366,8 +474,16 @@ describe('EvaluatorBlockHandler', () => { const inputs = { content: 'Test', metrics: [ - { name: 'presentScore', description: 'Desc1', range: { min: 0, max: 5 } }, - { name: 'missingScore', description: 'Desc2', range: { min: 0, max: 5 } }, + { + name: 'presentScore', + description: 'Desc1', + range: { min: 0, max: 5 }, + }, + { + name: 'missingScore', + description: 'Desc2', + range: { min: 0, max: 5 }, + }, ], apiKey: 'test-api-key', } @@ -410,7 +526,13 @@ describe('EvaluatorBlockHandler', () => { it('should handle Azure OpenAI models with endpoint and API version', async () => { const inputs = { content: 'Test content to evaluate', - metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }], + metrics: [ + { + name: 'quality', + description: 'Quality score', + range: { min: 1, max: 10 }, + }, + ], model: 'gpt-4o', apiKey: 'test-azure-key', azureEndpoint: 'https://test.openai.azure.com', @@ -450,7 +572,13 @@ describe('EvaluatorBlockHandler', () => { it('should handle Vertex AI models with OAuth credential', async () => { const inputs = { content: 'Test content to evaluate', - metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }], + metrics: [ + { + name: 'quality', + description: 'Quality score', + range: { min: 1, max: 10 }, + }, + ], model: 'gemini-2.0-flash-exp', vertexCredential: 'test-vertex-credential-id', vertexProject: 'test-gcp-project', diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts index 1f96fff1d6f..a3a4475aa39 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts @@ -1,4 +1,10 @@ import { createLogger } from '@sim/logger' +import { + type AutoRoutingResult, + addAutoRoutingCost, + resolveAutoModel, + SIM_AUTO_SYSTEM_PREAMBLE, +} from '@/lib/model-router/resolve' import type { BlockOutput } from '@/blocks/types' import { validateModelProvider } from '@/ee/access-control/utils/permission-check' import { BlockType, DEFAULTS, EVALUATOR } from '@/executor/constants' @@ -7,6 +13,7 @@ import { buildAPIUrl, buildAuthHeaders, extractAPIErrorMessage } from '@/executo import { isJSONString, parseJSON, stringifyJSON } from '@/executor/utils/json' import { resolveVertexCredential } from '@/executor/utils/vertex-credential' import { resolveProxiedModelCost } from '@/providers/cost-policy' +import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models' import { getProviderFromModel } from '@/providers/utils' import type { SerializedBlock } from '@/serializer/types' @@ -36,19 +43,6 @@ export class EvaluatorBlockHandler implements BlockHandler { bedrockRegion: inputs.bedrockRegion, } - await validateModelProvider(ctx.userId, ctx.workspaceId, evaluatorConfig.model, ctx) - - const providerId = getProviderFromModel(evaluatorConfig.model) - - let finalApiKey: string | undefined = evaluatorConfig.apiKey - if (providerId === 'vertex' && evaluatorConfig.vertexCredential) { - finalApiKey = await resolveVertexCredential( - evaluatorConfig.vertexCredential, - ctx.userId, - 'vertex-evaluator' - ) - } - const processedContent = this.processContent(inputs.content) let systemPromptObj: { systemPrompt: string; responseFormat: any } = { @@ -105,12 +99,55 @@ export class EvaluatorBlockHandler implements BlockHandler { 'Evaluate the content and provide scores for each metric as JSON.' } + let model = evaluatorConfig.model + let autoRouting: AutoRoutingResult | null = null + if (isAutoModel(model)) { + autoRouting = await resolveAutoModel({ + ctx, + blockId: block.id, + signals: { + systemPrompt: systemPromptObj.systemPrompt, + lastMessage: processedContent, + messageCount: 1, + toolNames: [], + mediaKind: 'none', + hasResponseFormat: true, + approxInputTokens: Math.ceil( + (systemPromptObj.systemPrompt.length + processedContent.length) / 4 + ), + }, + fallbackModel: EVALUATOR.DEFAULT_MODEL, + }) + model = autoRouting.model + systemPromptObj.systemPrompt = [SIM_AUTO_SYSTEM_PREAMBLE, systemPromptObj.systemPrompt] + .filter(Boolean) + .join('\n\n') + logger.info('Resolved sim-auto model for evaluator', { + blockId: block.id, + model, + tier: autoRouting.tier, + decidedBy: autoRouting.decidedBy, + }) + } + + await validateModelProvider(ctx.userId, ctx.workspaceId, model, ctx) + const providerId = getProviderFromModel(model) + + let finalApiKey: string | undefined = evaluatorConfig.apiKey + if (providerId === 'vertex' && evaluatorConfig.vertexCredential) { + finalApiKey = await resolveVertexCredential( + evaluatorConfig.vertexCredential, + ctx.userId, + 'vertex-evaluator' + ) + } + try { const url = buildAPIUrl('/api/providers', ctx.userId ? { userId: ctx.userId } : {}) const providerRequest: Record = { provider: providerId, - model: evaluatorConfig.model, + model, systemPrompt: systemPromptObj.systemPrompt, responseFormat: systemPromptObj.responseFormat, context: stringifyJSON([ @@ -155,11 +192,14 @@ export class EvaluatorBlockHandler implements BlockHandler { const outputTokens = result.tokens?.output || result.tokens?.completion || DEFAULTS.TOKENS.COMPLETION - const cost = resolveProxiedModelCost(result.cost) + const cost = addAutoRoutingCost( + resolveProxiedModelCost(result.cost), + autoRouting?.billableRoutingCost ?? 0 + ) return { content: inputs.content, - model: result.model, + model: autoRouting ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: inputTokens, output: outputTokens, @@ -169,6 +209,7 @@ export class EvaluatorBlockHandler implements BlockHandler { input: cost.input, output: cost.output, total: cost.total, + ...(cost.routing === undefined ? {} : { routing: cost.routing }), }, ...metricScores, } diff --git a/apps/sim/executor/handlers/router/router-handler.test.ts b/apps/sim/executor/handlers/router/router-handler.test.ts index 3c57fe2de8f..2730366e893 100644 --- a/apps/sim/executor/handlers/router/router-handler.test.ts +++ b/apps/sim/executor/handlers/router/router-handler.test.ts @@ -3,6 +3,10 @@ import '@sim/testing/mocks/executor' import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest' +const { mockResolveAutoModel } = vi.hoisted(() => ({ + mockResolveAutoModel: vi.fn(), +})) + vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) vi.mock('@/lib/credentials/access', () => ({ @@ -20,6 +24,13 @@ vi.mock('@/lib/credentials/access', () => ({ }), })) +vi.mock('@/lib/model-router/resolve', () => ({ + addAutoRoutingCost: (cost: Record, routingCost: number) => + routingCost > 0 ? { ...cost, routing: routingCost, total: cost.total + routingCost } : cost, + resolveAutoModel: mockResolveAutoModel, + SIM_AUTO_SYSTEM_PREAMBLE: 'Sim auto system preamble', +})) + import { generateRouterPrompt, generateRouterV2Prompt } from '@/blocks/blocks/router' import { BlockType } from '@/executor/constants' import { RouterBlockHandler } from '@/executor/handlers/router/router-handler' @@ -71,8 +82,16 @@ describe('RouterBlockHandler', () => { mockWorkflow = { blocks: [mockBlock, mockTargetBlock1, mockTargetBlock2], connections: [ - { source: mockBlock.id, target: mockTargetBlock1.id, sourceHandle: 'condition-then1' }, - { source: mockBlock.id, target: mockTargetBlock2.id, sourceHandle: 'condition-else1' }, + { + source: mockBlock.id, + target: mockTargetBlock1.id, + sourceHandle: 'condition-then1', + }, + { + source: mockBlock.id, + target: mockTargetBlock2.id, + sourceHandle: 'condition-else1', + }, ], } @@ -108,6 +127,12 @@ describe('RouterBlockHandler', () => { }) mockGetProviderFromModel.mockReturnValue('openai') mockGenerateRouterPrompt.mockReturnValue('Generated System Prompt') + mockResolveAutoModel.mockResolvedValue({ + model: 'fireworks/glm-5.2', + tier: '2', + decidedBy: 'llm', + billableRoutingCost: 0.002, + }) mockFetch.mockImplementation(() => { return Promise.resolve({ @@ -126,7 +151,10 @@ describe('RouterBlockHandler', () => { it('should handle router blocks', () => { expect(handler.canHandle(mockBlock)).toBe(true) - const nonRouterBlock: SerializedBlock = { ...mockBlock, metadata: { id: 'other' } } + const nonRouterBlock: SerializedBlock = { + ...mockBlock, + metadata: { id: 'other' }, + } expect(handler.canHandle(nonRouterBlock)).toBe(false) }) @@ -438,6 +466,12 @@ describe('RouterBlockHandler V2', () => { }) mockGetProviderFromModel.mockReturnValue('openai') mockGenerateRouterV2Prompt.mockReturnValue('Generated V2 System Prompt') + mockResolveAutoModel.mockResolvedValue({ + model: 'fireworks/glm-5.2', + tier: '2', + decidedBy: 'llm', + billableRoutingCost: 0.002, + }) }) it('should handle router_v2 blocks', () => { @@ -450,8 +484,16 @@ describe('RouterBlockHandler V2', () => { model: 'gpt-4o', apiKey: 'test-api-key', routes: JSON.stringify([ - { id: 'route-support', title: 'Support', value: 'Customer support inquiries' }, - { id: 'route-sales', title: 'Sales', value: 'Sales and pricing questions' }, + { + id: 'route-support', + title: 'Support', + value: 'Customer support inquiries', + }, + { + id: 'route-sales', + title: 'Sales', + value: 'Sales and pricing questions', + }, ]), } @@ -485,6 +527,68 @@ describe('RouterBlockHandler V2', () => { }) }) + it('resolves sim-auto before executing router V2 and preserves its public identity', async () => { + const inputs = { + context: 'How do I get Tableau on my work laptop?', + model: 'sim-auto', + routes: [ + { id: 'route-support', title: 'Support', value: 'Something is broken' }, + { + id: 'route-sales', + title: 'Request', + value: 'User wants something new', + }, + ], + } + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => + Promise.resolve({ + content: JSON.stringify({ + route: 'route-sales', + reasoning: 'This is a new request.', + }), + model: 'fireworks/glm-5.2', + tokens: { input: 100, output: 20, total: 120 }, + cost: { input: 0.001, output: 0.0005, total: 0.0015 }, + }), + }) + + const result = await handler.execute(mockContext, mockRouterV2Block, inputs) + + expect(mockResolveAutoModel).toHaveBeenCalledWith({ + ctx: mockContext, + blockId: mockRouterV2Block.id, + signals: expect.objectContaining({ + lastMessage: inputs.context, + messageCount: 1, + toolNames: [], + mediaKind: 'none', + hasResponseFormat: true, + }), + fallbackModel: 'claude-sonnet-5', + }) + expect(mockGetProviderFromModel).toHaveBeenCalledWith('fireworks/glm-5.2') + + const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body) + expect(requestBody).toMatchObject({ + provider: 'openai', + model: 'fireworks/glm-5.2', + systemPrompt: 'Sim auto system preamble\n\nGenerated V2 System Prompt', + }) + expect(result).toMatchObject({ + model: 'sim-auto', + selectedRoute: 'route-sales', + cost: { + input: 0.001, + output: 0.0005, + routing: 0.002, + total: 0.0035, + }, + }) + }) + it('should include responseFormat in provider request', async () => { const inputs = { context: 'Test context', @@ -498,7 +602,10 @@ describe('RouterBlockHandler V2', () => { ok: true, json: () => Promise.resolve({ - content: JSON.stringify({ route: 'route-1', reasoning: 'Test reasoning' }), + content: JSON.stringify({ + route: 'route-1', + reasoning: 'Test reasoning', + }), model: 'gpt-4o', tokens: { input: 100, output: 20, total: 120 }, }), @@ -572,7 +679,10 @@ describe('RouterBlockHandler V2', () => { ok: true, json: () => Promise.resolve({ - content: JSON.stringify({ route: 'invalid-route', reasoning: 'Some reasoning' }), + content: JSON.stringify({ + route: 'invalid-route', + reasoning: 'Some reasoning', + }), model: 'gpt-4o', tokens: { input: 100, output: 20, total: 120 }, }), @@ -597,7 +707,10 @@ describe('RouterBlockHandler V2', () => { ok: true, json: () => Promise.resolve({ - content: JSON.stringify({ route: 'route-1', reasoning: 'Matched route 1' }), + content: JSON.stringify({ + route: 'route-1', + reasoning: 'Matched route 1', + }), model: 'gpt-4o', tokens: { input: 100, output: 20, total: 120 }, }), diff --git a/apps/sim/executor/handlers/router/router-handler.ts b/apps/sim/executor/handlers/router/router-handler.ts index 55184c40611..0a4961d7095 100644 --- a/apps/sim/executor/handlers/router/router-handler.ts +++ b/apps/sim/executor/handlers/router/router-handler.ts @@ -1,5 +1,11 @@ import { createLogger } from '@sim/logger' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' +import { + type AutoRoutingResult, + addAutoRoutingCost, + resolveAutoModel, + SIM_AUTO_SYSTEM_PREAMBLE, +} from '@/lib/model-router/resolve' import { generateRouterPrompt, generateRouterV2Prompt } from '@/blocks/blocks/router' import type { BlockOutput } from '@/blocks/types' import { validateModelProvider } from '@/ee/access-control/utils/permission-check' @@ -14,6 +20,7 @@ import type { BlockHandler, ExecutionContext } from '@/executor/types' import { buildAuthHeaders } from '@/executor/utils/http' import { resolveVertexCredential } from '@/executor/utils/vertex-credential' import { resolveProxiedModelCost } from '@/providers/cost-policy' +import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models' import { getProviderFromModel } from '@/providers/utils' import type { SerializedBlock } from '@/serializer/types' @@ -72,16 +79,23 @@ export class RouterBlockHandler implements BlockHandler { bedrockRegion: inputs.bedrockRegion, } - await validateModelProvider(ctx.userId, ctx.workspaceId, routerConfig.model, ctx) - - const providerId = getProviderFromModel(routerConfig.model) - try { const url = new URL('/api/providers', getInternalApiBaseUrl()) if (ctx.userId) url.searchParams.set('userId', ctx.userId) const messages = [{ role: 'user', content: routerConfig.prompt }] const systemPrompt = generateRouterPrompt(routerConfig.prompt, targetBlocks) + const resolved = await this.resolveModel( + ctx, + block.id, + routerConfig.model, + systemPrompt, + routerConfig.prompt, + false + ) + + await validateModelProvider(ctx.userId, ctx.workspaceId, resolved.model, ctx) + const providerId = getProviderFromModel(resolved.model) let finalApiKey: string | undefined = routerConfig.apiKey if (providerId === 'vertex' && routerConfig.vertexCredential) { @@ -94,8 +108,8 @@ export class RouterBlockHandler implements BlockHandler { const providerRequest: Record = { provider: providerId, - model: routerConfig.model, - systemPrompt: systemPrompt, + model: resolved.model, + systemPrompt: resolved.systemPrompt, context: JSON.stringify(messages), temperature: ROUTER.INFERENCE_TEMPERATURE, apiKey: finalApiKey, @@ -146,11 +160,14 @@ export class RouterBlockHandler implements BlockHandler { total: DEFAULTS.TOKENS.TOTAL, } - const cost = resolveProxiedModelCost(result.cost) + const cost = addAutoRoutingCost( + resolveProxiedModelCost(result.cost), + resolved.autoRouting?.billableRoutingCost ?? 0 + ) return { prompt: inputs.prompt, - model: result.model, + model: resolved.autoRouting ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: tokens.input || DEFAULTS.TOKENS.PROMPT, output: tokens.output || DEFAULTS.TOKENS.COMPLETION, @@ -160,6 +177,7 @@ export class RouterBlockHandler implements BlockHandler { input: cost.input, output: cost.output, total: cost.total, + ...(cost.routing === undefined ? {} : { routing: cost.routing }), }, selectedPath: { blockId: chosenBlock.id, @@ -201,16 +219,23 @@ export class RouterBlockHandler implements BlockHandler { bedrockRegion: inputs.bedrockRegion, } - await validateModelProvider(ctx.userId, ctx.workspaceId, routerConfig.model, ctx) - - const providerId = getProviderFromModel(routerConfig.model) - try { const url = new URL('/api/providers', getInternalApiBaseUrl()) if (ctx.userId) url.searchParams.set('userId', ctx.userId) const messages = [{ role: 'user', content: routerConfig.context }] const systemPrompt = generateRouterV2Prompt(routerConfig.context, routes) + const resolved = await this.resolveModel( + ctx, + block.id, + routerConfig.model, + systemPrompt, + routerConfig.context, + true + ) + + await validateModelProvider(ctx.userId, ctx.workspaceId, resolved.model, ctx) + const providerId = getProviderFromModel(resolved.model) let finalApiKey: string | undefined = routerConfig.apiKey if (providerId === 'vertex' && routerConfig.vertexCredential) { @@ -223,8 +248,8 @@ export class RouterBlockHandler implements BlockHandler { const providerRequest: Record = { provider: providerId, - model: routerConfig.model, - systemPrompt: systemPrompt, + model: resolved.model, + systemPrompt: resolved.systemPrompt, context: JSON.stringify(messages), temperature: ROUTER.INFERENCE_TEMPERATURE, apiKey: finalApiKey, @@ -303,7 +328,10 @@ export class RouterBlockHandler implements BlockHandler { const chosenRoute = routes.find((r) => r.id === chosenRouteId) if (!chosenRoute) { - const availableRoutes = routes.map((r) => ({ id: r.id, title: r.title })) + const availableRoutes = routes.map((r) => ({ + id: r.id, + title: r.title, + })) logger.error( `Invalid routing decision. Response content: "${result.content}". Available routes:`, availableRoutes @@ -327,11 +355,14 @@ export class RouterBlockHandler implements BlockHandler { total: DEFAULTS.TOKENS.TOTAL, } - const cost = resolveProxiedModelCost(result.cost) + const cost = addAutoRoutingCost( + resolveProxiedModelCost(result.cost), + resolved.autoRouting?.billableRoutingCost ?? 0 + ) return { context: inputs.context, - model: result.model, + model: resolved.autoRouting ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: tokens.input || DEFAULTS.TOKENS.PROMPT, output: tokens.output || DEFAULTS.TOKENS.COMPLETION, @@ -341,6 +372,7 @@ export class RouterBlockHandler implements BlockHandler { input: cost.input, output: cost.output, total: cost.total, + ...(cost.routing === undefined ? {} : { routing: cost.routing }), }, selectedRoute: chosenRoute.id, reasoning, @@ -380,6 +412,53 @@ export class RouterBlockHandler implements BlockHandler { } } + private async resolveModel( + ctx: ExecutionContext, + blockId: string, + configuredModel: string, + systemPrompt: string, + lastMessage: unknown, + hasResponseFormat: boolean + ): Promise<{ + model: string + systemPrompt: string + autoRouting: AutoRoutingResult | null + }> { + if (!isAutoModel(configuredModel)) { + return { model: configuredModel, systemPrompt, autoRouting: null } + } + + const message = + typeof lastMessage === 'string' ? lastMessage : JSON.stringify(lastMessage ?? '') + const autoRouting = await resolveAutoModel({ + ctx, + blockId, + signals: { + systemPrompt, + lastMessage: message, + messageCount: 1, + toolNames: [], + mediaKind: 'none', + hasResponseFormat, + approxInputTokens: Math.ceil((systemPrompt.length + message.length) / 4), + }, + fallbackModel: ROUTER.DEFAULT_MODEL, + }) + + logger.info('Resolved sim-auto model for router', { + blockId, + model: autoRouting.model, + tier: autoRouting.tier, + decidedBy: autoRouting.decidedBy, + }) + + return { + model: autoRouting.model, + systemPrompt: [SIM_AUTO_SYSTEM_PREAMBLE, systemPrompt].filter(Boolean).join('\n\n'), + autoRouting, + } + } + private getTargetBlocks(ctx: ExecutionContext, block: SerializedBlock) { return ctx.workflow?.connections .filter((conn) => conn.source === block.id) diff --git a/apps/sim/lib/model-router/resolve.test.ts b/apps/sim/lib/model-router/resolve.test.ts index d4f65f7b6e5..a0cbbccd3a1 100644 --- a/apps/sim/lib/model-router/resolve.test.ts +++ b/apps/sim/lib/model-router/resolve.test.ts @@ -40,7 +40,11 @@ vi.mock('@/providers/utils', () => ({ getProviderFromModel: mockGetProviderFromModel, })) -import { type AutoRoutingSignals, resolveAutoModel } from '@/lib/model-router/resolve' +import { + type AutoRoutingSignals, + addAutoRoutingCost, + resolveAutoModel, +} from '@/lib/model-router/resolve' import type { ExecutionContext } from '@/executor/types' const ctx = { @@ -69,6 +73,23 @@ function routerResponse(body: unknown, ok = true, status = 200) { return { ok, status, json: () => Promise.resolve(body) } } +describe('addAutoRoutingCost', () => { + it('adds a billable routing call to a settled provider cost', () => { + expect(addAutoRoutingCost({ input: 0.001, output: 0.002, total: 0.003 }, 0.0005)).toEqual({ + input: 0.001, + output: 0.002, + routing: 0.0005, + total: 0.0035, + }) + }) + + it('leaves provider cost unchanged when routing was not billable', () => { + const cost = { input: 0.001, output: 0.002, total: 0.003 } + + expect(addAutoRoutingCost(cost, 0)).toBe(cost) + }) +}) + describe('resolveAutoModel', () => { beforeEach(() => { vi.clearAllMocks() @@ -78,10 +99,22 @@ describe('resolveAutoModel', () => { }) /** The full pool, exactly as specified: media kind → tier → model. */ - const POOL_CASES: Array<{ mediaKind: AutoRoutingSignals['mediaKind']; byTier: string[] }> = [ - { mediaKind: 'none', byTier: ['fireworks/glm-5.2', 'fireworks/glm-5.2', 'fireworks/kimi-k3'] }, - { mediaKind: 'image', byTier: ['gemini-3.6-flash', 'fireworks/kimi-k3', 'fireworks/kimi-k3'] }, - { mediaKind: 'file', byTier: ['gemini-3.6-flash', 'claude-sonnet-5', 'gpt-5.6-sol'] }, + const POOL_CASES: Array<{ + mediaKind: AutoRoutingSignals['mediaKind'] + byTier: string[] + }> = [ + { + mediaKind: 'none', + byTier: ['fireworks/glm-5.2', 'fireworks/glm-5.2', 'fireworks/kimi-k3'], + }, + { + mediaKind: 'image', + byTier: ['gemini-3.6-flash', 'fireworks/kimi-k3', 'fireworks/kimi-k3'], + }, + { + mediaKind: 'file', + byTier: ['gemini-3.6-flash', 'claude-sonnet-5', 'gpt-5.6-sol'], + }, ] for (const { mediaKind, byTier } of POOL_CASES) { @@ -148,7 +181,11 @@ describe('resolveAutoModel', () => { const result = await resolveAutoModel({ ctx, blockId: 'b1', - signals: makeSignals({ approxInputTokens: 20, toolNames: [], hasResponseFormat: false }), + signals: makeSignals({ + approxInputTokens: 20, + toolNames: [], + hasResponseFormat: false, + }), fallbackModel: 'claude-sonnet-5', }) diff --git a/apps/sim/lib/model-router/resolve.ts b/apps/sim/lib/model-router/resolve.ts index 9a4a1921570..bba46bdfe38 100644 --- a/apps/sim/lib/model-router/resolve.ts +++ b/apps/sim/lib/model-router/resolve.ts @@ -7,6 +7,7 @@ import { env } from '@/lib/core/config/env' import { getCostMultiplier, isHosted } from '@/lib/core/config/env-flags' import { validateModelProvider } from '@/ee/access-control/utils/permission-check' import type { ExecutionContext } from '@/executor/types' +import type { ModelCost } from '@/providers/cost-policy' import { getProviderFromModel } from '@/providers/utils' const logger = createLogger('ModelRouter') @@ -83,7 +84,7 @@ const MAX_SIGNAL_CHARS = 2000 const DECISION_CACHE_TTL_MS = 5 * 60 * 1000 const DECISION_CACHE_MAX_ENTRIES = 500 -/** Compact facts about the pending agent-block task; never the full payload. */ +/** Compact facts about the pending LLM-backed block task; never the full payload. */ export interface AutoRoutingSignals { systemPrompt?: string lastMessage?: string @@ -128,6 +129,19 @@ export interface AutoRoutingResult { usage?: ModelRouterUsage } +export type AutoRoutedModelCost = ModelCost & { routing?: number } + +/** Adds a successful sim-auto classifier call to a settled provider cost. */ +export function addAutoRoutingCost(cost: ModelCost, routingCost: number): AutoRoutedModelCost { + if (routingCost <= 0) return cost + + return { + ...cost, + routing: routingCost, + total: cost.total + routingCost, + } +} + const decisionCache = new Map() function cacheKey(signals: AutoRoutingSignals): string { @@ -192,7 +206,9 @@ async function pickModelForTier( await validateModelProvider(ctx.userId, ctx.workspaceId ?? undefined, model, ctx) return model } catch { - logger.info('sim-auto candidate unavailable, trying the next one', { model }) + logger.info('sim-auto candidate unavailable, trying the next one', { + model, + }) } } } @@ -242,7 +258,7 @@ async function callModelRouter( } /** - * Resolves the sim-auto pseudo-model to a concrete model for one agent-block + * Resolves the sim-auto pseudo-model to a concrete model for one block * execution. Never throws and never fails the workflow: any error, timeout, * non-hosted deployment, or fully unavailable pool column falls back to * `fallbackModel` (the block's standard default). @@ -273,7 +289,12 @@ export async function resolveAutoModel(args: { if (cachedTier) { const model = await pickModelForTier(signals.mediaKind, cachedTier, ctx) if (!model) return fallback - return { model, tier: cachedTier, decidedBy: 'cache', billableRoutingCost: 0 } + return { + model, + tier: cachedTier, + decidedBy: 'cache', + billableRoutingCost: 0, + } } const response = await callModelRouter(signals, ctx, blockId)