From 3f65674a2b97f853ac610b6908638e9c82ae5372 Mon Sep 17 00:00:00 2001 From: Marcus Chandra Date: Thu, 17 Sep 2026 22:52:06 -0700 Subject: [PATCH 01/14] feat(workflow-types): register the model-fallback-list subblock type Adds the type to the shared SubBlockType union, the block registry test allowlist, the tool-input exclusion set, and the docs generator's semantic type map. --- apps/sim/blocks/blocks.test.ts | 1 + apps/sim/tools/params.ts | 1 + packages/workflow-types/src/blocks.ts | 1 + scripts/generate-docs.ts | 1 + 4 files changed, 4 insertions(+) diff --git a/apps/sim/blocks/blocks.test.ts b/apps/sim/blocks/blocks.test.ts index 693b2d00bdf..15077a66fdb 100644 --- a/apps/sim/blocks/blocks.test.ts +++ b/apps/sim/blocks/blocks.test.ts @@ -604,6 +604,7 @@ describe.concurrent('Blocks Module', () => { 'text', 'router-input', 'table-selector', + 'model-fallback-list', 'column-selector', 'filter-builder', 'sort-builder', diff --git a/apps/sim/tools/params.ts b/apps/sim/tools/params.ts index 7666075e19b..f1cce808250 100644 --- a/apps/sim/tools/params.ts +++ b/apps/sim/tools/params.ts @@ -659,6 +659,7 @@ const EXCLUDED_SUBBLOCK_TYPES = new Set([ 'mcp-dynamic-args', 'variables-input', 'messages-input', + 'model-fallback-list', 'router-input', 'text', ]) diff --git a/packages/workflow-types/src/blocks.ts b/packages/workflow-types/src/blocks.ts index e40cf7441d8..10e08a588b0 100644 --- a/packages/workflow-types/src/blocks.ts +++ b/packages/workflow-types/src/blocks.ts @@ -54,6 +54,7 @@ export type SubBlockType = | 'text' | 'router-input' | 'table-selector' + | 'model-fallback-list' | 'column-selector' | 'modal' diff --git a/scripts/generate-docs.ts b/scripts/generate-docs.ts index 92853f28f38..9849117afbe 100755 --- a/scripts/generate-docs.ts +++ b/scripts/generate-docs.ts @@ -4966,6 +4966,7 @@ const SUBBLOCK_TYPE_TO_SEMANTIC: Record = { 'oauth-input': 'string', code: 'string', 'file-upload': 'string', + 'model-fallback-list': 'json', text: 'string', } From 425bc7d090156b7c34474688e6f0d3fccf668e8f Mon Sep 17 00:00:00 2001 From: Marcus Chandra Date: Thu, 17 Sep 2026 22:52:08 -0700 Subject: [PATCH 02/14] feat(agent): add the fallbackModels subblock and its shared helpers The Agent block gains an ordered list of fallback models as an advanced field. The helpers normalize stored rows into execution candidates, decide which models a row may offer (credentials the block can actually supply), whether a row needs its own env-var key, and how the primary's tuning carries over: graded knobs only when the fallback declares the value, temperature and max output tokens clamped to the fallback's caps. --- apps/sim/blocks/blocks/agent.test.ts | 21 ++ apps/sim/blocks/blocks/agent.ts | 22 +- apps/sim/blocks/utils.test.ts | 25 ++ apps/sim/blocks/utils.ts | 23 +- apps/sim/executor/handlers/agent/types.ts | 6 +- .../workflows/blocks/fallback-models.test.ts | 307 ++++++++++++++++++ .../lib/workflows/blocks/fallback-models.ts | 287 ++++++++++++++++ 7 files changed, 687 insertions(+), 4 deletions(-) create mode 100644 apps/sim/lib/workflows/blocks/fallback-models.test.ts create mode 100644 apps/sim/lib/workflows/blocks/fallback-models.ts diff --git a/apps/sim/blocks/blocks/agent.test.ts b/apps/sim/blocks/blocks/agent.test.ts index 23ec48fed05..9be09f82a0a 100644 --- a/apps/sim/blocks/blocks/agent.test.ts +++ b/apps/sim/blocks/blocks/agent.test.ts @@ -30,6 +30,27 @@ describe('AgentBlock', () => { } describe('tools.config.params function', () => { + it('normalizes fallback models and drops the key when none survive', () => { + const withRows = paramsFunction({ + model: 'gpt-4o', + fallbackModels: [ + { id: 'a', model: ' claude-sonnet-5 ' }, + { id: 'b', model: 'sim-auto' }, + { id: 'c', model: 'claude-sonnet-5' }, + { id: 'd', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { id: 'e', model: 'openrouter/y', apiKey: '' }, + ], + }) + expect(withRows.fallbackModels).toEqual([ + { model: 'claude-sonnet-5' }, + { model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { model: 'openrouter/y' }, + ]) + + const empty = paramsFunction({ model: 'gpt-4o', fallbackModels: [{ id: 'a', model: '' }] }) + expect(empty).not.toHaveProperty('fallbackModels') + }) + it('should pass through params when no tools array is provided', () => { const params = { model: 'gpt-4o', diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index b09eae6e203..7bf4d1e697a 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -1,5 +1,7 @@ import { createLogger } from '@sim/logger' +import { omit } from '@sim/utils/object' import { AgentIcon } from '@/components/icons' +import { normalizeFallbackModels } from '@/lib/workflows/blocks/fallback-models' import type { BlockConfig } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' import { @@ -429,6 +431,14 @@ Return ONLY the JSON array.`, value: MODELS_WITH_DEEP_RESEARCH, }, }, + { + id: 'fallbackModels', + title: 'Fallback models', + type: 'model-fallback-list', + mode: 'advanced', + description: + 'Ordered models tried in sequence when the request to the selected model fails. Each row is { model, apiKey?, reasoningEffort?, thinkingLevel?, verbosity? }; apiKey, when present, must be a whole {{ENV_VAR}} reference, and a tuning value must be one the row model declares. sim-auto is not allowed. Max 5.', + }, ], tools: { access: [ @@ -448,7 +458,12 @@ Return ONLY the JSON array.`, }, params: (params: Record) => { const normalizedFiles = normalizeFileInput(params.files) - const baseParams = normalizedFiles ? { ...params, files: normalizedFiles } : params + const withFiles = normalizedFiles ? { ...params, files: normalizedFiles } : params + const fallbackModels = normalizeFallbackModels(params.fallbackModels) + const baseParams = + fallbackModels.length > 0 + ? { ...withFiles, fallbackModels } + : omit(withFiles, ['fallbackModels']) // If tools array is provided, handle tool usage control if (params.tools && Array.isArray(params.tools)) { @@ -586,6 +601,11 @@ Return ONLY the JSON array.`, type: 'boolean', description: 'Cache the system prompt and tool definitions on models that support it', }, + fallbackModels: { + type: 'json', + description: + 'Ordered fallback models tried when the selected model fails, each { model, apiKey?: "{{ENV_VAR}}", reasoningEffort?, thinkingLevel?, verbosity? }', + }, tools: { type: 'json', description: 'Available tools configuration' }, skills: { type: 'json', description: 'Selected skills configuration' }, }, diff --git a/apps/sim/blocks/utils.test.ts b/apps/sim/blocks/utils.test.ts index c260de06e87..56845776835 100644 --- a/apps/sim/blocks/utils.test.ts +++ b/apps/sim/blocks/utils.test.ts @@ -75,6 +75,7 @@ import { parseOptionalBooleanInput, parseOptionalJsonInput, parseOptionalNumberInput, + requiresProviderFamilyCredentials, } from '@/blocks/utils' import { getProviderFromModel } from '@/providers/utils' @@ -97,6 +98,30 @@ const BASE_CLOUD_MODELS: Record = { 'mistral-large-latest': 'mistral', } +describe('requiresProviderFamilyCredentials', () => { + beforeEach(() => { + setEnvFlags({ isHosted: false, isAzureConfigured: false, isOllamaConfigured: false }) + }) + + it('is true for Vertex and Bedrock, whose credentials live on the block', () => { + expect(requiresProviderFamilyCredentials('vertex/gemini-2.5-pro')).toBe(true) + expect(requiresProviderFamilyCredentials('bedrock/my-inference-profile')).toBe(true) + }) + + it('is true for Azure only until the deployment configures it server-side', () => { + expect(requiresProviderFamilyCredentials('azure/my-deployment')).toBe(true) + expect(requiresProviderFamilyCredentials('azure-anthropic/my-deployment')).toBe(true) + setEnvFlags({ isAzureConfigured: true }) + expect(requiresProviderFamilyCredentials('azure/my-deployment')).toBe(false) + }) + + it('is false for API-key providers, local servers, and unknown ids', () => { + expect(requiresProviderFamilyCredentials('openrouter/anthropic/claude')).toBe(false) + expect(requiresProviderFamilyCredentials('ollama/llama3')).toBe(false) + expect(requiresProviderFamilyCredentials('')).toBe(false) + }) +}) + describe('getApiKeyCondition / shouldRequireApiKeyForModel', () => { const evaluateCondition = (model: string): boolean => { const conditionFn = getApiKeyCondition() diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index bc81c39cc2a..09560d95d8c 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -137,7 +137,12 @@ function buildModelVisibilityCondition(model: string, shouldShow: boolean) { return shouldShow ? { field: 'model', value: model } : { field: 'model', value: model, not: true } } -function shouldRequireApiKeyForModel(model: string): boolean { +/** + * Whether the block must show an API Key field for `model` on this deployment: + * false for hosted models on hosted Sim (BYOK or the platform key serve them), + * for providers with their own credential fields, and for local servers. + */ +export function shouldRequireApiKeyForModel(model: string): boolean { const normalizedModel = model.trim().toLowerCase() if (!normalizedModel) return false @@ -278,6 +283,22 @@ export function getCohereRerankerApiKeyCondition() { } } +/** + * Whether `model` can only run with credentials that live on the block beyond an + * API key: a Vertex OAuth credential, Bedrock AWS keys and region, or an Azure + * endpoint the deployment has not configured server-side. Those fields render + * only while the block's own `model` is in that provider family, so nothing + * outside the family can inherit them. + */ +export function requiresProviderFamilyCredentials(model: string): boolean { + const provider = findProviderFromModel(model.trim()) + if (provider === 'vertex' || provider === 'bedrock') return true + if (provider === 'azure-openai' || provider === 'azure-anthropic') { + return !getDeploymentShape().azureConfigured + } + return false +} + function getModelProviderCondition(...providerIds: ProviderId[]) { return (values?: Record) => { const model = typeof values?.model === 'string' ? values.model : '' diff --git a/apps/sim/executor/handlers/agent/types.ts b/apps/sim/executor/handlers/agent/types.ts index d2e30314589..460dd676153 100644 --- a/apps/sim/executor/handlers/agent/types.ts +++ b/apps/sim/executor/handlers/agent/types.ts @@ -32,8 +32,8 @@ export interface AgentInputs { // Deep research multi-turn previousInteractionId?: string // Interactions API previous interaction reference // LLM parameters - temperature?: string - maxTokens?: string + temperature?: string | number + maxTokens?: string | number apiKey?: string azureEndpoint?: string azureApiVersion?: string @@ -48,6 +48,8 @@ export interface AgentInputs { thinkingLevel?: string promptCaching?: boolean files?: unknown + /** Ordered models tried when the request to `model` fails; see `normalizeFallbackModels`. */ + fallbackModels?: Array<{ id?: string; model: string; apiKey?: string }> } /** diff --git a/apps/sim/lib/workflows/blocks/fallback-models.test.ts b/apps/sim/lib/workflows/blocks/fallback-models.test.ts new file mode 100644 index 00000000000..2484f075c7e --- /dev/null +++ b/apps/sim/lib/workflows/blocks/fallback-models.test.ts @@ -0,0 +1,307 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockShouldRequireApiKey, mockRequiresFamilyCredentials } = vi.hoisted(() => ({ + mockShouldRequireApiKey: vi.fn((model: string) => false), + mockRequiresFamilyCredentials: vi.fn((model: string) => false), +})) + +vi.mock('@/blocks/utils', () => ({ + shouldRequireApiKeyForModel: mockShouldRequireApiKey, + requiresProviderFamilyCredentials: mockRequiresFamilyCredentials, +})) + +vi.mock('@/providers/models', () => ({ + isAutoModel: (model: string) => model.trim().toLowerCase() === 'sim-auto', + isKnownModelId: (model: string) => model.startsWith('gpt') || model.startsWith('claude'), + findProviderFromModel: (model: string) => { + const lower = model.toLowerCase() + if (lower.startsWith('gpt')) return 'openai' + if (lower.startsWith('claude')) return 'anthropic' + if (lower.startsWith('vertex/')) return 'vertex' + if (lower.startsWith('openrouter/')) return 'openrouter' + return null + }, + getReasoningEffortValuesForModel: (model: string) => + model === 'gpt-big' + ? ['low', 'medium', 'high', 'xhigh'] + : model === 'gpt-small' + ? ['low', 'high'] + : null, + getThinkingLevelsForModel: (model: string) => + model.startsWith('claude') ? ['low', 'medium', 'high'] : null, + getVerbosityValuesForModel: (model: string) => + model.startsWith('gpt') ? ['low', 'medium', 'high'] : null, + getMaxTemperature: (model: string) => + model.startsWith('claude') ? 1 : model.startsWith('gpt') ? 2 : undefined, + getModelCapabilities: (model: string) => + model === 'gpt-small' + ? { maxOutputTokens: 4096 } + : model.startsWith('gpt') + ? { maxOutputTokens: 16000 } + : model.startsWith('claude') + ? {} + : null, +})) + +import { + fallbackRowNeedsApiKey, + getFallbackTuningKnobsToShow, + getTuningOptionsForModel, + isTuningValueValidForModel, + isViableFallbackModel, + isWholeEnvVarReference, + MAX_FALLBACK_MODELS, + normalizeFallbackModels, + ordinalChoiceLabel, + resolveFallbackTuning, +} from '@/lib/workflows/blocks/fallback-models' + +beforeEach(() => { + vi.clearAllMocks() + mockShouldRequireApiKey.mockReturnValue(false) + mockRequiresFamilyCredentials.mockReturnValue(false) +}) + +describe('isWholeEnvVarReference', () => { + it('accepts exactly one braced variable name', () => { + expect(isWholeEnvVarReference('{{OPENROUTER_API_KEY}}')).toBe(true) + expect(isWholeEnvVarReference('{{ key_1 }}')).toBe(true) + }) + + it('refuses raw keys, partial references, and non-strings', () => { + expect(isWholeEnvVarReference('sk-live-abc')).toBe(false) + expect(isWholeEnvVarReference('prefix {{KEY}}')).toBe(false) + expect(isWholeEnvVarReference('{{A}}{{B}}')).toBe(false) + expect(isWholeEnvVarReference('{{1BAD}}')).toBe(false) + expect(isWholeEnvVarReference(42)).toBe(false) + expect(isWholeEnvVarReference(null)).toBe(false) + }) +}) + +describe('normalizeFallbackModels', () => { + it('returns an empty chain for anything that is not an array', () => { + expect(normalizeFallbackModels(undefined)).toEqual([]) + expect(normalizeFallbackModels('gpt-5')).toEqual([]) + expect(normalizeFallbackModels({ model: 'gpt-5' })).toEqual([]) + }) + + it('keeps order, trims, and drops rows without a model', () => { + expect( + normalizeFallbackModels([ + { id: 'a', model: ' gpt-5 ' }, + { id: 'b', model: '' }, + { id: 'c' }, + null, + { id: 'd', model: 'claude-sonnet-5' }, + ]) + ).toEqual([{ model: 'gpt-5' }, { model: 'claude-sonnet-5' }]) + }) + + it('drops sim-auto and case-insensitive duplicates, keeping the first position', () => { + expect( + normalizeFallbackModels([ + { model: 'sim-auto' }, + { model: 'gpt-5' }, + { model: 'GPT-5' }, + { model: 'claude-sonnet-5' }, + ]) + ).toEqual([{ model: 'gpt-5' }, { model: 'claude-sonnet-5' }]) + }) + + it('keeps a row tuning value, trimmed and lower-cased', () => { + expect( + normalizeFallbackModels([{ model: 'gpt-small', reasoningEffort: ' Low ', thinkingLevel: '' }]) + ).toEqual([{ model: 'gpt-small', reasoningEffort: 'low' }]) + }) + + it('keeps any non-empty key, reference or already resolved, and drops blanks', () => { + expect( + normalizeFallbackModels([ + { model: 'openrouter/a', apiKey: '{{OPENROUTER_API_KEY}}' }, + { model: 'openrouter/b', apiKey: ' sk-resolved-at-runtime ' }, + { model: 'openrouter/c', apiKey: '' }, + { model: 'openrouter/d', apiKey: 42 }, + ]) + ).toEqual([ + { model: 'openrouter/a', apiKey: '{{OPENROUTER_API_KEY}}' }, + { model: 'openrouter/b', apiKey: 'sk-resolved-at-runtime' }, + { model: 'openrouter/c' }, + { model: 'openrouter/d' }, + ]) + }) + + it('caps the chain', () => { + const rows = Array.from({ length: MAX_FALLBACK_MODELS + 3 }, (_, i) => ({ model: `m-${i}` })) + expect(normalizeFallbackModels(rows)).toHaveLength(MAX_FALLBACK_MODELS) + }) +}) + +describe('isViableFallbackModel', () => { + it('rejects empty, sim-auto, the primary itself, and unresolvable ids', () => { + expect(isViableFallbackModel('', 'gpt-5')).toBe(false) + expect(isViableFallbackModel('sim-auto', 'gpt-5')).toBe(false) + expect(isViableFallbackModel('GPT-5', 'gpt-5')).toBe(false) + expect(isViableFallbackModel('mystery-model', 'gpt-5')).toBe(false) + }) + + it('offers any resolvable model that needs no provider-family credentials', () => { + expect(isViableFallbackModel('claude-sonnet-5', 'gpt-5')).toBe(true) + mockShouldRequireApiKey.mockReturnValue(true) + expect(isViableFallbackModel('openrouter/x', 'gpt-5')).toBe(true) + }) + + it('offers a family-bound model only alongside a primary of the same family', () => { + mockRequiresFamilyCredentials.mockImplementation((model: string) => model.startsWith('vertex/')) + expect(isViableFallbackModel('vertex/gemini-b', 'vertex/gemini-a')).toBe(true) + expect(isViableFallbackModel('vertex/gemini-b', 'gpt-5')).toBe(false) + }) +}) + +describe('fallbackRowNeedsApiKey', () => { + it('is false when the model needs no key at all', () => { + expect(fallbackRowNeedsApiKey('claude-sonnet-5', 'gpt-5')).toBe(false) + }) + + it('is false when the block key on the same provider can be reused', () => { + mockShouldRequireApiKey.mockReturnValue(true) + expect(fallbackRowNeedsApiKey('gpt-5-mini', 'gpt-5')).toBe(false) + }) + + it('is true for a keyed model on another provider', () => { + mockShouldRequireApiKey.mockReturnValue(true) + expect(fallbackRowNeedsApiKey('openrouter/x', 'gpt-5')).toBe(true) + }) +}) + +describe('tuning options and validity', () => { + it('offers the provider-decides entry first, then what the model declares', () => { + expect(getTuningOptionsForModel('gpt-small', 'reasoningEffort')).toEqual([ + 'auto', + 'low', + 'high', + ]) + expect(getTuningOptionsForModel('claude-sonnet-5', 'thinkingLevel')).toEqual([ + 'none', + 'low', + 'medium', + 'high', + ]) + expect(getTuningOptionsForModel('claude-sonnet-5', 'reasoningEffort')).toBeNull() + expect(getTuningOptionsForModel('', 'verbosity')).toBeNull() + }) + + it('treats unset, the sentinel, and declared values as valid, and passes uncatalogued through', () => { + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', undefined)).toBe(true) + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', 'auto')).toBe(true) + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', 'High')).toBe(true) + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', 'xhigh')).toBe(false) + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', 42)).toBe(false) + expect(isTuningValueValidForModel('openrouter/x', 'reasoningEffort', 'anything')).toBe(true) + /** Catalogued but without the knob: nothing but unset or the sentinel is acceptable. */ + expect(isTuningValueValidForModel('claude-sonnet-5', 'reasoningEffort', 'high')).toBe(false) + expect(isTuningValueValidForModel('claude-sonnet-5', 'reasoningEffort', 'auto')).toBe(true) + }) +}) + +describe('getFallbackTuningKnobsToShow', () => { + it('shows a knob the primary lacks and one whose primary value the fallback does not declare', () => { + expect(getFallbackTuningKnobsToShow('claude-sonnet-5', 'gpt-big', {})).toEqual([ + 'thinkingLevel', + ]) + expect( + getFallbackTuningKnobsToShow('gpt-small', 'gpt-big', { reasoningEffort: 'xhigh' }) + ).toEqual(['reasoningEffort']) + }) + + it('stays bare when the primary value carries over or nothing is set', () => { + expect( + getFallbackTuningKnobsToShow('gpt-small', 'gpt-big', { reasoningEffort: 'high' }) + ).toEqual([]) + expect(getFallbackTuningKnobsToShow('gpt-small', 'gpt-big', {})).toEqual([]) + expect( + getFallbackTuningKnobsToShow('gpt-small', 'gpt-big', { reasoningEffort: 'auto' }) + ).toEqual([]) + }) +}) + +describe('resolveFallbackTuning', () => { + it('prefers the row value, inherits a declared primary value, and drops the rest', () => { + const resolved = resolveFallbackTuning( + { model: 'gpt-small', reasoningEffort: 'low' }, + 'gpt-big', + { + reasoningEffort: 'xhigh', + verbosity: 'high', + } + ) + expect(resolved.reasoningEffort).toBe('low') + expect(resolved.verbosity).toBe('high') + expect(resolved.thinkingLevel).toBeUndefined() + expect(resolved.adjustments).toEqual(['reasoningEffort: xhigh -> low']) + }) + + it('drops a primary value the fallback does not declare and says so', () => { + const resolved = resolveFallbackTuning({ model: 'gpt-small' }, 'gpt-big', { + reasoningEffort: 'xhigh', + }) + expect(resolved.reasoningEffort).toBeUndefined() + expect(resolved.adjustments).toEqual(['reasoningEffort: xhigh -> provider default']) + }) + + it('never inherits a knob the primary does not have', () => { + const resolved = resolveFallbackTuning({ model: 'claude-sonnet-5' }, 'gpt-big', { + thinkingLevel: 'high', + }) + expect(resolved.thinkingLevel).toBeUndefined() + }) + + it('clamps temperature and max tokens to the fallback caps, keeping the input type', () => { + const resolved = resolveFallbackTuning({ model: 'claude-sonnet-5' }, 'gpt-big', { + temperature: '1.5', + maxTokens: 20000, + }) + expect(resolved.temperature).toBe('1') + expect(resolved.maxTokens).toBe(20000) + expect(resolved.adjustments).toEqual(['temperature: 1.5 -> 1']) + + const small = resolveFallbackTuning({ model: 'gpt-small' }, 'gpt-big', { + temperature: 0.2, + maxTokens: '20000', + }) + expect(small.temperature).toBe(0.2) + expect(small.maxTokens).toBe('4096') + }) + + it('passes everything through for an uncatalogued fallback', () => { + const resolved = resolveFallbackTuning({ model: 'openrouter/x' }, 'gpt-big', { + reasoningEffort: 'xhigh', + temperature: '1.9', + maxTokens: '99999', + }) + expect(resolved).toEqual({ + reasoningEffort: 'xhigh', + thinkingLevel: undefined, + verbosity: undefined, + temperature: '1.9', + maxTokens: '99999', + adjustments: [], + }) + }) +}) + +describe('ordinalChoiceLabel', () => { + it('starts at the 2nd choice and handles English ordinals', () => { + expect([0, 1, 2, 3, 9, 10, 11].map(ordinalChoiceLabel)).toEqual([ + '2nd choice', + '3rd choice', + '4th choice', + '5th choice', + '11th choice', + '12th choice', + '13th choice', + ]) + }) +}) diff --git a/apps/sim/lib/workflows/blocks/fallback-models.ts b/apps/sim/lib/workflows/blocks/fallback-models.ts new file mode 100644 index 00000000000..9f21075e58e --- /dev/null +++ b/apps/sim/lib/workflows/blocks/fallback-models.ts @@ -0,0 +1,287 @@ +import { requiresProviderFamilyCredentials, shouldRequireApiKeyForModel } from '@/blocks/utils' +import { + findProviderFromModel, + getMaxTemperature, + getModelCapabilities, + getReasoningEffortValuesForModel, + getThinkingLevelsForModel, + getVerbosityValuesForModel, + isAutoModel, + isKnownModelId, +} from '@/providers/models' + +/** Upper bound on fallback rows; enough for a full provider spread without an unbounded chain. */ +export const MAX_FALLBACK_MODELS = 5 + +/** The graded tuning knobs a fallback row may set for itself. */ +export const FALLBACK_TUNING_KNOBS = ['reasoningEffort', 'thinkingLevel', 'verbosity'] as const +export type FallbackTuningKnob = (typeof FALLBACK_TUNING_KNOBS)[number] + +/** The "let the provider decide" entry each knob's field offers first, as the block's own fields do. */ +const KNOB_SENTINEL: Record = { + reasoningEffort: 'auto', + thinkingLevel: 'none', + verbosity: 'auto', +} + +export const FALLBACK_TUNING_LABELS: Record = { + reasoningEffort: 'Reasoning effort', + thinkingLevel: 'Thinking level', + verbosity: 'Verbosity', +} + +/** Per-row tuning: present only when the builder chose a value for that knob. */ +export type FallbackTuningValues = Partial> + +/** + * One stored fallback row. `id` is a React key only. `apiKey`, when present, is + * always a whole `{{ENV_VAR}}` reference: a raw secret nested inside a list + * value would bypass every redaction path that keys on a top-level + * `password: true` field, so the picker cannot produce one and the copilot + * validator refuses one. The tuning knobs hold a value only when the primary's + * setting could not be carried over (see `getFallbackTuningKnobsToShow`). + */ +export interface FallbackModelEntry extends FallbackTuningValues { + id: string + model: string + apiKey?: string +} + +/** A row the executor acts on: the React key is gone, and `apiKey` is a validated reference. */ +export interface FallbackModelCandidate extends FallbackTuningValues { + model: string + apiKey?: string +} + +const WHOLE_ENV_VAR_REFERENCE = /^\{\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}\}$/ + +/** Whether a value is exactly one `{{ENV_VAR}}` reference and nothing else. */ +export function isWholeEnvVarReference(value: unknown): value is string { + return typeof value === 'string' && WHOLE_ENV_VAR_REFERENCE.test(value.trim()) +} + +/** + * Normalizes a stored fallback list into the ordered candidates execution walks. + * + * Tolerant rather than strict because it runs on every execution: rows the + * editor could not have written (missing model, sim-auto) are dropped instead + * of failing the block, and duplicates keep their first position so the order + * the builder chose is preserved. + * + * A row's `apiKey` is kept as any non-empty string. By the time this runs the + * input resolver has already turned the stored `{{ENV_VAR}}` reference into the + * key itself, exactly as it does for the block's own API Key field; the + * reference-only rule is enforced where rows are written (the picker, the + * copilot validator) and where they leave the workspace (the export sanitizer). + */ +export function normalizeFallbackModels(raw: unknown): FallbackModelCandidate[] { + if (!Array.isArray(raw)) return [] + + const seen = new Set() + const candidates: FallbackModelCandidate[] = [] + for (const row of raw) { + if (!row || typeof row !== 'object') continue + const { model, apiKey } = row as { model?: unknown; apiKey?: unknown } + if (typeof model !== 'string') continue + const trimmed = model.trim() + if (!trimmed || isAutoModel(trimmed)) continue + const key = trimmed.toLowerCase() + if (seen.has(key)) continue + seen.add(key) + const resolvedKey = typeof apiKey === 'string' ? apiKey.trim() : '' + const candidate: FallbackModelCandidate = { + model: trimmed, + ...(resolvedKey ? { apiKey: resolvedKey } : {}), + } + for (const knob of FALLBACK_TUNING_KNOBS) { + const value = (row as Record)[knob] + const level = typeof value === 'string' ? value.trim().toLowerCase() : '' + if (level) candidate[knob] = level + } + candidates.push(candidate) + if (candidates.length >= MAX_FALLBACK_MODELS) break + } + return candidates +} + +/** + * Whether a model can serve as a fallback for `primaryModel` with the + * credentials the block can actually give it. + * + * A fallback resolves its key the way the primary does — workspace BYOK, the + * platform key, or the block's own field — with one addition: a row may name a + * workspace variable holding its key. What it can never do is inherit a Vertex + * credential, Bedrock keys, or an Azure endpoint from a primary in another + * family, because those fields only render for the primary's own provider. + */ +export function isViableFallbackModel(model: string, primaryModel: string): boolean { + const trimmed = model.trim() + if (!trimmed || isAutoModel(trimmed)) return false + if (trimmed.toLowerCase() === primaryModel.trim().toLowerCase()) return false + + const provider = findProviderFromModel(trimmed) + if (!provider) return false + + if (requiresProviderFamilyCredentials(trimmed)) { + return provider === findProviderFromModel(primaryModel.trim()) + } + return true +} + +/** + * Whether a fallback row must name a workspace variable for its key. + * + * A model that needs a key and shares the primary's provider reuses the + * block's own API Key field, so only a cross-provider fallback asks for one. + */ +export function fallbackRowNeedsApiKey(model: string, primaryModel: string): boolean { + const trimmed = model.trim() + if (!trimmed || !shouldRequireApiKeyForModel(trimmed)) return false + const provider = findProviderFromModel(trimmed) + return provider === null || provider !== findProviderFromModel(primaryModel.trim()) +} + +/** + * The values `model` accepts for `knob`, with the provider-decides entry first, + * exactly as the block's own field offers them. Null when the model lacks the + * knob or is not in the catalog, which is also how the block's field decides + * whether to render. + */ +export function getTuningOptionsForModel(model: string, knob: FallbackTuningKnob): string[] | null { + const trimmed = model.trim() + if (!trimmed) return null + const declared = + knob === 'reasoningEffort' + ? getReasoningEffortValuesForModel(trimmed) + : knob === 'thinkingLevel' + ? getThinkingLevelsForModel(trimmed) + : getVerbosityValuesForModel(trimmed) + if (!declared) return null + const sentinel = KNOB_SENTINEL[knob] + return [sentinel, ...declared.filter((value) => value !== sentinel)] +} + +/** + * Whether `value` can be sent to `model` for `knob`. Unset and the sentinel + * always can. Otherwise the model must declare it; a model the catalog does not + * know declares nothing, so anything passes through, as it does for the primary, + * while a catalogued model without the knob accepts nothing for it. + */ +export function isTuningValueValidForModel( + model: string, + knob: FallbackTuningKnob, + value: unknown +): boolean { + if (typeof value !== 'string') return value === undefined || value === null + const normalized = value.trim().toLowerCase() + if (!normalized || normalized === KNOB_SENTINEL[knob]) return true + const options = getTuningOptionsForModel(model, knob) + if (options === null) return !isKnownModelId(model.trim()) + return options.includes(normalized) +} + +/** + * The knobs a fallback row has to ask about: those the fallback model has that + * the primary's current setting cannot fill, either because the primary lacks + * the knob or because its value is not one the fallback declares. A value the + * fallback accepts is inherited silently, so a same-family row stays bare. + */ +export function getFallbackTuningKnobsToShow( + fallbackModel: string, + primaryModel: string, + primaryValues: Partial> +): FallbackTuningKnob[] { + return FALLBACK_TUNING_KNOBS.filter((knob) => { + if (getTuningOptionsForModel(fallbackModel, knob) === null) return false + if (getTuningOptionsForModel(primaryModel, knob) === null) return true + return !isTuningValueValidForModel(fallbackModel, knob, primaryValues[knob]) + }) +} + +export interface PrimaryTuningInputs extends Partial> { + temperature?: string | number + maxTokens?: string | number +} + +export interface ResolvedFallbackTuning extends Partial> { + temperature?: string | number + maxTokens?: string | number + /** Human-readable notes on every value that differs from the primary's, for the run log. */ + adjustments: string[] +} + +function clampToCap( + value: string | number | undefined, + cap: number | undefined +): string | number | undefined { + if (value === undefined || value === null || value === '' || cap === undefined) return value + const parsed = Number(value) + if (!Number.isFinite(parsed) || parsed <= cap) return value + return typeof value === 'number' ? cap : String(cap) +} + +/** + * The tuning a fallback candidate runs with. + * + * Graded knobs: the row's own value wins; otherwise the primary's value is + * carried over only when the fallback declares it, and dropped to the + * provider's default otherwise, which is what the row field exists to override. + * Temperature and max output tokens are caps in the primary's terms, so they + * are clamped to what the fallback allows rather than dropped: a low + * temperature chosen for repeatability must survive, and "no more than N" still + * holds under a smaller ceiling. A model the catalog does not know has no caps + * and no lists, so everything passes through unchanged. + */ +export function resolveFallbackTuning( + candidate: FallbackModelCandidate, + primaryModel: string, + primary: PrimaryTuningInputs +): ResolvedFallbackTuning { + const adjustments: string[] = [] + const resolved: ResolvedFallbackTuning = { adjustments } + + for (const knob of FALLBACK_TUNING_KNOBS) { + const own = candidate[knob] + if (own) { + resolved[knob] = own + if (own !== primary[knob]) adjustments.push(`${knob}: ${primary[knob] ?? 'unset'} -> ${own}`) + continue + } + const inherit = + getTuningOptionsForModel(primaryModel, knob) !== null && + isTuningValueValidForModel(candidate.model, knob, primary[knob]) + resolved[knob] = inherit ? primary[knob] : undefined + if (!inherit && primary[knob]) adjustments.push(`${knob}: ${primary[knob]} -> provider default`) + } + + resolved.temperature = clampToCap(primary.temperature, getMaxTemperature(candidate.model)) + if (resolved.temperature !== primary.temperature) { + adjustments.push(`temperature: ${primary.temperature} -> ${resolved.temperature}`) + } + resolved.maxTokens = clampToCap( + primary.maxTokens, + getModelCapabilities(candidate.model)?.maxOutputTokens + ) + if (resolved.maxTokens !== primary.maxTokens) { + adjustments.push(`maxTokens: ${primary.maxTokens} -> ${resolved.maxTokens}`) + } + + return resolved +} + +/** "2nd choice", "3rd choice", ... for the row at `index` (0-based) below the primary. */ +export function ordinalChoiceLabel(index: number): string { + const n = index + 2 + const mod100 = n % 100 + const suffix = + mod100 >= 11 && mod100 <= 13 + ? 'th' + : n % 10 === 1 + ? 'st' + : n % 10 === 2 + ? 'nd' + : n % 10 === 3 + ? 'rd' + : 'th' + return `${n}${suffix} choice` +} From 2b93caa352827425d2b3582a00520ac54d56f71a Mon Sep 17 00:00:00 2001 From: Marcus Chandra Date: Thu, 17 Sep 2026 22:52:09 -0700 Subject: [PATCH 03/14] feat(executor): walk fallback models when the agent's provider request fails One handler invocation tries the primary then each fallback in order, so block retry wraps the whole chain. Falling through is as indiscriminate as retry: only a stop or a non-retryable failure ends it early. Messages are built once; hydration is cached per provider; a fallback that is blacklisted, not permitted, or cannot take the attachments is skipped. The models that failed are recorded on the block log and rendered in the trace as a Fell back from row. --- .../components/trace-view/trace-view.tsx | 3 + .../handlers/agent/agent-handler.test.ts | 225 +++++++++++++- .../executor/handlers/agent/agent-handler.ts | 290 ++++++++++++++++-- apps/sim/executor/types.ts | 6 + .../execution/trace-spans/span-factory.ts | 1 + apps/sim/lib/logs/types.ts | 2 + 6 files changed, 500 insertions(+), 27 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx index e97cc186c3f..98aee8c718a 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx @@ -694,6 +694,9 @@ const TraceDetailPane = memo(function TraceDetailPane({ span }: { span: TraceSpa if (span.tries !== undefined) metaEntries.push({ label: 'Tries', value: String(span.tries) }) if (span.provider) metaEntries.push({ label: 'Provider', value: span.provider }) if (span.model) metaEntries.push({ label: 'Model', value: span.model }) + if (span.modelFallbacks?.length) { + metaEntries.push({ label: 'Fell back from', value: span.modelFallbacks.join(', ') }) + } if (span.finishReason) metaEntries.push({ label: 'Finish reason', value: span.finishReason }) const ttftFormatted = formatTtft(span.ttft) if (ttftFormatted) metaEntries.push({ label: 'TTFT', value: ttftFormatted }) diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 85d14bd6e27..c3dcb49d035 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -28,7 +28,7 @@ import type { ExecutionContext, StreamingExecution } from '@/executor/types' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { executeProviderRequest } from '@/providers' import { installStreamingCostPolicy } from '@/providers/cost-policy' -import { SIM_AUTO_MODEL_ID } from '@/providers/models' +import { getModelCapabilities, SIM_AUTO_MODEL_ID } from '@/providers/models' import { getProviderToolInputProvenance, getProviderToolModelInputRegistry, @@ -495,6 +495,229 @@ describe('AgentBlockHandler', () => { }) }) + describe('model fallback', () => { + const baseInputs = { + model: 'gpt-4o', + userPrompt: 'Hello', + apiKey: 'primary-key', + temperature: 0.4, + previousInteractionId: 'interaction-1', + } + + const providerFor = (model: string) => { + if (model.startsWith('gpt')) return 'openai' + if (model.startsWith('claude')) return 'anthropic' + if (model === 'blacklisted-model') throw new Error('provider blacklisted') + return 'openai' + } + + beforeEach(() => { + mockGetProviderFromModel.mockImplementation(providerFor) + }) + + it('never touches the fallbacks when the primary answers', async () => { + await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + expect(mockExecuteProviderRequest.mock.calls[0][1].model).toBe('gpt-4o') + expect(mockAgentLogger.warn).not.toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.anything() + ) + expect(mockContext.blockLogs).toEqual([]) + }) + + it('falls through to the next model with the same request and no primary-only fields', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce({ + content: 'from fallback', + model: 'claude-sonnet-5', + tokens: { input: 1, output: 1, total: 2 }, + toolCalls: [], + cost: 0.001, + timing: { total: 10 }, + }) + const blockLog = { + blockId: mockBlock.id, + startedAt: new Date().toISOString(), + endedAt: '', + durationMs: 0, + success: false, + executionOrder: 1, + } + const ctx = { ...mockContext, blockLogs: [blockLog] } + + const result = await handler.execute(ctx, mockBlock, { + ...baseInputs, + fallbackModels: [{ id: 'row-1', model: 'claude-sonnet-5' }], + }) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + const [primaryProvider, primaryRequest] = mockExecuteProviderRequest.mock.calls[0] + const [fallbackProvider, fallbackRequest] = mockExecuteProviderRequest.mock.calls[1] + expect(primaryProvider).toBe('openai') + expect(fallbackProvider).toBe('anthropic') + expect(fallbackRequest.model).toBe('claude-sonnet-5') + expect(fallbackRequest.messages).toEqual(primaryRequest.messages) + expect(fallbackRequest.temperature).toBe(primaryRequest.temperature) + expect(primaryRequest.previousInteractionId).toBe('interaction-1') + expect(fallbackRequest.previousInteractionId).toBeUndefined() + expect((result as { model: string }).model).toBe('claude-sonnet-5') + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.objectContaining({ failedModel: 'gpt-4o', nextModel: 'claude-sonnet-5' }) + ) + expect(blockLog).toMatchObject({ modelFallbacks: ['gpt-4o'] }) + }) + + it('gives a fallback its own key, the block key on the same provider, and nothing otherwise', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('one')) + .mockRejectedValueOnce(new Error('two')) + .mockRejectedValueOnce(new Error('three')) + .mockResolvedValueOnce({ + content: 'ok', + model: 'gpt-4o-mini', + tokens: { input: 1, output: 1, total: 2 }, + toolCalls: [], + cost: 0, + timing: { total: 1 }, + }) + + await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [ + { model: 'claude-sonnet-5', apiKey: '{{ANTHROPIC_KEY}}' }, + { model: 'claude-haiku-5' }, + { model: 'gpt-4o-mini' }, + ], + }) + + const keys = mockExecuteProviderRequest.mock.calls.map(([, request]) => request.apiKey) + expect(keys).toEqual(['primary-key', '{{ANTHROPIC_KEY}}', undefined, 'primary-key']) + }) + + it('re-resolves tuning for the fallback: row value wins, caps clamp, undeclared values drop', async () => { + const fallbackCap = getModelCapabilities('gpt-5.4-mini')?.maxOutputTokens + expect(fallbackCap).toEqual(expect.any(Number)) + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('down')).mockResolvedValueOnce({ + content: 'ok', + model: 'gpt-5.4-mini', + tokens: { input: 1, output: 1, total: 2 }, + toolCalls: [], + cost: 0, + timing: { total: 1 }, + }) + + await handler.execute(mockContext, mockBlock, { + ...baseInputs, + model: 'claude-sonnet-5', + thinkingLevel: 'high', + temperature: 0.9, + maxTokens: (fallbackCap as number) + 5000, + fallbackModels: [{ model: 'gpt-5.4-mini', reasoningEffort: 'low' }], + }) + + const [, primaryRequest] = mockExecuteProviderRequest.mock.calls[0] + const [, fallbackRequest] = mockExecuteProviderRequest.mock.calls[1] + expect(primaryRequest.thinkingLevel).toBe('high') + expect(primaryRequest.maxTokens).toBe((fallbackCap as number) + 5000) + expect(fallbackRequest.reasoningEffort).toBe('low') + expect(fallbackRequest.thinkingLevel).toBeUndefined() + expect(fallbackRequest.temperature).toBe(0.9) + expect(fallbackRequest.maxTokens).toBe(fallbackCap) + expect(mockAgentLogger.info).toHaveBeenCalledWith( + 'Fallback model tuning adjusted', + expect.objectContaining({ model: 'gpt-5.4-mini' }) + ) + }) + + it('rethrows the last attempted model error unchanged when every model fails', async () => { + const first = new Error('primary down') + const last = new Error('fallback down') + mockExecuteProviderRequest.mockRejectedValueOnce(first).mockRejectedValueOnce(last) + const blockLog = { + blockId: mockBlock.id, + startedAt: '', + endedAt: '', + durationMs: 0, + success: false, + executionOrder: 1, + } + + await expect( + handler.execute({ ...mockContext, blockLogs: [blockLog] }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toBe(last) + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + expect(blockLog).toMatchObject({ modelFallbacks: ['gpt-4o'] }) + }) + + it('skips sim-auto, duplicates, the primary itself, and unusable providers', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('down')).mockResolvedValueOnce({ + content: 'ok', + model: 'claude-sonnet-5', + tokens: { input: 1, output: 1, total: 2 }, + toolCalls: [], + cost: 0, + timing: { total: 1 }, + }) + + await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [ + { model: 'sim-auto' }, + { model: 'GPT-4o' }, + { model: 'blacklisted-model' }, + { model: 'claude-sonnet-5' }, + { model: 'claude-sonnet-5' }, + ], + }) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + expect(mockExecuteProviderRequest.mock.calls[1][1].model).toBe('claude-sonnet-5') + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Fallback model unusable; skipping', + expect.objectContaining({ model: 'blacklisted-model' }) + ) + }) + + it('does not fall back after a stop', async () => { + const controller = new AbortController() + mockExecuteProviderRequest.mockImplementationOnce(async () => { + controller.abort() + throw new Error('Provider request timed out') + }) + + await expect( + handler.execute({ ...mockContext, abortSignal: controller.signal }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toThrow('timed out') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + + it('does not fall back on an explicitly non-retryable failure', async () => { + const error = Object.assign(new Error('permanent'), { retryable: false }) + mockExecuteProviderRequest.mockRejectedValueOnce(error) + + await expect( + handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toBe(error) + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + }) + describe('execute', () => { it('should execute a basic agent block request', async () => { const inputs = { diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index d96aeed8f74..530e24c3015 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' +import { getErrorMessage, toError } from '@sim/utils/errors' import { isPlainRecord, omit } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import { normalizeStringRecord, normalizeWorkflowVariables } from '@/lib/core/utils/records' @@ -39,6 +39,10 @@ import { selectModelBoundFileInputPaths, } from '@/lib/uploads/utils/model-input' import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' +import { + normalizeFallbackModels, + resolveFallbackTuning, +} from '@/lib/workflows/blocks/fallback-models' import { resolveCustomBlockToolBinding } from '@/lib/workflows/custom-blocks/operations' import { getAgentToolUsageControlMode, @@ -54,6 +58,7 @@ import { validateModelProvider, } from '@/ee/access-control/utils/permission-check' import { AGENT, BlockType, DEFAULTS, stripCustomToolPrefix } from '@/executor/constants' +import { isRetryableBlockError } from '@/executor/execution/block-retry' import { memoryService } from '@/executor/handlers/agent/memory' import { buildLoadSkillTool, @@ -131,6 +136,7 @@ const AGENT_RAW_PROVIDER_ERROR_INPUT_PATHS: readonly ResolvedSecretInputPath[] = ['thinkingLevel'], ['promptCaching'], ['previousInteractionId'], + ['fallbackModels'], ] interface IndexedToolInput { @@ -138,6 +144,28 @@ interface IndexedToolInput { toolIndex: number } +/** One model in the order the block tries them; the primary carries the block's own key. */ +interface ModelCandidate { + model: string + apiKey?: string + isPrimary: boolean +} + +interface ExecuteAcrossModelsConfig { + candidates: ModelCandidate[] + primaryModel: string + primaryProviderId: string + messages: Message[] | undefined + fileProjection: ReturnType + modelInputs: AgentInputs + formattedTools: any[] + responseFormat: any + streaming: boolean + settledInputRegistry: ResolvedSecretTraceRegistry | undefined + resultRegistry: ResolvedSecretTraceRegistry | undefined + providerErrorRegistry: ResolvedSecretTraceRegistry | undefined +} + interface FormattedAgentTools { tools: ProviderToolConfig[] inputProvenance: Map> @@ -391,24 +419,6 @@ export class AgentBlockHandler implements BlockHandler { skillMetadata, fileProjection ) - const messagesWithFiles = await this.hydrateMessageFilesForProvider( - ctx, - messagesWithInputFiles, - providerId, - fileProjection.projectedNameByFile, - fileProjection.modelBoundInputPaths - ) - - const providerRequest = this.buildProviderRequest({ - ctx, - providerId, - model, - messages: messagesWithFiles, - inputs: modelInputs, - formattedTools: formatted.tools, - responseFormat, - streaming: streamingConfig.shouldUseStreaming ?? false, - }) settlePrivateAgentSelectors() @@ -427,21 +437,39 @@ export class AgentBlockHandler implements BlockHandler { }) } } - const result = await this.executeProviderRequest( - ctx, - providerRequest, - block, + + const candidates: ModelCandidate[] = [ + { model, apiKey: modelInputs.apiKey, isPrimary: true }, + ...normalizeFallbackModels(filteredInputs.fallbackModels) + .filter((candidate) => candidate.model.toLowerCase() !== model.toLowerCase()) + .map((candidate) => ({ ...candidate, isPrimary: false })), + ] + const { result, servedModel, failedModels } = await this.executeAcrossModels(ctx, block, { + candidates, + primaryModel: model, + primaryProviderId: providerId, + messages: messagesWithInputFiles, + fileProjection, + modelInputs, + formattedTools: formatted.tools, responseFormat, + streaming: streamingConfig.shouldUseStreaming ?? false, + settledInputRegistry, resultRegistry, - providerErrorRegistry - ) + providerErrorRegistry, + }) + if (failedModels.length > 0) this.recordModelFallbacks(ctx, block, failedModels) if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry if (autoRouting && autoRouting.billableRoutingCost > 0) { this.applyRoutingCost(result, autoRouting.billableRoutingCost) } - if (autoRouting) { + /** + * A fallback the builder named explicitly is not a pool model, so it keeps + * its own name; only the routed pool model hides behind the auto label. + */ + if (autoRouting && servedModel === model) { this.applyAutoModelLabel(result, model) } @@ -2302,6 +2330,216 @@ export class AgentBlockHandler implements BlockHandler { return paths } + /** + * Runs the provider request against each candidate in order until one answers. + * + * Which model serves the request is decided here, inside one handler + * invocation. How many invocations the block gets is the executor's retry + * policy, which wraps this whole chain: with retry on, every try walks the + * chain again from the primary. + * + * Falling through is deliberately as indiscriminate as block retry + * (`isRetryableBlockError`): a provider error carries no status, so an + * overloaded upstream cannot be told from any other failure, and a builder who + * lists fallbacks wants the block to answer. Only a stop and an explicitly + * non-retryable failure end the chain early. The abort signal is checked + * before the error itself because `handleExecutionError` rewrites a timeout + * into a plain `Error` without a cause, which the predicate would then treat + * as replayable. + * + * Messages are built once by the caller, since building them appends to + * memory. Hydration runs per provider, because attachment support and the + * inline budget differ, and is cached so two candidates on one provider do + * not download the same files twice. A fallback whose provider is + * blacklisted, not permitted, or cannot take the attachments is skipped + * rather than counted as a failed try. + * + * When every candidate fails, the last attempted candidate's error is thrown + * exactly as it escaped `executeProviderRequest`, so error ports and the + * block-level error handling see the shapes they see today. + */ + private async executeAcrossModels( + ctx: ExecutionContext, + block: SerializedBlock, + config: ExecuteAcrossModelsConfig + ): Promise<{ + result: BlockOutput | StreamingExecution + servedModel: string + failedModels: string[] + }> { + const hydratedByProvider = new Map() + const failedModels: string[] = [] + let lastError: unknown + + for (let index = 0; index < config.candidates.length; index++) { + const candidate = config.candidates[index] + const hasNext = index < config.candidates.length - 1 + + let candidateProviderId: string + if (candidate.isPrimary) { + candidateProviderId = config.primaryProviderId + } else { + try { + candidateProviderId = getProviderFromModel(candidate.model) + await validateModelProvider(ctx.userId, ctx.workspaceId, candidate.model, ctx) + } catch (error) { + logger.warn( + 'Fallback model unusable; skipping', + projectAgentDiagnosticMetadata( + ctx, + { blockId: block.id, model: candidate.model, error: getErrorMessage(error) }, + { blockId: block.id } + ) + ) + continue + } + } + + let messages: Message[] | undefined + if (hydratedByProvider.has(candidateProviderId)) { + messages = hydratedByProvider.get(candidateProviderId) + } else { + try { + messages = await this.hydrateMessageFilesForProvider( + ctx, + config.messages, + candidateProviderId, + config.fileProjection.projectedNameByFile, + config.fileProjection.modelBoundInputPaths + ) + } catch (error) { + if (candidate.isPrimary) throw error + logger.warn( + 'Fallback model cannot take the attached files; skipping', + projectAgentDiagnosticMetadata( + ctx, + { blockId: block.id, model: candidate.model, error: getErrorMessage(error) }, + { blockId: block.id } + ) + ) + continue + } + hydratedByProvider.set(candidateProviderId, messages) + } + + /** + * A fallback's own key wins; without one it may reuse the block's key only + * on the primary's provider. Otherwise the provider layer resolves BYOK or + * the platform key, or reports that a key is required, which counts as + * this candidate failing. A previous interaction id belongs to the primary's + * provider alone. Tuning is re-resolved against the fallback's own + * capabilities so the request is one its provider accepts. + */ + let inputs: AgentInputs = config.modelInputs + if (!candidate.isPrimary) { + const { adjustments, ...tuning } = resolveFallbackTuning( + candidate, + config.primaryModel, + config.modelInputs + ) + inputs = { + ...config.modelInputs, + apiKey: + candidate.apiKey ?? + (candidateProviderId === config.primaryProviderId + ? config.modelInputs.apiKey + : undefined), + previousInteractionId: undefined, + ...tuning, + } + if (adjustments.length > 0) { + logger.info( + 'Fallback model tuning adjusted', + projectAgentDiagnosticMetadata( + ctx, + { blockId: block.id, model: candidate.model, adjustments }, + { blockId: block.id, adjustmentCount: adjustments.length } + ) + ) + } + } + + const providerRequest = this.buildProviderRequest({ + ctx, + providerId: candidateProviderId, + model: candidate.model, + messages, + inputs, + formattedTools: config.formattedTools, + responseFormat: config.responseFormat, + streaming: config.streaming, + }) + + try { + const result = await this.executeProviderRequest( + ctx, + providerRequest, + block, + config.responseFormat, + config.resultRegistry, + config.providerErrorRegistry + ) + return { result, servedModel: candidate.model, failedModels } + } catch (error) { + lastError = error + failedModels.push(candidate.model) + if (!hasNext || ctx.abortSignal?.aborted || !isRetryableBlockError(error)) { + this.recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) + throw error + } + + /** + * `executeProviderRequest` swapped both registries for its error + * projection; the next candidate starts from the settled inputs again. + */ + ctx.errorResolvedSecretTraceRegistry = config.providerErrorRegistry + ctx.resolvedSecretTraceRegistry = config.settledInputRegistry + + logger.warn( + 'Agent model failed; trying fallback', + projectAgentDiagnosticMetadata( + ctx, + { + blockId: block.id, + failedModel: candidate.model, + nextModel: config.candidates[index + 1].model, + attempt: index + 1, + error: getErrorMessage(error), + }, + { blockId: block.id, attempt: index + 1 } + ) + ) + } + } + + /** Reached only when every candidate after the last failure was skipped. */ + this.recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) + throw lastError + } + + /** + * Writes the models that failed onto the block's open log entry so the trace + * can show them beside the model that answered. Handlers get no log handle; + * the executor pushes the entry before running the handler with `endedAt` + * still empty, which is what tells it apart from earlier runs of the same + * block in a loop or an earlier retry. + */ + private recordModelFallbacks( + ctx: ExecutionContext, + block: SerializedBlock, + failedModels: string[] + ): void { + if (failedModels.length === 0) return + const logs = ctx.blockLogs ?? [] + for (let index = logs.length - 1; index >= 0; index--) { + const entry = logs[index] + if (entry.blockId === block.id && entry.endedAt === '') { + entry.modelFallbacks = [...failedModels] + return + } + } + } + private buildProviderRequest(config: { ctx: ExecutionContext providerId: string diff --git a/apps/sim/executor/types.ts b/apps/sim/executor/types.ts index 164d3de0874..073ac1e2bb7 100644 --- a/apps/sim/executor/types.ts +++ b/apps/sim/executor/types.ts @@ -286,6 +286,12 @@ export interface BlockLog { errorHandled?: boolean /** Total handler tries, present only when the block retried at least once. */ tries?: number + /** + * Models that failed before the one that answered, in the order tried. + * Present only when an Agent block fell back at least once; under retry the + * last try's chain wins, matching `tries`. + */ + modelFallbacks?: string[] loopId?: string parallelId?: string iterationIndex?: number diff --git a/apps/sim/lib/logs/execution/trace-spans/span-factory.ts b/apps/sim/lib/logs/execution/trace-spans/span-factory.ts index e1b4a837c8d..7abe0bcbc71 100644 --- a/apps/sim/lib/logs/execution/trace-spans/span-factory.ts +++ b/apps/sim/lib/logs/execution/trace-spans/span-factory.ts @@ -115,6 +115,7 @@ function createBaseSpan(log: ValidBlockLog): TraceSpan { ...(log.childTraceDisabled ? { childTraceDisabled: true } : {}), ...(log.errorHandled && { errorHandled: true }), ...(log.tries !== undefined && { tries: log.tries }), + ...(log.modelFallbacks?.length && { modelFallbacks: log.modelFallbacks }), ...(log.loopId && { loopId: log.loopId }), ...(log.parallelId && { parallelId: log.parallelId }), ...(log.iterationIndex !== undefined && { iterationIndex: log.iterationIndex }), diff --git a/apps/sim/lib/logs/types.ts b/apps/sim/lib/logs/types.ts index 80d9f7542a1..19c9f4a5c1b 100644 --- a/apps/sim/lib/logs/types.ts +++ b/apps/sim/lib/logs/types.ts @@ -285,6 +285,8 @@ export interface TraceSpan { errorHandled?: boolean /** Total handler tries, present only when the block retried at least once. */ tries?: number + /** Models that failed before the one that answered; present only when the block fell back. */ + modelFallbacks?: string[] tokens?: TokenInfo relativeStartMs?: number blockId?: string From b6cc5e093178811442b9c7a88cd06e905c0842e4 Mon Sep 17 00:00:00 2001 From: Marcus Chandra Date: Thu, 17 Sep 2026 22:52:10 -0700 Subject: [PATCH 04/14] feat(workflows): validate, sanitize, index, and label fallback model rows Copilot and YAML writes are refused when a row names an unknown model, sim-auto, a raw key instead of a {{ENV_VAR}} reference, or a tuning value the model does not declare. Export keeps only reference keys, search never rewrites a row key, and the canvas card summarizes the models. --- .../workflow-block/workflow-block.tsx | 8 ++ .../components/block/block.tsx | 3 + .../credentials/credential-extractor.test.ts | 14 +++ .../credentials/credential-extractor.ts | 23 ++++ .../lib/workflows/editing/validation.test.ts | 69 +++++++++++ apps/sim/lib/workflows/editing/validation.ts | 108 ++++++++++++++++++ .../workflows/search-replace/indexer.test.ts | 19 +++ .../lib/workflows/search-replace/indexer.ts | 4 + .../lib/workflows/subblocks/display.test.ts | 21 ++++ apps/sim/lib/workflows/subblocks/display.ts | 23 ++++ 10 files changed, 292 insertions(+) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx index badc3293c31..1d7eea3beca 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx @@ -61,6 +61,7 @@ import { getDisplayValue, hasDisplayableRowValue, resolveDropdownLabel, + resolveFallbackModelsLabel, resolveFilterFieldLabel, resolveFolderPathLabel, resolveSandboxLabel, @@ -158,6 +159,7 @@ const SUBBLOCK_META_ICONS_BY_TYPE: Record = { 'messages-input': MessageSquareText, 'tool-input': Wrench, 'skill-input': Sparkles, + 'model-fallback-list': ArrowLeftRight, 'oauth-input': Key, switch: ToggleLeft, 'file-upload': Paperclip, @@ -574,6 +576,11 @@ const SubBlockRow = memo(function SubBlockRow({ [subBlock, rawValue, workspaceSkills] ) + const fallbackModelsDisplayValue = useMemo( + () => resolveFallbackModelsLabel(subBlock, rawValue), + [subBlock, rawValue] + ) + /** * Hydrates the Function block's sandbox id to its name. Deliberately scoped to * the sandbox row: this row is memoized per subblock, and the shared list query @@ -605,6 +612,7 @@ const SubBlockRow = memo(function SubBlockRow({ filterDisplayValue || toolsDisplayValue || skillsDisplayValue || + fallbackModelsDisplayValue || sandboxDisplayValue || knowledgeBaseDisplayName || workflowSelectionName || diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx index b3935655903..04dfea0021a 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx @@ -23,6 +23,7 @@ import { getDisplayValue, hasDisplayableRowValue, resolveDropdownLabel, + resolveFallbackModelsLabel, resolveFolderPathLabel, resolveSkillsLabel, resolveToolsLabel, @@ -151,6 +152,7 @@ function resolvePreviewDisplayValue( // schema/registry fallbacks rather than the API. const toolsDisplay = resolveToolsLabel(subBlock, rawValue, []) const skillsDisplay = resolveSkillsLabel(subBlock, rawValue, []) + const fallbackModelsDisplay = resolveFallbackModelsLabel(subBlock, rawValue) const workflowName = resolveWorkflowSelectionLabel(subBlock, rawValue, workflowLookup) const workflowMultiSelectionNames = resolveWorkflowMultiSelectLabel( subBlock, @@ -165,6 +167,7 @@ function resolvePreviewDisplayValue( variablesDisplay || toolsDisplay || skillsDisplay || + fallbackModelsDisplay || workflowName || workflowMultiSelectionNames || /* diff --git a/apps/sim/lib/workflows/credentials/credential-extractor.test.ts b/apps/sim/lib/workflows/credentials/credential-extractor.test.ts index ccdd1be166d..146d1fb07cd 100644 --- a/apps/sim/lib/workflows/credentials/credential-extractor.test.ts +++ b/apps/sim/lib/workflows/credentials/credential-extractor.test.ts @@ -102,6 +102,20 @@ describe('export sanitizer resource coverage', () => { expect(sanitizedValue('oauth-input', 'cred-123')).toBeNull() }) + it('keeps fallback models and only env-var-referenced row keys', () => { + expect( + sanitizedValue('model-fallback-list', [ + { id: 'a', model: 'gpt-5' }, + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { id: 'c', model: 'openrouter/y', apiKey: 'sk-raw-secret' }, + ]) + ).toEqual([ + { id: 'a', model: 'gpt-5' }, + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { id: 'c', model: 'openrouter/y' }, + ]) + }) + it('leaves an ordinary field untouched', () => { expect(sanitizedValue('short-input', 'plain text')).toBe('plain text') }) diff --git a/apps/sim/lib/workflows/credentials/credential-extractor.ts b/apps/sim/lib/workflows/credentials/credential-extractor.ts index d9e3aa86fa7..49b02a80b5a 100644 --- a/apps/sim/lib/workflows/credentials/credential-extractor.ts +++ b/apps/sim/lib/workflows/credentials/credential-extractor.ts @@ -170,6 +170,26 @@ function isEnvironmentVariableReference(value: unknown): value is string { return typeof value === 'string' && value.startsWith('{{') && value.endsWith('}}') } +/** + * Keeps a fallback list's models and drops every row key that is not a whole + * environment-variable reference. The editor only ever writes references, but the + * realtime subblock-value op runs no validator, so this is what guarantees a raw + * key can never leave the workspace in an export or template. + */ +function sanitizeFallbackModelsValue( + value: unknown, + options: WorkflowSanitizationOptions +): unknown { + if (!Array.isArray(value)) return value + return value.map((row) => { + if (!row || typeof row !== 'object' || Array.isArray(row)) return row + const { apiKey, ...rest } = row as Record + return options.preserveEnvVars && isEnvironmentVariableReference(apiKey) + ? { ...rest, apiKey } + : rest + }) +} + /** * Sanitizes nested tool parameters using the same codecs as workflow search and fork remapping. * Only parameters resolved from a registered definition retain non-sensitive values. Custom, MCP, @@ -245,6 +265,9 @@ function sanitizeConfiguredSubBlockValue( if (config.password === true) { return options.preserveEnvVars && isEnvironmentVariableReference(value) ? value : null } + if (config.type === 'model-fallback-list') { + return sanitizeFallbackModelsValue(value, options) + } if ( WORKSPACE_SPECIFIC_TYPES.has(config.type) || WORKSPACE_SPECIFIC_FIELDS.has(config.id) || diff --git a/apps/sim/lib/workflows/editing/validation.test.ts b/apps/sim/lib/workflows/editing/validation.test.ts index 1f192cb1721..9d1a5c355be 100644 --- a/apps/sim/lib/workflows/editing/validation.test.ts +++ b/apps/sim/lib/workflows/editing/validation.test.ts @@ -3,6 +3,8 @@ */ import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getTuningOptionsForModel } from '@/lib/workflows/blocks/fallback-models' +import { getThinkingLevelsForModel } from '@/providers/models' import { normalizeConditionRouterIds } from './builders' const { @@ -338,6 +340,73 @@ describe('validateInputsForBlock', () => { ).toBe(false) }) + describe('model-fallback-list', () => { + const config = { id: 'fallbackModels', type: 'model-fallback-list' as const } + const validate = (value: unknown) => + validateValueForSubBlockType(config, value, 'fallbackModels', 'agent', 'agent-1') + + it('accepts known models with env-var-referenced keys and fills missing row ids', () => { + const result = validate([ + { id: 'row-1', model: ' claude-sonnet-5 ' }, + { model: 'openrouter/anthropic/claude', apiKey: '{{OPENROUTER_API_KEY}}' }, + ]) + expect(result.valid).toBe(true) + const rows = (result as { value: Array<{ id: string; model: string; apiKey?: string }> }) + .value + expect(rows[0]).toEqual({ id: 'row-1', model: 'claude-sonnet-5' }) + expect(rows[1].id).toEqual(expect.any(String)) + expect(rows[1]).toMatchObject({ + model: 'openrouter/anthropic/claude', + apiKey: '{{OPENROUTER_API_KEY}}', + }) + }) + + it('refuses a raw key rather than repairing it', () => { + const result = validate([{ model: 'claude-sonnet-5', apiKey: 'sk-live-raw' }]) + expect(result.valid).toBe(false) + expect((result as { error: { error: string } }).error.error).toContain( + 'apiKey must be a whole {{ENV_VAR}} reference' + ) + }) + + it('refuses sim-auto, unknown models, missing models, and non-arrays', () => { + expect(validate([{ model: 'sim-auto' }]).valid).toBe(false) + expect(validate([{ model: 'definitely-not-a-model-9000' }]).valid).toBe(false) + expect(validate([{ apiKey: '{{KEY}}' }]).valid).toBe(false) + expect(validate({ model: 'claude-sonnet-5' }).valid).toBe(false) + }) + + it('accepts a row tuning value the model declares and refuses one it does not', () => { + const levels = getThinkingLevelsForModel('claude-sonnet-5') + expect(levels?.length).toBeGreaterThan(0) + const ok = validate([ + { model: 'claude-sonnet-5', thinkingLevel: ` ${levels![0].toUpperCase()} ` }, + ]) + expect(ok.valid).toBe(true) + expect((ok as { value: Array<{ thinkingLevel?: string }> }).value[0].thinkingLevel).toBe( + levels![0] + ) + + const bad = validate([{ model: 'claude-sonnet-5', thinkingLevel: 'bogus' }]) + expect(bad.valid).toBe(false) + expect((bad as { error: { error: string } }).error.error).toContain('thinking level option') + + const undeclared = (['reasoningEffort', 'verbosity', 'thinkingLevel'] as const).find( + (knob) => getTuningOptionsForModel('claude-sonnet-5', knob) === null + ) + expect(undeclared).toBeDefined() + const missingKnob = validate([{ model: 'claude-sonnet-5', [undeclared as string]: 'low' }]) + expect(missingKnob.valid).toBe(false) + }) + + it('refuses more rows than the cap', () => { + const rows = Array.from({ length: 6 }, () => ({ model: 'claude-sonnet-5' })) + const result = validate(rows) + expect(result.valid).toBe(false) + expect((result as { error: { error: string } }).error.error).toContain('at most 5') + }) + }) + it('accepts condition-input arrays with arbitrary item ids', () => { const result = validateInputsForBlock( 'condition', diff --git a/apps/sim/lib/workflows/editing/validation.ts b/apps/sim/lib/workflows/editing/validation.ts index e02ca16ef02..75318f6460c 100644 --- a/apps/sim/lib/workflows/editing/validation.ts +++ b/apps/sim/lib/workflows/editing/validation.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { generateShortId } from '@sim/utils/id' import { omit } from '@sim/utils/object' import { isHosted as isHostedDeployment } from '@/lib/core/config/env-flags' import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' @@ -8,6 +9,14 @@ import { MCP_SERVER_ADVANCED_TOOL_TYPE } from '@/lib/mcp/shared' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' import { resolveAccessControlBlockType } from '@/lib/permission-groups/integration-allowlist' +import { + FALLBACK_TUNING_KNOBS, + FALLBACK_TUNING_LABELS, + getTuningOptionsForModel, + isTuningValueValidForModel, + isWholeEnvVarReference, + MAX_FALLBACK_MODELS, +} from '@/lib/workflows/blocks/fallback-models' import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' import { validateSelectorIds } from '@/lib/workflows/editing/selector-validator' import { containsReference } from '@/lib/workflows/sanitization/references' @@ -360,6 +369,50 @@ function validateAgentToolEntry(item: any, index: number): string | null { * Skills are a SEPARATE array from tools; each entry references a workspace or * builtin skill by `skillId`. Returns an error string or null when valid. */ +/** + * Validates one fallback-model row. Returns an error string or null when valid. + * + * Refuses rather than repairs: an unknown model, sim-auto, or a raw key is an + * authoring mistake the caller must see. A missing React-key `id` is the one + * thing filled in, since it carries no meaning. + */ +function validateFallbackModelEntry(item: any, index: number): string | null { + const where = `fallbackModels[${index}]` + if (item === null || typeof item !== 'object' || Array.isArray(item)) { + return `${where} must be an object { model, apiKey? }` + } + const model = typeof item.model === 'string' ? item.model.trim() : '' + if (model === '') { + return `${where} is missing a string "model"` + } + if (isAutoModel(model)) { + return `${where}: sim-auto cannot be a fallback model; it already routes and falls back on its own` + } + if (!isKnownModelId(model) && !isCustomModelId(model)) { + const suggestions = suggestModelIdsForUnknownModel(model) + const suggestionText = + suggestions.length > 0 ? ` Valid options include: ${suggestions.join(', ')}.` : '' + return `${where}: unknown model id "${model}".${suggestionText}` + } + if (item.apiKey !== undefined && item.apiKey !== null && item.apiKey !== '') { + if (!isWholeEnvVarReference(item.apiKey)) { + return `${where}.apiKey must be a whole {{ENV_VAR}} reference; put the key in an environment variable instead of pasting it` + } + } + for (const knob of FALLBACK_TUNING_KNOBS) { + const value = item[knob] + if (value === undefined || value === null || value === '') continue + if (typeof value !== 'string' || !isTuningValueValidForModel(model, knob, value)) { + const options = getTuningOptionsForModel(model, knob) + const hint = options + ? ` Valid options: ${options.join(', ')}.` + : ` ${model} has no such setting.` + return `${where}.${knob}: "${String(value)}" is not a ${FALLBACK_TUNING_LABELS[knob].toLowerCase()} option for ${model}.${hint}` + } + } + return null +} + function validateAgentSkillEntry(item: any, index: number): string | null { const where = `skills[${index}]` if (item === null || typeof item !== 'object' || Array.isArray(item)) { @@ -581,6 +634,61 @@ export function validateValueForSubBlockType( return { valid: true, value } } + case 'model-fallback-list': { + if (!Array.isArray(value)) { + return { + valid: false, + error: { + blockId, + blockType, + field: fieldName, + value, + error: `Invalid model-fallback-list value for field "${fieldName}" - expected an array of { model, apiKey? } objects`, + }, + } + } + if (value.length > MAX_FALLBACK_MODELS) { + return { + valid: false, + error: { + blockId, + blockType, + field: fieldName, + value, + error: `"${fieldName}" allows at most ${MAX_FALLBACK_MODELS} fallback models`, + }, + } + } + const fallbackErrors = value + .map((item, index) => validateFallbackModelEntry(item, index)) + .filter((err): err is string => err !== null) + if (fallbackErrors.length > 0) { + return { + valid: false, + error: { + blockId, + blockType, + field: fieldName, + value, + error: `Invalid fallback ${fallbackErrors.length === 1 ? 'entry' : 'entries'} in "${fieldName}": ${fallbackErrors.join('; ')}`, + }, + } + } + return { + valid: true, + value: value.map((item: Record & { model: string }) => ({ + id: typeof item.id === 'string' && item.id ? item.id : generateShortId(), + model: item.model.trim(), + ...(isWholeEnvVarReference(item.apiKey) ? { apiKey: item.apiKey.trim() } : {}), + ...Object.fromEntries( + FALLBACK_TUNING_KNOBS.filter( + (knob) => typeof item[knob] === 'string' && (item[knob] as string).trim() !== '' + ).map((knob) => [knob, (item[knob] as string).trim().toLowerCase()]) + ), + })), + } + } + case 'skill-input': { // Should be an array of skill reference objects ({ skillId, name? }) if (!Array.isArray(value)) { diff --git a/apps/sim/lib/workflows/search-replace/indexer.test.ts b/apps/sim/lib/workflows/search-replace/indexer.test.ts index e2581a05a3c..1ef6f8e6ecb 100644 --- a/apps/sim/lib/workflows/search-replace/indexer.test.ts +++ b/apps/sim/lib/workflows/search-replace/indexer.test.ts @@ -1384,6 +1384,11 @@ describe('indexWorkflowSearchMatches', () => { type: 'input-mapping', value: { childInput: 'mapped visible value' }, }, + fallbackModels: { + id: 'fallbackModels', + type: 'model-fallback-list', + value: [{ id: 'row-1', model: 'fallback-visible-model', apiKey: '{{HIDDEN_KEY_REF}}' }], + }, }, } const blockConfigs = { @@ -1396,6 +1401,7 @@ describe('indexWorkflowSearchMatches', () => { { id: 'skills', title: 'Skills', type: 'skill-input' }, { id: 'runAt', title: 'Run At', type: 'time-input' }, { id: 'mapping', title: 'Input Mapping', type: 'input-mapping' }, + { id: 'fallbackModels', title: 'Fallback models', type: 'model-fallback-list' }, ], }, } @@ -1430,7 +1436,20 @@ describe('indexWorkflowSearchMatches', () => { mode: 'text', blockConfigs, }).filter((match) => match.blockId === 'structured-1') + const fallbackMatches = indexWorkflowSearchMatches({ + workflow, + query: 'fallback-visible', + mode: 'text', + blockConfigs, + }).filter((match) => match.blockId === 'structured-1') + expect(fallbackMatches).toEqual([ + expect.objectContaining({ + subBlockId: 'fallbackModels', + valuePath: [0, 'model'], + searchText: 'fallback-visible-model', + }), + ]) expect(containsMatches).toEqual([ expect.objectContaining({ subBlockId: 'filters', diff --git a/apps/sim/lib/workflows/search-replace/indexer.ts b/apps/sim/lib/workflows/search-replace/indexer.ts index 128972236fd..c2f6880c816 100644 --- a/apps/sim/lib/workflows/search-replace/indexer.ts +++ b/apps/sim/lib/workflows/search-replace/indexer.ts @@ -212,6 +212,10 @@ function isSearchableLeafPath( if (mode === 'text' && subBlockType === 'messages-input' && lastSegment === 'role') { return false } + /** A fallback row's key is a `{{VAR}}` reference; rewriting it would turn it into a raw value. */ + if (mode === 'text' && subBlockType === 'model-fallback-list' && lastSegment === 'apiKey') { + return false + } if (mode === 'text' && subBlockType === 'tool-input') { if (TOOL_INPUT_TEXT_EXCLUDED_LEAF_KEYS.has(lastSegment)) return false if (lastSegment.endsWith('Id')) return false diff --git a/apps/sim/lib/workflows/subblocks/display.test.ts b/apps/sim/lib/workflows/subblocks/display.test.ts index 863765cd467..50cfbfa0095 100644 --- a/apps/sim/lib/workflows/subblocks/display.test.ts +++ b/apps/sim/lib/workflows/subblocks/display.test.ts @@ -14,6 +14,7 @@ vi.mock('@/blocks', () => ({ import { getDisplayValue, resolveDropdownLabel, + resolveFallbackModelsLabel, resolveFilterFieldLabel, resolveFolderPathLabel, resolveSandboxLabel, @@ -187,6 +188,26 @@ describe('resolveSkillsLabel', () => { }) }) +describe('resolveFallbackModelsLabel', () => { + const fallbackList = { id: 'fallbackModels', type: 'model-fallback-list' } as SubBlockConfig + + it('lists the models in order and never the row keys', () => { + expect( + resolveFallbackModelsLabel(fallbackList, [ + { id: 'a', model: 'gpt-5' }, + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { id: 'c', model: 'gemini-3.6-flash' }, + ]) + ).toBe('gpt-5, openrouter/x +1') + }) + + it('returns null for other subblocks and for an empty or model-less list', () => { + expect(resolveFallbackModelsLabel(skillInput, [{ model: 'gpt-5' }])).toBeNull() + expect(resolveFallbackModelsLabel(fallbackList, [])).toBeNull() + expect(resolveFallbackModelsLabel(fallbackList, [{ id: 'a', model: '' }])).toBeNull() + }) +}) + describe('resolveSandboxLabel', () => { const sandboxes = [{ id: '443f4934-26ab-44ab-8000-000000000000', name: 'Test' }] diff --git a/apps/sim/lib/workflows/subblocks/display.ts b/apps/sim/lib/workflows/subblocks/display.ts index b371fda035e..3d47d307a09 100644 --- a/apps/sim/lib/workflows/subblocks/display.ts +++ b/apps/sim/lib/workflows/subblocks/display.ts @@ -582,6 +582,29 @@ export function resolveSkillsLabel( return summarizeNames(names) } +/** + * Resolves a fallback-model list to its model ids, e.g. "gpt-5.6, gemini-3.6-flash +1". + * Returns null for other subblocks and for an empty list so callers fall through. + * Row keys are never shown. + */ +export function resolveFallbackModelsLabel( + subBlock: SubBlockConfig | undefined, + rawValue: unknown +): string | null { + if (subBlock?.type !== 'model-fallback-list') return null + if (!Array.isArray(rawValue) || rawValue.length === 0) return null + + const models = rawValue + .map((row: unknown) => { + if (!row || typeof row !== 'object') return null + const model = (row as { model?: unknown }).model + return typeof model === 'string' && model.trim() ? model.trim() : null + }) + .filter((model): model is string => !!model) + + return summarizeNames(models) +} + /** * Resolves the Function block's stored sandbox id to the sandbox name. * From 259e391b4574fb9b51784c8f56168f2475c4ead8 Mon Sep 17 00:00:00 2001 From: Marcus Chandra Date: Thu, 17 Sep 2026 22:52:10 -0700 Subject: [PATCH 05/14] feat(editor): fallback models list on the Agent block Ordered rows (2nd choice, 3rd choice, ...) directly above Retry on fail. Each row picks a model the block can supply credentials for, an env-var reference for its key when a different provider needs one, and a tuning value only for the knobs the primary's setting cannot fill. --- .../components/sub-block/components/index.ts | 1 + .../components/model-fallback-list/index.ts | 1 + .../model-fallback-list.test.tsx | 196 +++++++++ .../model-fallback-list.tsx | 377 ++++++++++++++++++ .../editor/components/sub-block/sub-block.tsx | 12 + 5 files changed, 587 insertions(+) create mode 100644 apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts index 921e0c15285..0ec29554681 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts @@ -18,6 +18,7 @@ export { LongInput } from './long-input' export { McpDynamicArgs } from './mcp-dynamic-args' export { McpServerSelector, McpToolSelector } from './mcp-server-modal' export { MessagesInput } from './messages-input' +export { ModelFallbackList } from './model-fallback-list' export { maskSecretText, PASSWORD_MASKED_SUBBLOCK_TYPES } from './password-mask' export { ResponseFormat } from './response' export { ScheduleInfo } from './schedule-info' diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/index.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/index.ts new file mode 100644 index 00000000000..630453e439a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/index.ts @@ -0,0 +1 @@ +export { ModelFallbackList } from './model-fallback-list' diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx new file mode 100644 index 00000000000..aa837b5775c --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx @@ -0,0 +1,196 @@ +/** + * @vitest-environment node + */ +import { renderToStaticMarkup } from 'react-dom/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { subBlockValues, mockSetValue } = vi.hoisted(() => ({ + subBlockValues: { + model: 'claude-sonnet-5' as string, + fallbackModels: [] as Array<{ + id: string + model: string + apiKey?: string + reasoningEffort?: string + }>, + }, + mockSetValue: vi.fn(), +})) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) + +vi.mock('@sim/emcn', () => ({ + Button: ({ + children, + disabled, + 'aria-label': ariaLabel, + }: { + children?: React.ReactNode + disabled?: boolean + 'aria-label'?: string + }) => ( + + ), + ChipCombobox: ({ + options, + value, + placeholder, + }: { + options: Array<{ value: string; label: string; disabled?: boolean }> + value?: string + placeholder?: string + }) => ( +
+ {options.map((option) => ( + + {option.label} + + ))} +
+ ), + Label: ({ children }: { children?: React.ReactNode }) => {children}, + Tooltip: { + Root: ({ children }: { children?: React.ReactNode }) => <>{children}, + Trigger: ({ children }: { children?: React.ReactNode }) => <>{children}, + Content: () => null, + }, +})) + +vi.mock('@sim/emcn/icons', () => ({ + ChevronDown: () => null, + ChevronUp: () => null, + Plus: () => null, + Trash: () => null, +})) + +vi.mock( + '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value', + () => ({ + useSubBlockValue: (_blockId: string, subBlockId: string) => [ + subBlockId === 'model' || subBlockId === 'fallbackModels' ? subBlockValues[subBlockId] : null, + mockSetValue, + ], + }) +) + +vi.mock('@/hooks/queries/environment', () => ({ + usePersonalEnvironment: () => ({ data: { PERSONAL_KEY: 'x' } }), + useWorkspaceEnvironment: () => ({ + data: { workspace: { OPENROUTER_API_KEY: 'x' }, personal: {}, conflicts: [] }, + }), +})) + +vi.mock('@/hooks/use-permission-config', () => ({ + usePermissionConfig: () => ({ isModelUsable: (model: string) => model !== 'denied-model' }), +})) + +vi.mock('@/hooks/use-settings-navigation', () => ({ + useSettingsNavigation: () => ({ navigateToSettings: vi.fn() }), +})) + +vi.mock('@/lib/credentials/client-state', () => ({ + writePendingCredentialCreateRequest: vi.fn(), +})) + +vi.mock('@/stores/providers/store', () => ({ + useProvidersStore: (selector: (state: { providers: object }) => unknown) => + selector({ providers: {} }), +})) + +vi.mock('@/blocks/utils', () => ({ + getModelOptions: () => [ + { id: 'claude-sonnet-5', label: 'claude-sonnet-5' }, + { id: 'gpt-5', label: 'gpt-5' }, + { id: 'denied-model', label: 'denied-model' }, + { id: 'openrouter/x', label: 'openrouter/x' }, + { id: 'sim-auto', label: 'Auto' }, + ], +})) + +vi.mock('@/lib/workflows/blocks/fallback-models', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + isViableFallbackModel: (model: string, primary: string) => + model !== 'sim-auto' && model !== primary, + fallbackRowNeedsApiKey: (model: string) => model.startsWith('openrouter/'), + getFallbackTuningKnobsToShow: (model: string) => (model === 'gpt-5' ? ['reasoningEffort'] : []), + getTuningOptionsForModel: (model: string, knob: string) => + model === 'gpt-5' && knob === 'reasoningEffort' ? ['auto', 'low', 'high'] : null, + } +}) + +import { ModelFallbackList } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list' + +function render() { + return renderToStaticMarkup( + + ) +} + +describe('ModelFallbackList', () => { + beforeEach(() => { + subBlockValues.model = 'claude-sonnet-5' + subBlockValues.fallbackModels = [] + mockSetValue.mockReset() + }) + + it('renders only the add affordance when nothing is configured', () => { + const html = render() + expect(html).toContain('Add fallback model') + expect(html).not.toContain('choice') + }) + + it('labels rows as ordinal choices and offers viable, permitted models', () => { + subBlockValues.fallbackModels = [ + { id: 'r1', model: 'gpt-5' }, + { id: 'r2', model: '' }, + ] + const html = render() + expect(html).toContain('2nd choice') + expect(html).toContain('3rd choice') + expect(html).not.toContain('Auto') + expect(html).not.toContain('denied-model') + /** The primary is never offered; a model already chosen is offered but disabled. */ + expect(html).not.toContain('>claude-sonnet-5<') + expect(html).toContain('data-disabled="true">gpt-5<') + }) + + it('asks for an environment variable only when the row model needs its own key', () => { + subBlockValues.fallbackModels = [{ id: 'r1', model: 'gpt-5' }] + expect(render()).not.toContain('data-combobox="Select an environment variable"') + + subBlockValues.fallbackModels = [ + { id: 'r1', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + ] + const html = render() + expect(html).toContain('data-combobox="Select an environment variable"') + expect(html).toContain('data-value="{{OPENROUTER_API_KEY}}"') + expect(html).toContain('OPENROUTER_API_KEY') + expect(html).toContain('Create variable') + }) + + it('shows a tuning field only for the knobs the helper says need one', () => { + subBlockValues.fallbackModels = [ + { id: 'r1', model: 'gpt-5', reasoningEffort: 'low' }, + { id: 'r2', model: 'openrouter/x' }, + ] + const html = render() + expect(html).toContain('data-combobox="Select reasoning effort" data-value="low"') + expect(html.match(/Select reasoning effort/g)).toHaveLength(1) + expect(html).not.toContain('Thinking level') + }) + + it('disables the add affordance at the cap', () => { + subBlockValues.fallbackModels = Array.from({ length: 5 }, (_, i) => ({ + id: `r${i}`, + model: `m-${i}`, + })) + const html = render() + expect(html).toMatch(/]*disabled=""[^>]*>Add fallback model/) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx new file mode 100644 index 00000000000..6900372b2ca --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx @@ -0,0 +1,377 @@ +'use client' + +import { useCallback, useMemo } from 'react' +import { Button, ChipCombobox, type ComboboxOption, Label, Tooltip } from '@sim/emcn' +import { ChevronDown, ChevronUp, Plus, Trash } from '@sim/emcn/icons' +import { generateShortId } from '@sim/utils/id' +import { useParams } from 'next/navigation' +import { writePendingCredentialCreateRequest } from '@/lib/credentials/client-state' +import type { WorkspaceEnvironmentData } from '@/lib/environment/api' +import { + FALLBACK_TUNING_LABELS, + type FallbackModelEntry, + type FallbackTuningKnob, + fallbackRowNeedsApiKey, + getFallbackTuningKnobsToShow, + getTuningOptionsForModel, + isViableFallbackModel, + MAX_FALLBACK_MODELS, + ordinalChoiceLabel, +} from '@/lib/workflows/blocks/fallback-models' +import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' +import { getModelOptions } from '@/blocks/utils' +import { usePersonalEnvironment, useWorkspaceEnvironment } from '@/hooks/queries/environment' +import { usePermissionConfig } from '@/hooks/use-permission-config' +import { useSettingsNavigation } from '@/hooks/use-settings-navigation' +import { useProvidersStore } from '@/stores/providers/store' + +const CREATE_VARIABLE_VALUE = 'action-create-variable' + +interface ModelFallbackListProps { + blockId: string + subBlockId: string + isPreview?: boolean + previewValue?: FallbackModelEntry[] | null + disabled?: boolean +} + +interface FallbackRowProps { + row: FallbackModelEntry + index: number + count: number + primaryModel: string + primaryTuning: Partial> + modelOptions: ComboboxOption[] + envVarOptions: ComboboxOption[] + readOnly: boolean + onChangeModel: (id: string, model: string) => void + onChangeApiKey: (id: string, apiKey: string) => void + onChangeTuning: (id: string, knob: FallbackTuningKnob, value: string) => void + onMove: (id: string, direction: -1 | 1) => void + onRemove: (id: string) => void +} + +function selectWorkspaceEnvironment(data: WorkspaceEnvironmentData): WorkspaceEnvironmentData { + return { + workspace: data.workspace || {}, + personal: data.personal || {}, + conflicts: data.conflicts || [], + } +} + +function FallbackRow({ + row, + index, + count, + primaryModel, + primaryTuning, + modelOptions, + envVarOptions, + readOnly, + onChangeModel, + onChangeApiKey, + onChangeTuning, + onMove, + onRemove, +}: FallbackRowProps) { + const needsApiKey = fallbackRowNeedsApiKey(row.model, primaryModel) + const tuningKnobs = row.model + ? getFallbackTuningKnobsToShow(row.model, primaryModel, primaryTuning) + : [] + + return ( +
+
+ {ordinalChoiceLabel(index)} +
+ + + + + Move up + + + + + + Move down + + + + + + Remove + +
+
+ +
+
+ + onChangeModel(row.id, model)} + placeholder='Select a model' + disabled={readOnly} + searchable + searchPlaceholder='Search models...' + maxHeight={240} + emptyMessage='No models available' + /> +
+ {needsApiKey && ( +
+ + onChangeApiKey(row.id, apiKey)} + placeholder='Select an environment variable' + disabled={readOnly} + searchable + searchPlaceholder='Search variables...' + maxHeight={240} + emptyMessage='No environment variables' + /> +
+ )} + {tuningKnobs.map((knob) => { + const options = getTuningOptionsForModel(row.model, knob) ?? [] + return ( +
+ + ({ label: value, value }))} + value={row[knob] ?? options[0] ?? ''} + onChange={(value) => onChangeTuning(row.id, knob, value)} + placeholder={`Select ${FALLBACK_TUNING_LABELS[knob].toLowerCase()}`} + disabled={readOnly} + /> +
+ ) + })} +
+
+ ) +} + +/** + * Ordered fallback models for a model-driven block: the 2nd, 3rd, ... choice + * tried in sequence when the request to the block's own model fails. + * + * Every write is the whole array, so a collaborator's concurrent edit and an + * undo both flow straight through the store. A row's key is stored only as a + * `{{ENV_VAR}}` reference: the picker offers the workspace's variable names and + * nothing else, which is what keeps a raw secret out of the list value (see + * `FallbackModelEntry`). + */ +export function ModelFallbackList({ + blockId, + subBlockId, + isPreview = false, + previewValue, + disabled = false, +}: ModelFallbackListProps) { + const params = useParams() + const workspaceId = typeof params?.workspaceId === 'string' ? params.workspaceId : '' + const { navigateToSettings } = useSettingsNavigation() + const { isModelUsable } = usePermissionConfig() + const providers = useProvidersStore((state) => state.providers) + const [storeValue, setStoreValue] = useSubBlockValue(blockId, subBlockId) + const [primaryModelValue] = useSubBlockValue(blockId, 'model') + const [primaryReasoningEffort] = useSubBlockValue(blockId, 'reasoningEffort') + const [primaryThinkingLevel] = useSubBlockValue(blockId, 'thinkingLevel') + const [primaryVerbosity] = useSubBlockValue(blockId, 'verbosity') + const { data: personalEnv = {} } = usePersonalEnvironment() + const { data: workspaceEnv } = useWorkspaceEnvironment(workspaceId, { + enabled: Boolean(workspaceId), + select: selectWorkspaceEnvironment, + }) + + const readOnly = isPreview || disabled + const primaryModel = typeof primaryModelValue === 'string' ? primaryModelValue : '' + const primaryTuning = useMemo( + () => ({ + reasoningEffort: primaryReasoningEffort, + thinkingLevel: primaryThinkingLevel, + verbosity: primaryVerbosity, + }), + [primaryReasoningEffort, primaryThinkingLevel, primaryVerbosity] + ) + const rows: FallbackModelEntry[] = useMemo(() => { + const value = isPreview ? previewValue : storeValue + return Array.isArray(value) ? value : [] + }, [isPreview, previewValue, storeValue]) + + const modelOptions = useMemo((): ComboboxOption[] => { + const chosen = new Set(rows.map((row) => row.model)) + return getModelOptions() + .filter( + (option) => isModelUsable(option.id) && isViableFallbackModel(option.id, primaryModel) + ) + .map((option) => ({ + label: option.label, + value: option.id, + ...(option.icon ? { icon: option.icon } : {}), + disabled: chosen.has(option.id), + })) + // `providers` is what changes the option list; `getModelOptions` reads it from the store. + }, [rows, primaryModel, isModelUsable, providers]) + + const envVarOptions = useMemo((): ComboboxOption[] => { + const names = workspaceId + ? [ + ...Object.keys(workspaceEnv?.workspace ?? {}), + ...Object.keys(workspaceEnv?.personal ?? {}), + ] + : Object.keys(personalEnv) + const options: ComboboxOption[] = [...new Set(names)].map((name) => ({ + label: name, + value: `{{${name}}}`, + })) + options.push({ + label: 'Create variable', + value: CREATE_VARIABLE_VALUE, + icon: Plus, + onSelect: () => { + if (workspaceId) { + writePendingCredentialCreateRequest({ + workspaceId, + type: 'env_personal', + requestedAt: Date.now(), + }) + } + navigateToSettings({ section: 'secrets' }) + }, + }) + return options + }, [workspaceId, workspaceEnv, personalEnv, navigateToSettings]) + + const write = useCallback( + (next: FallbackModelEntry[]) => { + if (readOnly) return + setStoreValue(next) + }, + [readOnly, setStoreValue] + ) + + const handleAdd = useCallback(() => { + if (rows.length >= MAX_FALLBACK_MODELS) return + write([...rows, { id: generateShortId(), model: '' }]) + }, [rows, write]) + + const handleRemove = useCallback( + (id: string) => write(rows.filter((row) => row.id !== id)), + [rows, write] + ) + + const handleMove = useCallback( + (id: string, direction: -1 | 1) => { + const index = rows.findIndex((row) => row.id === id) + const target = index + direction + if (index === -1 || target < 0 || target >= rows.length) return + const next = [...rows] + ;[next[index], next[target]] = [next[target], next[index]] + write(next) + }, + [rows, write] + ) + + const handleChangeModel = useCallback( + (id: string, model: string) => { + write( + rows.map((row) => { + if (row.id !== id) return row + /** A new model gets a clean row: a key it no longer needs and tuning it may not declare both go. */ + const keepKey = row.apiKey && fallbackRowNeedsApiKey(model, primaryModel) + return { id: row.id, model, ...(keepKey ? { apiKey: row.apiKey } : {}) } + }) + ) + }, + [rows, primaryModel, write] + ) + + const handleChangeTuning = useCallback( + (id: string, knob: FallbackTuningKnob, value: string) => { + write( + rows.map((row) => { + if (row.id !== id) return row + /** The provider-decides entry is the field's default, so it is stored as absence. */ + const sentinel = getTuningOptionsForModel(row.model, knob)?.[0] + const { [knob]: _previous, ...rest } = row + return value && value !== sentinel ? { ...rest, [knob]: value } : rest + }) + ) + }, + [rows, write] + ) + + const handleChangeApiKey = useCallback( + (id: string, apiKey: string) => { + if (apiKey === CREATE_VARIABLE_VALUE) return + write(rows.map((row) => (row.id === id ? { ...row, apiKey } : row))) + }, + [rows, write] + ) + + return ( +
+ {rows.map((row, index) => ( + + ))} + +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx index 2fd31809eac..9a8b9ce294c 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx @@ -32,6 +32,7 @@ import { McpServerSelector, McpToolSelector, MessagesInput, + ModelFallbackList, ResponseFormat, ScheduleInfo, SelectorInput, @@ -1196,6 +1197,17 @@ function SubBlockComponent({ } return } + case 'model-fallback-list': + return ( + + ) + case 'messages-input': return ( Date: Thu, 17 Sep 2026 22:52:11 -0700 Subject: [PATCH 06/14] docs(agent): document fallback models and their interplay with retry --- apps/docs/content/docs/workflows/blocks/agent.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/docs/content/docs/workflows/blocks/agent.mdx b/apps/docs/content/docs/workflows/blocks/agent.mdx index dde3a58689c..ef83c5aad43 100644 --- a/apps/docs/content/docs/workflows/blocks/agent.mdx +++ b/apps/docs/content/docs/workflows/blocks/agent.mdx @@ -25,7 +25,7 @@ Answer in two sentences, cite the doc you used, and never guess a price. ### Model -The model that runs the step. Defaults to `claude-sonnet-4-6`. Type or pick any model from OpenAI, Anthropic, Google, xAI, Groq, Cerebras, DeepSeek, Azure, AWS Bedrock, Google Vertex, or OpenRouter, or a local model through Ollama or VLLM. +The model that runs the step. Defaults to `claude-sonnet-5`. Type or pick any model from OpenAI, Anthropic, Google, xAI, Groq, Cerebras, DeepSeek, Azure, AWS Bedrock, Google Vertex, or OpenRouter, or a local model through Ollama or VLLM. For a custom cloud deployment, enter its provider prefix and model ID: `azure/my-deployment`, `azure-anthropic/my-deployment`, `bedrock/my-inference-profile`, or `vertex/my-gemini-model`. The prefix selects the provider and shows its credential fields even when the ID is absent from the catalog. Bedrock accepts full inference profile ARNs after `bedrock/`; Vertex uses the Gemini API and accepts Google model resource names. The deployment must support the selected provider's API. Custom IDs have no catalog pricing or token limits. @@ -87,6 +87,8 @@ Some settings live under advanced, or appear only for models that support them: - **Reasoning effort / Thinking level.** For models with extended reasoning, how much the model thinks before answering. Higher is more thorough but slower and costs more tokens. - **Prompt caching.** For Anthropic Claude models, reuses the system prompt and tool definitions between runs instead of re-reading them every time. Cached input costs a tenth of the normal rate, but writing the cache costs 1.25x, so leave it off for one-off runs and turn it on when the same agent runs repeatedly. The cache covers a prefix only if it reaches 1,024 tokens (2,048 on Haiku) — below that Anthropic ignores it and nothing changes. Entries expire after five minutes of no use. - **API key.** Your key for the chosen provider. Hidden on hosted Sim, which supplies one. +- **Fallback models.** An ordered list of models to try when the request to the selected model fails, whether the provider is overloaded, rate-limited, or down. Sim tries the 2nd choice, then the 3rd, and so on, and `` reports the model that answered. Hosted models need no setup. A model that needs its own key takes it from a workspace environment variable you pick on the row; a model on the same provider as the selected model reuses the block's key. Azure, Bedrock, and Vertex models can only be fallbacks for a selected model of the same family, since they use that model's credentials. The Auto model cannot be a fallback. A fallback runs with the selected model's settings where its provider accepts them: temperature and max output tokens are clamped to the fallback's limits, and when the fallback has a reasoning effort, thinking level, or verbosity setting that the selected model's value does not fit, the row shows that field so you can pick a value for it, otherwise the provider's default applies. +- **Retry on fail.** Runs the block again after a failure, up to a maximum number of tries with a wait between them. Fallback models work inside each try: one try walks the selected model and then every fallback, and only when all of them fail does the next try begin. A failure that happens after the model already called a tool runs that conversation again on the next model, so keep fallbacks and retry off for agents whose tools must not repeat. OpenAI and Gemini cache automatically at no extra cost and need no setting; their discount is already reflected in what you are charged. @@ -148,4 +150,5 @@ The Agent reads the message from Start with `` and returns a result From c77f940103280d783c3d68e2f6345cc38cbbc721 Mon Sep 17 00:00:00 2001 From: Marcus Chandra Date: Thu, 17 Sep 2026 23:15:11 -0700 Subject: [PATCH 07/14] fix(agent): harden fallback models after pre-landing review Executor: hydrate the primary before the secret registries settle and fork again, re-forking per fallback provider, so file provenance stays in the result registry; project the fall-through warn against the failed attempt's error registry and reinstate it before a post-skip rethrow; prime a streaming candidate's first chunk when another candidate follows, so a tool-loop startup failure still falls back; never fall back on a deep-research follow-up turn; strip the sim-auto identity preamble from a named fallback's messages; treat a row key still in {{VAR}} form as no key; stop starting candidates after an abort; record failed models on every exit, clearing them when a retry succeeds, and only when the names project safely. Helpers: row tuning applies only while its field is shown; the editor's row transforms are pure functions; changing a row's model to another provider drops the key reference; Bedrock honors NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS. Editor: legacy Combobox to match the block editor, a row's own model is never disabled in its own picker, move controls only for two or more rows, the shared dashed add-row button, Create Secret wording, non-reference keys never rendered. Trace: one Failed model row per fallback. Export sanitizer uses the strict whole-reference check. --- .../components/trace-view/trace-view.tsx | 4 +- .../model-fallback-list.test.tsx | 24 +- .../model-fallback-list.tsx | 284 +++++++++--------- apps/sim/blocks/blocks/agent.ts | 16 +- apps/sim/blocks/utils.test.ts | 8 +- apps/sim/blocks/utils.ts | 12 +- .../handlers/agent/agent-handler.test.ts | 281 +++++++++++++---- .../executor/handlers/agent/agent-handler.ts | 226 ++++++++++++-- apps/sim/executor/handlers/agent/types.ts | 3 +- .../execution/trace-spans/trace-spans.test.ts | 30 ++ .../workflows/blocks/fallback-models.test.ts | 95 ++++++ .../lib/workflows/blocks/fallback-models.ts | 101 ++++++- .../credentials/credential-extractor.test.ts | 25 +- .../credentials/credential-extractor.ts | 5 +- apps/sim/lib/workflows/editing/validation.ts | 41 ++- .../workflows/search-replace/indexer.test.ts | 8 + 16 files changed, 878 insertions(+), 285 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx index 98aee8c718a..38a774317b5 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx @@ -694,8 +694,8 @@ const TraceDetailPane = memo(function TraceDetailPane({ span }: { span: TraceSpa if (span.tries !== undefined) metaEntries.push({ label: 'Tries', value: String(span.tries) }) if (span.provider) metaEntries.push({ label: 'Provider', value: span.provider }) if (span.model) metaEntries.push({ label: 'Model', value: span.model }) - if (span.modelFallbacks?.length) { - metaEntries.push({ label: 'Fell back from', value: span.modelFallbacks.join(', ') }) + for (const failedModel of span.modelFallbacks ?? []) { + metaEntries.push({ label: 'Failed model', value: failedModel }) } if (span.finishReason) metaEntries.push({ label: 'Finish reason', value: span.finishReason }) const ttftFormatted = formatTtft(span.ttft) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx index aa837b5775c..af7958f51a6 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx @@ -35,7 +35,7 @@ vi.mock('@sim/emcn', () => ({ {children} ), - ChipCombobox: ({ + Combobox: ({ options, value, placeholder, @@ -155,23 +155,35 @@ describe('ModelFallbackList', () => { expect(html).toContain('3rd choice') expect(html).not.toContain('Auto') expect(html).not.toContain('denied-model') - /** The primary is never offered; a model already chosen is offered but disabled. */ + /** The primary is never offered. A model another row holds is disabled there, never in its own row. */ expect(html).not.toContain('>claude-sonnet-5<') - expect(html).toContain('data-disabled="true">gpt-5<') + expect(html.match(/data-disabled="true">gpt-5gpt-5 { + subBlockValues.fallbackModels = [ + { id: 'r1', model: 'openrouter/x', apiKey: 'sk-raw-through-socket' }, + ] + const html = render() + expect(html).not.toContain('aria-label="Move up"') + expect(html).not.toContain('sk-raw-through-socket') + expect(html).toContain('data-combobox="Select a secret" data-value=""') }) it('asks for an environment variable only when the row model needs its own key', () => { subBlockValues.fallbackModels = [{ id: 'r1', model: 'gpt-5' }] - expect(render()).not.toContain('data-combobox="Select an environment variable"') + expect(render()).not.toContain('data-combobox="Select a secret"') subBlockValues.fallbackModels = [ { id: 'r1', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, ] const html = render() - expect(html).toContain('data-combobox="Select an environment variable"') + expect(html).toContain('data-combobox="Select a secret"') expect(html).toContain('data-value="{{OPENROUTER_API_KEY}}"') expect(html).toContain('OPENROUTER_API_KEY') - expect(html).toContain('Create variable') + expect(html).toContain('Create Secret') }) it('shows a tuning field only for the knobs the helper says need one', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx index 6900372b2ca..502835b8361 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx @@ -1,13 +1,16 @@ 'use client' -import { useCallback, useMemo } from 'react' -import { Button, ChipCombobox, type ComboboxOption, Label, Tooltip } from '@sim/emcn' +import { memo, useCallback, useMemo } from 'react' +import { Button, Combobox, type ComboboxOption, Label, Tooltip } from '@sim/emcn' import { ChevronDown, ChevronUp, Plus, Trash } from '@sim/emcn/icons' import { generateShortId } from '@sim/utils/id' import { useParams } from 'next/navigation' import { writePendingCredentialCreateRequest } from '@/lib/credentials/client-state' -import type { WorkspaceEnvironmentData } from '@/lib/environment/api' import { + addFallbackRow, + changeFallbackRowApiKey, + changeFallbackRowModel, + changeFallbackRowTuning, FALLBACK_TUNING_LABELS, type FallbackModelEntry, type FallbackTuningKnob, @@ -15,8 +18,11 @@ import { getFallbackTuningKnobsToShow, getTuningOptionsForModel, isViableFallbackModel, + isWholeEnvVarReference, MAX_FALLBACK_MODELS, + moveFallbackRow, ordinalChoiceLabel, + removeFallbackRow, } from '@/lib/workflows/blocks/fallback-models' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' import { getModelOptions } from '@/blocks/utils' @@ -25,7 +31,7 @@ import { usePermissionConfig } from '@/hooks/use-permission-config' import { useSettingsNavigation } from '@/hooks/use-settings-navigation' import { useProvidersStore } from '@/stores/providers/store' -const CREATE_VARIABLE_VALUE = 'action-create-variable' +const CREATE_SECRET_VALUE = 'action-create-secret' interface ModelFallbackListProps { blockId: string @@ -35,13 +41,22 @@ interface ModelFallbackListProps { disabled?: boolean } +/** A viable model before the per-row `disabled` flag is stamped on it. */ +interface ViableModelOption { + label: string + value: string + icon?: React.ComponentType<{ className?: string }> +} + interface FallbackRowProps { row: FallbackModelEntry index: number count: number primaryModel: string primaryTuning: Partial> - modelOptions: ComboboxOption[] + viableOptions: ViableModelOption[] + /** Models any row holds; a row's own model is exempted when its options are built. */ + takenModels: ReadonlySet envVarOptions: ComboboxOption[] readOnly: boolean onChangeModel: (id: string, model: string) => void @@ -51,21 +66,14 @@ interface FallbackRowProps { onRemove: (id: string) => void } -function selectWorkspaceEnvironment(data: WorkspaceEnvironmentData): WorkspaceEnvironmentData { - return { - workspace: data.workspace || {}, - personal: data.personal || {}, - conflicts: data.conflicts || [], - } -} - -function FallbackRow({ +const FallbackRow = memo(function FallbackRow({ row, index, count, primaryModel, primaryTuning, - modelOptions, + viableOptions, + takenModels, envVarOptions, readOnly, onChangeModel, @@ -74,10 +82,27 @@ function FallbackRow({ onMove, onRemove, }: FallbackRowProps) { - const needsApiKey = fallbackRowNeedsApiKey(row.model, primaryModel) - const tuningKnobs = row.model - ? getFallbackTuningKnobsToShow(row.model, primaryModel, primaryTuning) - : [] + const modelOptions = useMemo( + (): ComboboxOption[] => + viableOptions.map((option) => ({ + ...option, + disabled: option.value !== row.model && takenModels.has(option.value), + })), + [viableOptions, takenModels, row.model] + ) + + const { needsApiKey, tuningFields } = useMemo(() => { + if (!row.model) return { needsApiKey: false, tuningFields: [] } + return { + needsApiKey: fallbackRowNeedsApiKey(row.model, primaryModel), + tuningFields: getFallbackTuningKnobsToShow(row.model, primaryModel, primaryTuning).map( + (knob) => ({ knob, options: getTuningOptionsForModel(row.model, knob) ?? [] }) + ), + } + }, [row.model, primaryModel, primaryTuning]) + + /** Only a reference is ever shown; anything else that reached the store reads as unset. */ + const apiKeyValue = isWholeEnvVarReference(row.apiKey) ? row.apiKey : '' return (
{ordinalChoiceLabel(index)}
- - - - - Move up - - - - - - Move down - + {count > 1 && ( + <> + + + + + Move up + + + + + + Move down + + + )}
) -} +}) /** * Ordered fallback models for a model-driven block: the 2nd, 3rd, ... choice @@ -189,9 +215,9 @@ function FallbackRow({ * * Every write is the whole array, so a collaborator's concurrent edit and an * undo both flow straight through the store. A row's key is stored only as a - * `{{ENV_VAR}}` reference: the picker offers the workspace's variable names and + * `{{ENV_VAR}}` reference: the picker offers the workspace's secret names and * nothing else, which is what keeps a raw secret out of the list value (see - * `FallbackModelEntry`). + * `FallbackModelEntry`). The row transforms live in `fallback-models.ts`. */ export function ModelFallbackList({ blockId, @@ -213,7 +239,6 @@ export function ModelFallbackList({ const { data: personalEnv = {} } = usePersonalEnvironment() const { data: workspaceEnv } = useWorkspaceEnvironment(workspaceId, { enabled: Boolean(workspaceId), - select: selectWorkspaceEnvironment, }) const readOnly = isPreview || disabled @@ -231,20 +256,26 @@ export function ModelFallbackList({ return Array.isArray(value) ? value : [] }, [isPreview, previewValue, storeValue]) - const modelOptions = useMemo((): ComboboxOption[] => { - const chosen = new Set(rows.map((row) => row.model)) - return getModelOptions() - .filter( - (option) => isModelUsable(option.id) && isViableFallbackModel(option.id, primaryModel) - ) - .map((option) => ({ - label: option.label, - value: option.id, - ...(option.icon ? { icon: option.icon } : {}), - disabled: chosen.has(option.id), - })) - // `providers` is what changes the option list; `getModelOptions` reads it from the store. - }, [rows, primaryModel, isModelUsable, providers]) + /** + * `getModelOptions` reads the providers store itself; subscribing to + * `providers` here is what recomputes the list when a dynamic provider's + * models finish loading. + */ + const viableOptions = useMemo( + (): ViableModelOption[] => + getModelOptions() + .filter( + (option) => isModelUsable(option.id) && isViableFallbackModel(option.id, primaryModel) + ) + .map((option) => ({ + label: option.label, + value: option.id, + ...(option.icon ? { icon: option.icon } : {}), + })), + [primaryModel, isModelUsable, providers] + ) + + const takenModels = useMemo(() => new Set(rows.map((row) => row.model).filter(Boolean)), [rows]) const envVarOptions = useMemo((): ComboboxOption[] => { const names = workspaceId @@ -258,8 +289,8 @@ export function ModelFallbackList({ value: `{{${name}}}`, })) options.push({ - label: 'Create variable', - value: CREATE_VARIABLE_VALUE, + label: 'Create Secret', + value: CREATE_SECRET_VALUE, icon: Plus, onSelect: () => { if (workspaceId) { @@ -277,67 +308,34 @@ export function ModelFallbackList({ const write = useCallback( (next: FallbackModelEntry[]) => { - if (readOnly) return + if (readOnly || next === rows) return setStoreValue(next) }, - [readOnly, setStoreValue] + [readOnly, rows, setStoreValue] ) - const handleAdd = useCallback(() => { - if (rows.length >= MAX_FALLBACK_MODELS) return - write([...rows, { id: generateShortId(), model: '' }]) - }, [rows, write]) - + const handleAdd = useCallback(() => write(addFallbackRow(rows, generateShortId())), [rows, write]) const handleRemove = useCallback( - (id: string) => write(rows.filter((row) => row.id !== id)), + (id: string) => write(removeFallbackRow(rows, id)), [rows, write] ) - const handleMove = useCallback( - (id: string, direction: -1 | 1) => { - const index = rows.findIndex((row) => row.id === id) - const target = index + direction - if (index === -1 || target < 0 || target >= rows.length) return - const next = [...rows] - ;[next[index], next[target]] = [next[target], next[index]] - write(next) - }, + (id: string, direction: -1 | 1) => write(moveFallbackRow(rows, id, direction)), [rows, write] ) - const handleChangeModel = useCallback( - (id: string, model: string) => { - write( - rows.map((row) => { - if (row.id !== id) return row - /** A new model gets a clean row: a key it no longer needs and tuning it may not declare both go. */ - const keepKey = row.apiKey && fallbackRowNeedsApiKey(model, primaryModel) - return { id: row.id, model, ...(keepKey ? { apiKey: row.apiKey } : {}) } - }) - ) - }, + (id: string, model: string) => write(changeFallbackRowModel(rows, id, model, primaryModel)), [rows, primaryModel, write] ) - const handleChangeTuning = useCallback( - (id: string, knob: FallbackTuningKnob, value: string) => { - write( - rows.map((row) => { - if (row.id !== id) return row - /** The provider-decides entry is the field's default, so it is stored as absence. */ - const sentinel = getTuningOptionsForModel(row.model, knob)?.[0] - const { [knob]: _previous, ...rest } = row - return value && value !== sentinel ? { ...rest, [knob]: value } : rest - }) - ) - }, + (id: string, knob: FallbackTuningKnob, value: string) => + write(changeFallbackRowTuning(rows, id, knob, value)), [rows, write] ) - const handleChangeApiKey = useCallback( (id: string, apiKey: string) => { - if (apiKey === CREATE_VARIABLE_VALUE) return - write(rows.map((row) => (row.id === id ? { ...row, apiKey } : row))) + if (apiKey === CREATE_SECRET_VALUE) return + write(changeFallbackRowApiKey(rows, id, apiKey)) }, [rows, write] ) @@ -352,7 +350,8 @@ export function ModelFallbackList({ count={rows.length} primaryModel={primaryModel} primaryTuning={primaryTuning} - modelOptions={modelOptions} + viableOptions={viableOptions} + takenModels={takenModels} envVarOptions={envVarOptions} readOnly={readOnly} onChangeModel={handleChangeModel} @@ -362,16 +361,17 @@ export function ModelFallbackList({ onRemove={handleRemove} /> ))} - + {!readOnly && ( + + )}
) } diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index 7bf4d1e697a..d715b76cb9c 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -1,7 +1,10 @@ import { createLogger } from '@sim/logger' import { omit } from '@sim/utils/object' import { AgentIcon } from '@/components/icons' -import { normalizeFallbackModels } from '@/lib/workflows/blocks/fallback-models' +import { + MAX_FALLBACK_MODELS, + normalizeFallbackModels, +} from '@/lib/workflows/blocks/fallback-models' import type { BlockConfig } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' import { @@ -33,6 +36,8 @@ const logger = createLogger('AgentBlock') /** Model the agent block falls back to when `model` is unset or the auto pseudo-model. */ const AGENT_FALLBACK_MODEL = 'claude-sonnet-5' + +const FALLBACK_MODELS_DESCRIPTION = `Ordered models tried in sequence when the request to the selected model fails. Each row is { model, apiKey?, reasoningEffort?, thinkingLevel?, verbosity? }; apiKey, when present, must be a whole {{ENV_VAR}} reference, and a tuning value must be one the row model declares. sim-auto is not allowed. Max ${MAX_FALLBACK_MODELS}.` const MODELS_WITH_REASONING_EFFORT = getModelsWithReasoningEffort() const MODELS_WITH_VERBOSITY = getModelsWithVerbosity() const MODELS_WITH_THINKING = getModelsWithThinking() @@ -436,8 +441,7 @@ Return ONLY the JSON array.`, title: 'Fallback models', type: 'model-fallback-list', mode: 'advanced', - description: - 'Ordered models tried in sequence when the request to the selected model fails. Each row is { model, apiKey?, reasoningEffort?, thinkingLevel?, verbosity? }; apiKey, when present, must be a whole {{ENV_VAR}} reference, and a tuning value must be one the row model declares. sim-auto is not allowed. Max 5.', + description: FALLBACK_MODELS_DESCRIPTION, }, ], tools: { @@ -601,11 +605,7 @@ Return ONLY the JSON array.`, type: 'boolean', description: 'Cache the system prompt and tool definitions on models that support it', }, - fallbackModels: { - type: 'json', - description: - 'Ordered fallback models tried when the selected model fails, each { model, apiKey?: "{{ENV_VAR}}", reasoningEffort?, thinkingLevel?, verbosity? }', - }, + fallbackModels: { type: 'json', description: FALLBACK_MODELS_DESCRIPTION }, tools: { type: 'json', description: 'Available tools configuration' }, skills: { type: 'json', description: 'Selected skills configuration' }, }, diff --git a/apps/sim/blocks/utils.test.ts b/apps/sim/blocks/utils.test.ts index 56845776835..acacf32e90f 100644 --- a/apps/sim/blocks/utils.test.ts +++ b/apps/sim/blocks/utils.test.ts @@ -103,9 +103,15 @@ describe('requiresProviderFamilyCredentials', () => { setEnvFlags({ isHosted: false, isAzureConfigured: false, isOllamaConfigured: false }) }) - it('is true for Vertex and Bedrock, whose credentials live on the block', () => { + it('is true for Vertex, and for Bedrock until the deployment provides default credentials', () => { expect(requiresProviderFamilyCredentials('vertex/gemini-2.5-pro')).toBe(true) expect(requiresProviderFamilyCredentials('bedrock/my-inference-profile')).toBe(true) + vi.stubEnv('NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS', 'true') + try { + expect(requiresProviderFamilyCredentials('bedrock/my-inference-profile')).toBe(false) + } finally { + vi.unstubAllEnvs() + } }) it('is true for Azure only until the deployment configures it server-side', () => { diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index 09560d95d8c..58eaf30fe4e 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -1,6 +1,7 @@ import { toError } from '@sim/utils/errors' import { SimAutoIcon } from '@/components/icons' import { getDeploymentShape } from '@/lib/core/config/deployment-shape' +import { getEnv, isTruthy } from '@/lib/core/config/env' import { isOllamaConfigured } from '@/lib/core/config/env-flags' import { getScopesForService } from '@/lib/oauth/utils' import { containsReference } from '@/lib/workflows/sanitization/references' @@ -285,14 +286,15 @@ export function getCohereRerankerApiKeyCondition() { /** * Whether `model` can only run with credentials that live on the block beyond an - * API key: a Vertex OAuth credential, Bedrock AWS keys and region, or an Azure - * endpoint the deployment has not configured server-side. Those fields render - * only while the block's own `model` is in that provider family, so nothing - * outside the family can inherit them. + * API key: a Vertex OAuth credential, Bedrock AWS keys, or an Azure endpoint, + * unless the deployment supplies them server-side (the same env flags that hide + * those fields). The fields render only while the block's own `model` is in + * that provider family, so nothing outside the family can inherit them. */ export function requiresProviderFamilyCredentials(model: string): boolean { const provider = findProviderFromModel(model.trim()) - if (provider === 'vertex' || provider === 'bedrock') return true + if (provider === 'vertex') return true + if (provider === 'bedrock') return !isTruthy(getEnv('NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS')) if (provider === 'azure-openai' || provider === 'azure-anthropic') { return !getDeploymentShape().azureConfigured } diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index c3dcb49d035..7a55a12fc85 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -43,15 +43,23 @@ process.env.NEXT_PUBLIC_APP_URL = 'http://localhost:3000' const { mockDiscoverMcpServerToolsAsExecutor, mockImportWorkspaceFileSecretProvenanceForModelView, + mockValidateModelProvider, } = vi.hoisted(() => ({ mockDiscoverMcpServerToolsAsExecutor: vi.fn().mockResolvedValue([]), mockImportWorkspaceFileSecretProvenanceForModelView: vi.fn().mockResolvedValue(true), + mockValidateModelProvider: vi.fn().mockResolvedValue(undefined), })) vi.mock('@/lib/internal/mcp/discover-tools', () => ({ discoverMcpServerToolsAsExecutor: mockDiscoverMcpServerToolsAsExecutor, })) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + assertPermissionsAllowed: vi.fn().mockResolvedValue(undefined), + validateBlockType: vi.fn().mockResolvedValue(undefined), + validateModelProvider: mockValidateModelProvider, +})) + vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ importWorkspaceFileSecretProvenanceForModelView: mockImportWorkspaceFileSecretProvenanceForModelView, @@ -501,7 +509,6 @@ describe('AgentBlockHandler', () => { userPrompt: 'Hello', apiKey: 'primary-key', temperature: 0.4, - previousInteractionId: 'interaction-1', } const providerFor = (model: string) => { @@ -511,12 +518,62 @@ describe('AgentBlockHandler', () => { return 'openai' } + const providerResponse = (model: string, content = 'ok') => ({ + content, + model, + tokens: { input: 1, output: 1, total: 2 }, + toolCalls: [], + cost: 0, + timing: { total: 1 }, + }) + + const openLog = (blockId = mockBlock.id, endedAt = '') => ({ + blockId, + startedAt: '2026-01-01T00:00:00.000Z', + endedAt, + durationMs: 0, + success: false, + executionOrder: 1, + }) + + const streamingResponse = ( + chunks: string[], + options: { failBeforeFirstChunk?: Error; failAfterFirstChunk?: Error } = {} + ) => ({ + stream: new ReadableStream({ + async pull(controller) { + if (options.failBeforeFirstChunk) throw options.failBeforeFirstChunk + const chunk = chunks.shift() + if (chunk !== undefined) { + controller.enqueue(chunk) + return + } + if (options.failAfterFirstChunk) throw options.failAfterFirstChunk + controller.close() + }, + }), + execution: { output: { content: '' } }, + }) + + const drain = async (stream: ReadableStream) => { + const reader = stream.getReader() + const chunks: string[] = [] + for (;;) { + const { done, value } = await reader.read() + if (done) return chunks + chunks.push(value) + } + } + beforeEach(() => { mockGetProviderFromModel.mockImplementation(providerFor) + mockValidateModelProvider.mockResolvedValue(undefined) }) it('never touches the fallbacks when the primary answers', async () => { - await handler.execute(mockContext, mockBlock, { + const log = openLog() + log.modelFallbacks = ['stale-from-earlier-try'] + await handler.execute({ ...mockContext, blockLogs: [log] }, mockBlock, { ...baseInputs, fallbackModels: [{ model: 'claude-sonnet-5' }], }) @@ -527,28 +584,15 @@ describe('AgentBlockHandler', () => { 'Agent model failed; trying fallback', expect.anything() ) - expect(mockContext.blockLogs).toEqual([]) + /** A try that succeeds on the primary clears what an earlier try wrote. */ + expect(log.modelFallbacks).toBeUndefined() }) it('falls through to the next model with the same request and no primary-only fields', async () => { mockExecuteProviderRequest .mockRejectedValueOnce(new Error('overloaded')) - .mockResolvedValueOnce({ - content: 'from fallback', - model: 'claude-sonnet-5', - tokens: { input: 1, output: 1, total: 2 }, - toolCalls: [], - cost: 0.001, - timing: { total: 10 }, - }) - const blockLog = { - blockId: mockBlock.id, - startedAt: new Date().toISOString(), - endedAt: '', - durationMs: 0, - success: false, - executionOrder: 1, - } + .mockResolvedValueOnce(providerResponse('claude-sonnet-5', 'from fallback')) + const blockLog = openLog() const ctx = { ...mockContext, blockLogs: [blockLog] } const result = await handler.execute(ctx, mockBlock, { @@ -564,8 +608,6 @@ describe('AgentBlockHandler', () => { expect(fallbackRequest.model).toBe('claude-sonnet-5') expect(fallbackRequest.messages).toEqual(primaryRequest.messages) expect(fallbackRequest.temperature).toBe(primaryRequest.temperature) - expect(primaryRequest.previousInteractionId).toBe('interaction-1') - expect(fallbackRequest.previousInteractionId).toBeUndefined() expect((result as { model: string }).model).toBe('claude-sonnet-5') expect(mockAgentLogger.warn).toHaveBeenCalledWith( 'Agent model failed; trying fallback', @@ -574,44 +616,66 @@ describe('AgentBlockHandler', () => { expect(blockLog).toMatchObject({ modelFallbacks: ['gpt-4o'] }) }) + it('never falls back on a deep-research follow-up turn', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + + await expect( + handler.execute(mockContext, mockBlock, { + ...baseInputs, + previousInteractionId: 'interaction-1', + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toThrow('overloaded') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + expect(mockAgentLogger.info).toHaveBeenCalledWith( + 'Fallback models skipped for a deep-research follow-up turn', + expect.objectContaining({ blockId: mockBlock.id }) + ) + }) + it('gives a fallback its own key, the block key on the same provider, and nothing otherwise', async () => { mockExecuteProviderRequest .mockRejectedValueOnce(new Error('one')) .mockRejectedValueOnce(new Error('two')) .mockRejectedValueOnce(new Error('three')) - .mockResolvedValueOnce({ - content: 'ok', - model: 'gpt-4o-mini', - tokens: { input: 1, output: 1, total: 2 }, - toolCalls: [], - cost: 0, - timing: { total: 1 }, - }) + .mockResolvedValueOnce(providerResponse('gpt-4o-mini')) await handler.execute(mockContext, mockBlock, { ...baseInputs, fallbackModels: [ - { model: 'claude-sonnet-5', apiKey: '{{ANTHROPIC_KEY}}' }, + { model: 'claude-sonnet-5', apiKey: 'anthropic-row-key' }, { model: 'claude-haiku-5' }, { model: 'gpt-4o-mini' }, ], }) const keys = mockExecuteProviderRequest.mock.calls.map(([, request]) => request.apiKey) - expect(keys).toEqual(['primary-key', '{{ANTHROPIC_KEY}}', undefined, 'primary-key']) + expect(keys).toEqual(['primary-key', 'anthropic-row-key', undefined, 'primary-key']) + }) + + it('treats a row key that was never resolved as no key and says which variable', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('one')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + + await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5', apiKey: '{{MISSING_KEY}}' }], + }) + + expect(mockExecuteProviderRequest.mock.calls[1][1].apiKey).toBeUndefined() + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Fallback key variable is not set for this run', + expect.objectContaining({ model: 'claude-sonnet-5', variable: '{{MISSING_KEY}}' }) + ) }) it('re-resolves tuning for the fallback: row value wins, caps clamp, undeclared values drop', async () => { const fallbackCap = getModelCapabilities('gpt-5.4-mini')?.maxOutputTokens expect(fallbackCap).toEqual(expect.any(Number)) - mockExecuteProviderRequest.mockRejectedValueOnce(new Error('down')).mockResolvedValueOnce({ - content: 'ok', - model: 'gpt-5.4-mini', - tokens: { input: 1, output: 1, total: 2 }, - toolCalls: [], - cost: 0, - timing: { total: 1 }, - }) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('gpt-5.4-mini')) await handler.execute(mockContext, mockBlock, { ...baseInputs, @@ -640,14 +704,7 @@ describe('AgentBlockHandler', () => { const first = new Error('primary down') const last = new Error('fallback down') mockExecuteProviderRequest.mockRejectedValueOnce(first).mockRejectedValueOnce(last) - const blockLog = { - blockId: mockBlock.id, - startedAt: '', - endedAt: '', - durationMs: 0, - success: false, - executionOrder: 1, - } + const blockLog = openLog() await expect( handler.execute({ ...mockContext, blockLogs: [blockLog] }, mockBlock, { @@ -659,15 +716,25 @@ describe('AgentBlockHandler', () => { expect(blockLog).toMatchObject({ modelFallbacks: ['gpt-4o'] }) }) + it('rethrows the primary error when every fallback was skipped as unusable', async () => { + const primaryError = new Error('primary down') + mockExecuteProviderRequest.mockRejectedValueOnce(primaryError) + const blockLog = openLog() + + await expect( + handler.execute({ ...mockContext, blockLogs: [blockLog] }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'blacklisted-model' }], + }) + ).rejects.toBe(primaryError) + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + expect(blockLog.modelFallbacks).toBeUndefined() + }) + it('skips sim-auto, duplicates, the primary itself, and unusable providers', async () => { - mockExecuteProviderRequest.mockRejectedValueOnce(new Error('down')).mockResolvedValueOnce({ - content: 'ok', - model: 'claude-sonnet-5', - tokens: { input: 1, output: 1, total: 2 }, - toolCalls: [], - cost: 0, - timing: { total: 1 }, - }) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) await handler.execute(mockContext, mockBlock, { ...baseInputs, @@ -688,6 +755,33 @@ describe('AgentBlockHandler', () => { ) }) + it('skips a fallback the workspace does not permit', async () => { + mockValidateModelProvider.mockImplementation(async (_user, _workspace, model: string) => { + if (model === 'claude-sonnet-5') throw new Error('Model not permitted') + }) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('gpt-4o-mini')) + + await handler.execute( + { ...mockContext, userId: 'user-1', workspaceId: 'workspace-1' }, + mockBlock, + { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }, { model: 'gpt-4o-mini' }], + } + ) + + expect(mockExecuteProviderRequest.mock.calls.map(([, request]) => request.model)).toEqual([ + 'gpt-4o', + 'gpt-4o-mini', + ]) + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Fallback model unusable; skipping', + expect.objectContaining({ model: 'claude-sonnet-5', error: 'Model not permitted' }) + ) + }) + it('does not fall back after a stop', async () => { const controller = new AbortController() mockExecuteProviderRequest.mockImplementationOnce(async () => { @@ -716,6 +810,83 @@ describe('AgentBlockHandler', () => { ).rejects.toBe(error) expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) }) + + it('keeps the fallback name when a routed sim-auto primary fails and a fallback answers', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('pool model down')) + .mockResolvedValueOnce(providerResponse('gpt-4o-mini')) + + const result = (await handler.execute(mockContext, mockBlock, { + model: SIM_AUTO_MODEL_ID, + systemPrompt: 'Be brief.', + userPrompt: 'Hello!', + fallbackModels: [{ model: 'gpt-4o-mini' }], + })) as { model: string } + + expect(result.model).toBe('gpt-4o-mini') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + /** The auto identity preamble belongs to the pool model, not a named fallback. */ + const systemText = (request: { messages?: Array<{ role: string; content: string }> }) => + (request.messages ?? []) + .filter((message) => message.role === 'system') + .map((message) => message.content) + .join('\n') + expect(systemText(mockExecuteProviderRequest.mock.calls[0][1])).toContain('Sim auto model') + expect(systemText(mockExecuteProviderRequest.mock.calls[1][1])).not.toContain( + 'Sim auto model' + ) + }) + + it('records the failed models on the open log entry, not an earlier closed one', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + const closed = openLog(mockBlock.id, '2026-01-01T00:00:01.000Z') + const open = openLog() + + await handler.execute({ ...mockContext, blockLogs: [closed, open] }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + + expect(closed.modelFallbacks).toBeUndefined() + expect(open.modelFallbacks).toEqual(['gpt-4o']) + }) + + it('falls back when a streaming primary fails before its first chunk, and replays the first chunk otherwise', async () => { + mockExecuteProviderRequest + .mockResolvedValueOnce( + streamingResponse([], { failBeforeFirstChunk: new Error('429 at stream start') }) + ) + .mockResolvedValueOnce(streamingResponse(['first', 'second'])) + + const result = (await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + })) as StreamingExecution + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.objectContaining({ failedModel: 'gpt-4o', error: '429 at stream start' }) + ) + expect(await drain(result.stream as ReadableStream)).toEqual(['first', 'second']) + }) + + it('leaves a failure after the first chunk to the stream, as before', async () => { + const midStream = new Error('dropped mid-stream') + mockExecuteProviderRequest.mockResolvedValueOnce( + streamingResponse(['first'], { failAfterFirstChunk: midStream }) + ) + + const result = (await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + })) as StreamingExecution + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + await expect(drain(result.stream as ReadableStream)).rejects.toBe(midStream) + }) }) describe('execute', () => { diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 530e24c3015..990a75f048b 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -40,6 +40,8 @@ import { } from '@/lib/uploads/utils/model-input' import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' import { + type FallbackModelCandidate, + isWholeEnvVarReference, normalizeFallbackModels, resolveFallbackTuning, } from '@/lib/workflows/blocks/fallback-models' @@ -144,10 +146,26 @@ interface IndexedToolInput { toolIndex: number } +/** + * Removes the sim-auto identity preamble from the system messages built for a + * routed primary. A fallback the builder named is not a pool model, so it must + * not be told to hide which model it is. Messages are built once per block run + * (building them appends to memory), which is why this strips rather than + * rebuilds. + */ +function stripAutoPreamble(messages: Message[] | undefined): Message[] | undefined { + if (!messages) return messages + const prefix = `${SIM_AUTO_SYSTEM_PREAMBLE}\n\n` + return messages.flatMap((message) => { + if (message.role !== 'system' || typeof message.content !== 'string') return [message] + if (message.content === SIM_AUTO_SYSTEM_PREAMBLE) return [] + if (!message.content.startsWith(prefix)) return [message] + return [{ ...message, content: message.content.slice(prefix.length) }] + }) +} + /** One model in the order the block tries them; the primary carries the block's own key. */ -interface ModelCandidate { - model: string - apiKey?: string +interface ModelCandidate extends FallbackModelCandidate { isPrimary: boolean } @@ -156,9 +174,17 @@ interface ExecuteAcrossModelsConfig { primaryModel: string primaryProviderId: string messages: Message[] | undefined + /** Provider id to hydrated messages; seeded with the primary, filled per fallback provider. */ + hydratedByProvider: Map fileProjection: ReturnType modelInputs: AgentInputs - formattedTools: any[] + /** + * The system prompt without the sim-auto identity preamble, present only when + * the primary was auto-routed: a fallback the builder named is not a pool + * model and must not be told to hide which model it is. + */ + fallbackSystemPrompt?: string + formattedTools: ProviderToolConfig[] responseFormat: any streaming: boolean settledInputRegistry: ResolvedSecretTraceRegistry | undefined @@ -419,6 +445,24 @@ export class AgentBlockHandler implements BlockHandler { skillMetadata, fileProjection ) + /** + * The primary hydrates before the registries settle and fork, as it always + * has: hydration imports file provenance into the live registry, and the + * result fork below must carry it. Fallbacks on another provider hydrate + * inside the chain and re-fork there. + */ + const hydratedByProvider = new Map([ + [ + providerId, + await this.hydrateMessageFilesForProvider( + ctx, + messagesWithInputFiles, + providerId, + fileProjection.projectedNameByFile, + fileProjection.modelBoundInputPaths + ), + ], + ]) settlePrivateAgentSelectors() @@ -438,19 +482,38 @@ export class AgentBlockHandler implements BlockHandler { } } + /** + * A follow-up turn of a deep-research interaction lives on the primary's + * provider; another model has none of that conversation, so a green answer + * from it would be built on a fresh context. Such a request never falls back. + */ + const fallbackCandidates = modelInputs.previousInteractionId + ? [] + : normalizeFallbackModels(filteredInputs.fallbackModels).filter( + (candidate) => candidate.model.toLowerCase() !== model.toLowerCase() + ) + if (modelInputs.previousInteractionId && filteredInputs.fallbackModels?.length) { + logger.info('Fallback models skipped for a deep-research follow-up turn', { + blockId: block.id, + }) + } const candidates: ModelCandidate[] = [ { model, apiKey: modelInputs.apiKey, isPrimary: true }, - ...normalizeFallbackModels(filteredInputs.fallbackModels) - .filter((candidate) => candidate.model.toLowerCase() !== model.toLowerCase()) - .map((candidate) => ({ ...candidate, isPrimary: false })), + ...fallbackCandidates.map((candidate) => ({ ...candidate, isPrimary: false })), ] - const { result, servedModel, failedModels } = await this.executeAcrossModels(ctx, block, { + const { + result, + servedModel, + resultRegistry: servedRegistry, + } = await this.executeAcrossModels(ctx, block, { candidates, primaryModel: model, primaryProviderId: providerId, messages: messagesWithInputFiles, + hydratedByProvider, fileProjection, modelInputs, + fallbackSystemPrompt: autoRouting ? filteredInputs.systemPrompt : undefined, formattedTools: formatted.tools, responseFormat, streaming: streamingConfig.shouldUseStreaming ?? false, @@ -458,8 +521,7 @@ export class AgentBlockHandler implements BlockHandler { resultRegistry, providerErrorRegistry, }) - if (failedModels.length > 0) this.recordModelFallbacks(ctx, block, failedModels) - if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry + if (servedRegistry) ctx.resolvedSecretTraceRegistry = servedRegistry if (autoRouting && autoRouting.billableRoutingCost > 0) { this.applyRoutingCost(result, autoRouting.billableRoutingCost) @@ -2354,9 +2416,17 @@ export class AgentBlockHandler implements BlockHandler { * blacklisted, not permitted, or cannot take the attachments is skipped * rather than counted as a failed try. * + * A streaming candidate is accepted only once its first chunk has arrived + * (`primeStreamingExecution`), so a startup failure inside the stream still + * falls through; that wait is skipped when no candidate follows, which keeps + * blocks without fallbacks on today's path. + * * When every candidate fails, the last attempted candidate's error is thrown - * exactly as it escaped `executeProviderRequest`, so error ports and the - * block-level error handling see the shapes they see today. + * exactly as it escaped `executeProviderRequest`, with the registries that + * call installed for its projection, so error ports and the block-level + * error handling see the shapes they see today. The models that failed are + * written to the block log on every exit, cleared as well as set, because + * block retry reuses one log entry across tries. */ private async executeAcrossModels( ctx: ExecutionContext, @@ -2365,16 +2435,26 @@ export class AgentBlockHandler implements BlockHandler { ): Promise<{ result: BlockOutput | StreamingExecution servedModel: string - failedModels: string[] + resultRegistry: ResolvedSecretTraceRegistry | undefined }> { - const hydratedByProvider = new Map() + const { hydratedByProvider } = config + let resultRegistry = config.resultRegistry const failedModels: string[] = [] let lastError: unknown + let lastErrorRegistries: + | { + error: ResolvedSecretTraceRegistry | undefined + resolved: ResolvedSecretTraceRegistry | undefined + } + | undefined for (let index = 0; index < config.candidates.length; index++) { const candidate = config.candidates[index] const hasNext = index < config.candidates.length - 1 + /** A run stopped while a candidate was being skipped must not start another. */ + if (!candidate.isPrimary && ctx.abortSignal?.aborted) break + let candidateProviderId: string if (candidate.isPrimary) { candidateProviderId = config.primaryProviderId @@ -2420,6 +2500,8 @@ export class AgentBlockHandler implements BlockHandler { continue } hydratedByProvider.set(candidateProviderId, messages) + /** Hydration imported this provider's file provenance; the result fork must carry it. */ + resultRegistry = config.settledInputRegistry?.forkForInputPaths([]) } /** @@ -2437,14 +2519,32 @@ export class AgentBlockHandler implements BlockHandler { config.primaryModel, config.modelInputs ) + /** + * A row key still in `{{NAME}}` form was never resolved: the variable is + * not set for the principal running this workflow. Sending the literal + * would only replace a platform or BYOK key with garbage, so it counts + * as no key at all. + */ + let rowKey = candidate.apiKey + if (rowKey && isWholeEnvVarReference(rowKey)) { + logger.warn('Fallback key variable is not set for this run', { + blockId: block.id, + model: candidate.model, + variable: rowKey, + }) + rowKey = undefined + } inputs = { ...config.modelInputs, apiKey: - candidate.apiKey ?? + rowKey ?? (candidateProviderId === config.primaryProviderId ? config.modelInputs.apiKey : undefined), previousInteractionId: undefined, + ...(config.fallbackSystemPrompt !== undefined + ? { systemPrompt: config.fallbackSystemPrompt } + : {}), ...tuning, } if (adjustments.length > 0) { @@ -2463,7 +2563,10 @@ export class AgentBlockHandler implements BlockHandler { ctx, providerId: candidateProviderId, model: candidate.model, - messages, + messages: + !candidate.isPrimary && config.fallbackSystemPrompt !== undefined + ? stripAutoPreamble(messages) + : messages, inputs, formattedTools: config.formattedTools, responseFormat: config.responseFormat, @@ -2471,15 +2574,19 @@ export class AgentBlockHandler implements BlockHandler { }) try { - const result = await this.executeProviderRequest( + let result = await this.executeProviderRequest( ctx, providerRequest, block, config.responseFormat, - config.resultRegistry, + resultRegistry, config.providerErrorRegistry ) - return { result, servedModel: candidate.model, failedModels } + if (hasNext && this.isStreamingExecution(result)) { + result = await this.primeStreamingExecution(result as StreamingExecution) + } + this.recordModelFallbacks(ctx, block, failedModels) + return { result, servedModel: candidate.model, resultRegistry } } catch (error) { lastError = error failedModels.push(candidate.model) @@ -2489,16 +2596,20 @@ export class AgentBlockHandler implements BlockHandler { } /** - * `executeProviderRequest` swapped both registries for its error - * projection; the next candidate starts from the settled inputs again. + * `executeProviderRequest` installed the failed attempt's error + * registry, which is the only one that knows secrets a tool call + * activated; the warn is projected against it before the next candidate + * starts from the settled inputs again. The pair is kept so a rethrow + * after every remaining candidate was skipped projects the same way. */ - ctx.errorResolvedSecretTraceRegistry = config.providerErrorRegistry - ctx.resolvedSecretTraceRegistry = config.settledInputRegistry - + const errorRegistry = ctx.errorResolvedSecretTraceRegistry + const diagnosticCtx = errorRegistry + ? { ...ctx, resolvedSecretTraceRegistry: errorRegistry } + : ctx logger.warn( 'Agent model failed; trying fallback', projectAgentDiagnosticMetadata( - ctx, + diagnosticCtx, { blockId: block.id, failedModel: candidate.model, @@ -2509,34 +2620,91 @@ export class AgentBlockHandler implements BlockHandler { { blockId: block.id, attempt: index + 1 } ) ) + lastErrorRegistries = { error: errorRegistry, resolved: ctx.resolvedSecretTraceRegistry } + ctx.errorResolvedSecretTraceRegistry = config.providerErrorRegistry + ctx.resolvedSecretTraceRegistry = config.settledInputRegistry } } /** Reached only when every candidate after the last failure was skipped. */ + if (lastErrorRegistries) { + ctx.errorResolvedSecretTraceRegistry = lastErrorRegistries.error + ctx.resolvedSecretTraceRegistry = lastErrorRegistries.resolved + } this.recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) throw lastError } + /** + * Waits for a streaming candidate's first chunk before accepting it. + * + * With tools attached, providers open the stream first and issue the initial + * upstream request inside it, so a 429 at startup would otherwise surface + * only when the executor drains the stream, past every fallback. Reading one + * chunk moves that failure back inside the candidate loop; the chunk is + * re-emitted at the head of the returned stream, and nothing has reached the + * client yet, so the next candidate cannot duplicate output. A stream that + * fails after its first chunk stays a stream failure, as it is today. + */ + private async primeStreamingExecution(result: StreamingExecution): Promise { + const reader = result.stream.getReader() + const first = await reader.read() + const stream = new ReadableStream({ + start(controller) { + if (first.done) { + controller.close() + return + } + controller.enqueue(first.value) + }, + async pull(controller) { + const next = await reader.read() + if (next.done) { + controller.close() + return + } + controller.enqueue(next.value) + }, + cancel(reason) { + return reader.cancel(reason) + }, + }) + return { ...result, stream } + } + /** * Writes the models that failed onto the block's open log entry so the trace * can show them beside the model that answered. Handlers get no log handle; * the executor pushes the entry before running the handler with `endedAt` * still empty, which is what tells it apart from earlier runs of the same - * block in a loop or an earlier retry. + * block in a loop or an earlier retry. An empty list clears the field, since + * a retry that succeeds on the primary reuses the entry a failed try wrote. + * + * A model id can itself come from a resolved reference, so the names are + * projected through the same secret registry as every other diagnostic and + * left off the log entirely when the projection is not safe. */ private recordModelFallbacks( ctx: ExecutionContext, block: SerializedBlock, failedModels: string[] ): void { - if (failedModels.length === 0) return const logs = ctx.blockLogs ?? [] for (let index = logs.length - 1; index >= 0; index--) { const entry = logs[index] - if (entry.blockId === block.id && entry.endedAt === '') { - entry.modelFallbacks = [...failedModels] + if (entry.blockId !== block.id || entry.endedAt !== '') continue + if (failedModels.length === 0) { + entry.modelFallbacks = undefined return } + const registry = ctx.errorResolvedSecretTraceRegistry ?? ctx.resolvedSecretTraceRegistry + const projection = projectResolvedSecretDiagnosticContent({ models: failedModels }, registry) + const models = projection.safe ? (projection.value as { models?: unknown }).models : undefined + entry.modelFallbacks = + Array.isArray(models) && models.every((model) => typeof model === 'string') + ? [...models] + : undefined + return } } diff --git a/apps/sim/executor/handlers/agent/types.ts b/apps/sim/executor/handlers/agent/types.ts index 460dd676153..f752859e34b 100644 --- a/apps/sim/executor/handlers/agent/types.ts +++ b/apps/sim/executor/handlers/agent/types.ts @@ -1,4 +1,5 @@ import type { McpOperationPolicy } from '@/lib/mcp/operation-policy' +import type { FallbackModelEntry } from '@/lib/workflows/blocks/fallback-models' import type { UserFile } from '@/executor/types' import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry' @@ -49,7 +50,7 @@ export interface AgentInputs { promptCaching?: boolean files?: unknown /** Ordered models tried when the request to `model` fails; see `normalizeFallbackModels`. */ - fallbackModels?: Array<{ id?: string; model: string; apiKey?: string }> + fallbackModels?: Array & { model: string }> } /** diff --git a/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts b/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts index 09a0c25858c..70444e6cb47 100644 --- a/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts +++ b/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts @@ -1027,6 +1027,36 @@ describe('buildTraceSpans', () => { }) }) +describe('modelFallbacks', () => { + const log = (modelFallbacks?: string[]) => ({ + blockId: 'agent-1', + blockName: 'Agent', + blockType: 'agent', + startedAt: '2024-01-01T10:00:00.000Z', + endedAt: '2024-01-01T10:00:01.000Z', + durationMs: 1000, + success: true, + output: { content: 'ok', model: 'gpt-5.4-mini' }, + executionOrder: 1, + ...(modelFallbacks ? { modelFallbacks } : {}), + }) + + it.concurrent('carries the failed models onto the span and omits the field when empty', () => { + const withFallbacks = buildTraceSpans({ + success: true, + output: {}, + logs: [log(['claude-sonnet-5'])], + }) + expect(withFallbacks.traceSpans[0].modelFallbacks).toEqual(['claude-sonnet-5']) + expect(withFallbacks.traceSpans[0].model).toBe('gpt-5.4-mini') + + const empty = buildTraceSpans({ success: true, output: {}, logs: [log([])] }) + expect(empty.traceSpans[0]).not.toHaveProperty('modelFallbacks') + const none = buildTraceSpans({ success: true, output: {}, logs: [log()] }) + expect(none.traceSpans[0]).not.toHaveProperty('modelFallbacks') + }) +}) + describe('errorHandled - handled errors should not bubble up', () => { it.concurrent('block span stays error but is marked errorHandled', () => { const result: ExecutionResult = { diff --git a/apps/sim/lib/workflows/blocks/fallback-models.test.ts b/apps/sim/lib/workflows/blocks/fallback-models.test.ts index 2484f075c7e..8018426b60c 100644 --- a/apps/sim/lib/workflows/blocks/fallback-models.test.ts +++ b/apps/sim/lib/workflows/blocks/fallback-models.test.ts @@ -47,6 +47,10 @@ vi.mock('@/providers/models', () => ({ })) import { + addFallbackRow, + changeFallbackRowApiKey, + changeFallbackRowModel, + changeFallbackRowTuning, fallbackRowNeedsApiKey, getFallbackTuningKnobsToShow, getTuningOptionsForModel, @@ -54,8 +58,11 @@ import { isViableFallbackModel, isWholeEnvVarReference, MAX_FALLBACK_MODELS, + moveFallbackRow, normalizeFallbackModels, + normalizeTuningValues, ordinalChoiceLabel, + removeFallbackRow, resolveFallbackTuning, } from '@/lib/workflows/blocks/fallback-models' @@ -292,6 +299,94 @@ describe('resolveFallbackTuning', () => { }) }) +describe('resolveFallbackTuning hidden overrides', () => { + it('ignores a stored row value once the primary value fits and the field is no longer shown', () => { + const resolved = resolveFallbackTuning( + { model: 'gpt-small', reasoningEffort: 'low' }, + 'gpt-big', + { + reasoningEffort: 'high', + } + ) + expect(resolved.reasoningEffort).toBe('high') + expect(resolved.adjustments).toEqual([]) + }) +}) + +describe('normalizeTuningValues', () => { + it('keeps trimmed lower-cased strings and drops blanks and non-strings', () => { + expect( + normalizeTuningValues({ + reasoningEffort: ' Low ', + thinkingLevel: '', + verbosity: 3, + model: 'x', + }) + ).toEqual({ reasoningEffort: 'low' }) + }) +}) + +describe('row transforms', () => { + const rows = [ + { id: 'a', model: 'gpt-big' }, + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}', reasoningEffort: 'low' }, + ] + + it('adds a blank row until the cap and never past it', () => { + expect(addFallbackRow(rows, 'c')).toEqual([...rows, { id: 'c', model: '' }]) + const full = Array.from({ length: MAX_FALLBACK_MODELS }, (_, i) => ({ + id: `r${i}`, + model: 'm', + })) + expect(addFallbackRow(full, 'extra')).toBe(full) + }) + + it('removes by id and moves within bounds', () => { + expect(removeFallbackRow(rows, 'a')).toEqual([rows[1]]) + expect(moveFallbackRow(rows, 'b', -1)).toEqual([rows[1], rows[0]]) + expect(moveFallbackRow(rows, 'a', -1)).toBe(rows) + expect(moveFallbackRow(rows, 'b', 1)).toBe(rows) + expect(moveFallbackRow(rows, 'missing', 1)).toBe(rows) + }) + + it('clears tuning on a model change and keeps the key only for the same keyed provider', () => { + mockShouldRequireApiKey.mockReturnValue(true) + expect(changeFallbackRowModel(rows, 'b', 'openrouter/y', 'claude-sonnet-5')[1]).toEqual({ + id: 'b', + model: 'openrouter/y', + apiKey: '{{OPENROUTER_API_KEY}}', + }) + /** Another provider must never receive the previous provider's credential. */ + expect(changeFallbackRowModel(rows, 'b', 'gpt-small', 'claude-sonnet-5')[1]).toEqual({ + id: 'b', + model: 'gpt-small', + }) + mockShouldRequireApiKey.mockReturnValue(false) + expect(changeFallbackRowModel(rows, 'b', 'openrouter/y', 'claude-sonnet-5')[1]).toEqual({ + id: 'b', + model: 'openrouter/y', + }) + }) + + it('stores a tuning value, and the provider-decides entry as absence', () => { + expect(changeFallbackRowTuning(rows, 'a', 'reasoningEffort', 'high')[0]).toEqual({ + id: 'a', + model: 'gpt-big', + reasoningEffort: 'high', + }) + expect(changeFallbackRowTuning(rows, 'b', 'reasoningEffort', 'auto')[1]).toEqual({ + id: 'b', + model: 'openrouter/x', + apiKey: '{{OPENROUTER_API_KEY}}', + }) + expect(changeFallbackRowApiKey(rows, 'a', '{{K}}')[0]).toEqual({ + id: 'a', + model: 'gpt-big', + apiKey: '{{K}}', + }) + }) +}) + describe('ordinalChoiceLabel', () => { it('starts at the 2nd choice and handles English ordinals', () => { expect([0, 1, 2, 3, 9, 10, 11].map(ordinalChoiceLabel)).toEqual([ diff --git a/apps/sim/lib/workflows/blocks/fallback-models.ts b/apps/sim/lib/workflows/blocks/fallback-models.ts index 9f21075e58e..ca97edab34b 100644 --- a/apps/sim/lib/workflows/blocks/fallback-models.ts +++ b/apps/sim/lib/workflows/blocks/fallback-models.ts @@ -89,21 +89,98 @@ export function normalizeFallbackModels(raw: unknown): FallbackModelCandidate[] if (seen.has(key)) continue seen.add(key) const resolvedKey = typeof apiKey === 'string' ? apiKey.trim() : '' - const candidate: FallbackModelCandidate = { + candidates.push({ model: trimmed, ...(resolvedKey ? { apiKey: resolvedKey } : {}), - } - for (const knob of FALLBACK_TUNING_KNOBS) { - const value = (row as Record)[knob] - const level = typeof value === 'string' ? value.trim().toLowerCase() : '' - if (level) candidate[knob] = level - } - candidates.push(candidate) + ...normalizeTuningValues(row as Record), + }) if (candidates.length >= MAX_FALLBACK_MODELS) break } return candidates } +/** The tuning knobs a row carries, trimmed and lower-cased; blanks and non-strings are dropped. */ +export function normalizeTuningValues(row: Record): FallbackTuningValues { + const values: FallbackTuningValues = {} + for (const knob of FALLBACK_TUNING_KNOBS) { + const value = row[knob] + const level = typeof value === 'string' ? value.trim().toLowerCase() : '' + if (level) values[knob] = level + } + return values +} + +/** + * The edits the editor makes to a fallback list, as pure transforms so the + * component stays a thin binding and the rules are unit-testable. + */ +export function addFallbackRow(rows: FallbackModelEntry[], id: string): FallbackModelEntry[] { + if (rows.length >= MAX_FALLBACK_MODELS) return rows + return [...rows, { id, model: '' }] +} + +export function removeFallbackRow(rows: FallbackModelEntry[], id: string): FallbackModelEntry[] { + return rows.filter((row) => row.id !== id) +} + +export function moveFallbackRow( + rows: FallbackModelEntry[], + id: string, + direction: -1 | 1 +): FallbackModelEntry[] { + const index = rows.findIndex((row) => row.id === id) + const target = index + direction + if (index === -1 || target < 0 || target >= rows.length) return rows + const next = [...rows] + ;[next[index], next[target]] = [next[target], next[index]] + return next +} + +/** + * A new model gets a clean row. Tuning always goes, since the new model may not + * declare it. The key survives only when the new model still needs one and sits + * on the same provider as the old one: a key reference is a credential for one + * provider, and carrying it to another would send that provider's secret to an + * unrelated service. + */ +export function changeFallbackRowModel( + rows: FallbackModelEntry[], + id: string, + model: string, + primaryModel: string +): FallbackModelEntry[] { + return rows.map((row) => { + if (row.id !== id) return row + const keepKey = + row.apiKey && + fallbackRowNeedsApiKey(model, primaryModel) && + findProviderFromModel(model.trim()) === findProviderFromModel(row.model.trim()) + return { id: row.id, model, ...(keepKey ? { apiKey: row.apiKey } : {}) } + }) +} + +export function changeFallbackRowApiKey( + rows: FallbackModelEntry[], + id: string, + apiKey: string +): FallbackModelEntry[] { + return rows.map((row) => (row.id === id ? { ...row, apiKey } : row)) +} + +/** The provider-decides entry is the field's default, so it is stored as absence. */ +export function changeFallbackRowTuning( + rows: FallbackModelEntry[], + id: string, + knob: FallbackTuningKnob, + value: string +): FallbackModelEntry[] { + return rows.map((row) => { + if (row.id !== id) return row + const { [knob]: _previous, ...rest } = row + return value && value !== KNOB_SENTINEL[knob] ? { ...rest, [knob]: value } : rest + }) +} + /** * Whether a model can serve as a fallback for `primaryModel` with the * credentials the block can actually give it. @@ -223,7 +300,10 @@ function clampToCap( /** * The tuning a fallback candidate runs with. * - * Graded knobs: the row's own value wins; otherwise the primary's value is + * Graded knobs: the row's own value wins, but only for a knob the row is + * currently asked about (`getFallbackTuningKnobsToShow`), so a value stored + * while the primary was incompatible stops applying once the primary's own + * value fits and the field is no longer shown. Otherwise the primary's value is * carried over only when the fallback declares it, and dropped to the * provider's default otherwise, which is what the row field exists to override. * Temperature and max output tokens are caps in the primary's terms, so they @@ -239,9 +319,10 @@ export function resolveFallbackTuning( ): ResolvedFallbackTuning { const adjustments: string[] = [] const resolved: ResolvedFallbackTuning = { adjustments } + const overridable = new Set(getFallbackTuningKnobsToShow(candidate.model, primaryModel, primary)) for (const knob of FALLBACK_TUNING_KNOBS) { - const own = candidate[knob] + const own = overridable.has(knob) ? candidate[knob] : undefined if (own) { resolved[knob] = own if (own !== primary[knob]) adjustments.push(`${knob}: ${primary[knob] ?? 'unset'} -> ${own}`) diff --git a/apps/sim/lib/workflows/credentials/credential-extractor.test.ts b/apps/sim/lib/workflows/credentials/credential-extractor.test.ts index 146d1fb07cd..c4f5a7c15e8 100644 --- a/apps/sim/lib/workflows/credentials/credential-extractor.test.ts +++ b/apps/sim/lib/workflows/credentials/credential-extractor.test.ts @@ -102,17 +102,40 @@ describe('export sanitizer resource coverage', () => { expect(sanitizedValue('oauth-input', 'cred-123')).toBeNull() }) - it('keeps fallback models and only env-var-referenced row keys', () => { + it('keeps fallback models and only whole env-var-referenced row keys', () => { expect( sanitizedValue('model-fallback-list', [ { id: 'a', model: 'gpt-5' }, { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, { id: 'c', model: 'openrouter/y', apiKey: 'sk-raw-secret' }, + { id: 'd', model: 'openrouter/z', apiKey: '{{A}} sk-raw {{B}}' }, + 'not-a-row', ]) ).toEqual([ { id: 'a', model: 'gpt-5' }, { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, { id: 'c', model: 'openrouter/y' }, + { id: 'd', model: 'openrouter/z' }, + 'not-a-row', + ]) + expect(sanitizedValue('model-fallback-list', 'opaque')).toBe('opaque') + }) + + it('drops even referenced fallback row keys when env vars are not preserved', () => { + vi.mocked(getBlock).mockReturnValue({ + name: 'Test', + description: '', + subBlocks: [{ id: 'field', title: 'Field', type: 'model-fallback-list' }], + outputs: {}, + } as never) + const sanitized = sanitizeWorkflowForSharing( + stateWithSubBlock('model-fallback-list', [ + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + ]), + { preserveEnvVars: false, redactOpaqueCredentialInputs: true } + ) + expect(sanitized.blocks?.b1?.subBlocks?.field?.value).toEqual([ + { id: 'b', model: 'openrouter/x' }, ]) }) diff --git a/apps/sim/lib/workflows/credentials/credential-extractor.ts b/apps/sim/lib/workflows/credentials/credential-extractor.ts index 49b02a80b5a..436125ea952 100644 --- a/apps/sim/lib/workflows/credentials/credential-extractor.ts +++ b/apps/sim/lib/workflows/credentials/credential-extractor.ts @@ -1,4 +1,5 @@ import { isPlainRecord } from '@sim/utils/object' +import { isWholeEnvVarReference } from '@/lib/workflows/blocks/fallback-models' import { coerceObjectArray } from '@/lib/workflows/persistence/remap-internal-ids' import { getToolInputParamConfigs } from '@/lib/workflows/search-replace/indexer' import { WORKFLOW_SEARCH_SUBBLOCK_RESOURCE_TYPES } from '@/lib/workflows/search-replace/resources/registry' @@ -184,9 +185,7 @@ function sanitizeFallbackModelsValue( return value.map((row) => { if (!row || typeof row !== 'object' || Array.isArray(row)) return row const { apiKey, ...rest } = row as Record - return options.preserveEnvVars && isEnvironmentVariableReference(apiKey) - ? { ...rest, apiKey } - : rest + return options.preserveEnvVars && isWholeEnvVarReference(apiKey) ? { ...rest, apiKey } : rest }) } diff --git a/apps/sim/lib/workflows/editing/validation.ts b/apps/sim/lib/workflows/editing/validation.ts index 75318f6460c..6d85d8a0999 100644 --- a/apps/sim/lib/workflows/editing/validation.ts +++ b/apps/sim/lib/workflows/editing/validation.ts @@ -16,6 +16,7 @@ import { isTuningValueValidForModel, isWholeEnvVarReference, MAX_FALLBACK_MODELS, + normalizeTuningValues, } from '@/lib/workflows/blocks/fallback-models' import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' import { validateSelectorIds } from '@/lib/workflows/editing/selector-validator' @@ -369,6 +370,23 @@ function validateAgentToolEntry(item: any, index: number): string | null { * Skills are a SEPARATE array from tools; each entry references a workspace or * builtin skill by `skillId`. Returns an error string or null when valid. */ +function validateAgentSkillEntry(item: any, index: number): string | null { + const where = `skills[${index}]` + if (item === null || typeof item !== 'object' || Array.isArray(item)) { + return `${where} must be a skill object like {"skillId":"","name":""}` + } + if (typeof item.skillId !== 'string' || item.skillId.trim() === '') { + if (typeof item.id === 'string') { + return `${where} uses "id" but skills require "skillId" (the "id" from agent/skills/{name}.json)` + } + if (typeof item.type === 'string' || item.schema || item.customToolId) { + return `${where} looks like a tool entry. Skills go in the SEPARATE "skills" array and need only {"skillId":""} - no "type"/"schema"/"customToolId"` + } + return `${where} must include "skillId" (the "id" from agent/skills/{name}.json)` + } + return null +} + /** * Validates one fallback-model row. Returns an error string or null when valid. * @@ -413,23 +431,6 @@ function validateFallbackModelEntry(item: any, index: number): string | null { return null } -function validateAgentSkillEntry(item: any, index: number): string | null { - const where = `skills[${index}]` - if (item === null || typeof item !== 'object' || Array.isArray(item)) { - return `${where} must be a skill object like {"skillId":"","name":""}` - } - if (typeof item.skillId !== 'string' || item.skillId.trim() === '') { - if (typeof item.id === 'string') { - return `${where} uses "id" but skills require "skillId" (the "id" from agent/skills/{name}.json)` - } - if (typeof item.type === 'string' || item.schema || item.customToolId) { - return `${where} looks like a tool entry. Skills go in the SEPARATE "skills" array and need only {"skillId":""} - no "type"/"schema"/"customToolId"` - } - return `${where} must include "skillId" (the "id" from agent/skills/{name}.json)` - } - return null -} - /** * Validates a value against its expected subBlock type * Returns validation result with the value or an error @@ -680,11 +681,7 @@ export function validateValueForSubBlockType( id: typeof item.id === 'string' && item.id ? item.id : generateShortId(), model: item.model.trim(), ...(isWholeEnvVarReference(item.apiKey) ? { apiKey: item.apiKey.trim() } : {}), - ...Object.fromEntries( - FALLBACK_TUNING_KNOBS.filter( - (knob) => typeof item[knob] === 'string' && (item[knob] as string).trim() !== '' - ).map((knob) => [knob, (item[knob] as string).trim().toLowerCase()]) - ), + ...normalizeTuningValues(item), })), } } diff --git a/apps/sim/lib/workflows/search-replace/indexer.test.ts b/apps/sim/lib/workflows/search-replace/indexer.test.ts index 1ef6f8e6ecb..861b4fa4ec5 100644 --- a/apps/sim/lib/workflows/search-replace/indexer.test.ts +++ b/apps/sim/lib/workflows/search-replace/indexer.test.ts @@ -1450,6 +1450,14 @@ describe('indexWorkflowSearchMatches', () => { searchText: 'fallback-visible-model', }), ]) + /** A row key is a `{{VAR}}` reference; text search must never offer to rewrite it. */ + const keyMatches = indexWorkflowSearchMatches({ + workflow, + query: 'HIDDEN_KEY_REF', + mode: 'text', + blockConfigs, + }).filter((match) => match.blockId === 'structured-1') + expect(keyMatches).toEqual([]) expect(containsMatches).toEqual([ expect.objectContaining({ subBlockId: 'filters', From 0b35cd7c9bf11fac22b1ba90248713464bee3998 Mon Sep 17 00:00:00 2001 From: Marcus Chandra Date: Fri, 18 Sep 2026 00:23:01 -0700 Subject: [PATCH 08/14] fix(agent): retry the selected model before falling back Retry on fail used to wrap the whole fallback chain, so tries 3 with fallbacks B and C ran A, B, C three times over. A builder who lists fallbacks wants the selected model retried and the fallbacks tried once each after its last try fails, which is also how LiteLLM orders retries and fallbacks and how OpenRouter treats each model in its list. The executor keeps the retry policy. Each try is now told where it sits in it through the node metadata (`BlockNodeMetadata.retry`, with the executor's own `isFinalTry` judgment), and the Agent handler keeps the fallbacks out of the candidate list until the final try. Every earlier try runs the primary alone and lets the failure escape for the policy to replay. Blocks without fallbacks, and blocks with retry off, behave as before; other handlers ignore the field. --- .../content/docs/workflows/blocks/agent.mdx | 6 +- apps/sim/blocks/blocks/agent.ts | 2 +- .../execution/block-executor.retry.test.ts | 30 +++++++++ apps/sim/executor/execution/block-executor.ts | 27 +++++--- .../handlers/agent/agent-handler.test.ts | 67 +++++++++++++++++++ .../executor/handlers/agent/agent-handler.ts | 44 +++++++++--- apps/sim/executor/types.ts | 16 +++++ 7 files changed, 169 insertions(+), 23 deletions(-) diff --git a/apps/docs/content/docs/workflows/blocks/agent.mdx b/apps/docs/content/docs/workflows/blocks/agent.mdx index ef83c5aad43..2418edd652c 100644 --- a/apps/docs/content/docs/workflows/blocks/agent.mdx +++ b/apps/docs/content/docs/workflows/blocks/agent.mdx @@ -87,8 +87,8 @@ Some settings live under advanced, or appear only for models that support them: - **Reasoning effort / Thinking level.** For models with extended reasoning, how much the model thinks before answering. Higher is more thorough but slower and costs more tokens. - **Prompt caching.** For Anthropic Claude models, reuses the system prompt and tool definitions between runs instead of re-reading them every time. Cached input costs a tenth of the normal rate, but writing the cache costs 1.25x, so leave it off for one-off runs and turn it on when the same agent runs repeatedly. The cache covers a prefix only if it reaches 1,024 tokens (2,048 on Haiku) — below that Anthropic ignores it and nothing changes. Entries expire after five minutes of no use. - **API key.** Your key for the chosen provider. Hidden on hosted Sim, which supplies one. -- **Fallback models.** An ordered list of models to try when the request to the selected model fails, whether the provider is overloaded, rate-limited, or down. Sim tries the 2nd choice, then the 3rd, and so on, and `` reports the model that answered. Hosted models need no setup. A model that needs its own key takes it from a workspace environment variable you pick on the row; a model on the same provider as the selected model reuses the block's key. Azure, Bedrock, and Vertex models can only be fallbacks for a selected model of the same family, since they use that model's credentials. The Auto model cannot be a fallback. A fallback runs with the selected model's settings where its provider accepts them: temperature and max output tokens are clamped to the fallback's limits, and when the fallback has a reasoning effort, thinking level, or verbosity setting that the selected model's value does not fit, the row shows that field so you can pick a value for it, otherwise the provider's default applies. -- **Retry on fail.** Runs the block again after a failure, up to a maximum number of tries with a wait between them. Fallback models work inside each try: one try walks the selected model and then every fallback, and only when all of them fail does the next try begin. A failure that happens after the model already called a tool runs that conversation again on the next model, so keep fallbacks and retry off for agents whose tools must not repeat. +- **Fallback models.** An ordered list of models to try when the request to the selected model fails, whether the provider is overloaded, rate-limited, or down. Sim tries the 2nd choice, then the 3rd, and so on, once each, and `` reports the model that answered. Hosted models need no setup. A model that needs its own key takes it from a workspace environment variable you pick on the row; a model on the same provider as the selected model reuses the block's key. Azure, Bedrock, and Vertex models can only be fallbacks for a selected model of the same family, since they use that model's credentials. The Auto model cannot be a fallback. A fallback runs with the selected model's settings where its provider accepts them: temperature and max output tokens are clamped to the fallback's limits, and when the fallback has a reasoning effort, thinking level, or verbosity setting that the selected model's value does not fit, the row shows that field so you can pick a value for it, otherwise the provider's default applies. +- **Retry on fail.** Retries the selected model after a failure, up to a maximum number of tries with a wait between them. When its tries run out, the fallback models are tried in order, once each, with no wait before the first of them. A fallback is never retried. A failure that happens after the model already called a tool runs that conversation again on the next try or the next model, so keep fallbacks and retry off for agents whose tools must not repeat. OpenAI and Gemini cache automatically at no extra cost and need no setting; their discount is already reflected in what you are charged. @@ -150,5 +150,5 @@ The Agent reads the message from Start with `` and returns a result diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index d715b76cb9c..a331e4c2963 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -37,7 +37,7 @@ const logger = createLogger('AgentBlock') /** Model the agent block falls back to when `model` is unset or the auto pseudo-model. */ const AGENT_FALLBACK_MODEL = 'claude-sonnet-5' -const FALLBACK_MODELS_DESCRIPTION = `Ordered models tried in sequence when the request to the selected model fails. Each row is { model, apiKey?, reasoningEffort?, thinkingLevel?, verbosity? }; apiKey, when present, must be a whole {{ENV_VAR}} reference, and a tuning value must be one the row model declares. sim-auto is not allowed. Max ${MAX_FALLBACK_MODELS}.` +const FALLBACK_MODELS_DESCRIPTION = `Ordered models tried in sequence, once each, when the request to the selected model fails; with Retry on fail, after the selected model's tries run out. Each row is { model, apiKey?, reasoningEffort?, thinkingLevel?, verbosity? }; apiKey, when present, must be a whole {{ENV_VAR}} reference, and a tuning value must be one the row model declares. sim-auto is not allowed. Max ${MAX_FALLBACK_MODELS}.` const MODELS_WITH_REASONING_EFFORT = getModelsWithReasoningEffort() const MODELS_WITH_VERBOSITY = getModelsWithVerbosity() const MODELS_WITH_THINKING = getModelsWithThinking() diff --git a/apps/sim/executor/execution/block-executor.retry.test.ts b/apps/sim/executor/execution/block-executor.retry.test.ts index e22c93104eb..29ac410620a 100644 --- a/apps/sim/executor/execution/block-executor.retry.test.ts +++ b/apps/sim/executor/execution/block-executor.retry.test.ts @@ -121,6 +121,36 @@ describe('BlockExecutor retry', () => { expect(execute).toHaveBeenCalledTimes(1) }) + it('tells each try where it sits in the policy, and a block without one nothing', async () => { + const block = createBlock({ enabled: true, maxTries: 3, waitBetweenTriesMs: 0 }) + const execute = vi + .fn() + .mockRejectedValueOnce(new Error('one')) + .mockRejectedValueOnce(new Error('two')) + .mockResolvedValueOnce({ ok: true }) + const state = new ExecutionState() + const executor = buildExecutor(block, { canHandle: () => true, execute }, state) + + await executor.execute(createContext(state), createNode(block), block) + + expect(execute.mock.calls.map(([, , , metadata]) => metadata.retry)).toEqual([ + { attempt: 1, maxTries: 3, isFinalTry: false }, + { attempt: 2, maxTries: 3, isFinalTry: false }, + { attempt: 3, maxTries: 3, isFinalTry: true }, + ]) + expect(execute.mock.calls[0][3].nodeId).toBe(block.id) + + const plain = createBlock() + const executePlain = vi.fn().mockResolvedValue({ ok: true }) + const plainState = new ExecutionState() + await buildExecutor( + plain, + { canHandle: () => true, execute: executePlain }, + plainState + ).execute(createContext(plainState), createNode(plain), plain) + expect(executePlain.mock.calls[0][3]).not.toHaveProperty('retry') + }) + it('replays any failure and succeeds on a later try', async () => { const block = createBlock(enabled) const execute = vi diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index e19b7a464ac..accb97dbe93 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -44,6 +44,7 @@ import { import { type BlockHandler, type BlockLog, + type BlockRetryAttempt, type BlockState, type ExecutionContext, getNextExecutionOrder, @@ -277,11 +278,12 @@ export class BlockExecutor { * token is drained, so a replay cannot duplicate output the client has * already seen. */ - const output = await this.runHandlerWithRetry(blockCtx, block, blockLog, () => - handler.executeWithNode - ? handler.executeWithNode(blockCtx, block, resolvedInputs, nodeMetadata) - : handler.execute(blockCtx, block, resolvedInputs, nodeMetadata) - ) + const output = await this.runHandlerWithRetry(blockCtx, block, blockLog, (retry) => { + const invocationMetadata = retry ? { ...nodeMetadata, retry } : nodeMetadata + return handler.executeWithNode + ? handler.executeWithNode(blockCtx, block, resolvedInputs, invocationMetadata) + : handler.execute(blockCtx, block, resolvedInputs, invocationMetadata) + }) completedHandlerCost = readTrustedExecutionCost(output) @@ -546,15 +548,20 @@ export class BlockExecutor { * Rethrows the final try's error so the caller's catch — and with it the error * port — behaves exactly as it does for a block that never retried. Retrying * only ever delays the existing outcome; it never changes it. + * + * Each try is told where it sits in the policy (`BlockRetryAttempt`). The + * policy stays here: a handler cannot ask for another try or skip the wait, + * it can only hold work for the try after which no other follows, the way the + * Agent block keeps its fallback models for the final try. */ private async runHandlerWithRetry( ctx: ExecutionContext, block: SerializedBlock, blockLog: BlockLog | undefined, - invoke: () => Promise + invoke: (retry: BlockRetryAttempt | undefined) => Promise ): Promise { const policy = resolveBlockRetryPolicy(block) - if (!policy) return invoke() + if (!policy) return invoke(undefined) const shouldAccumulateFunctionCost = block.metadata?.id === BlockType.FUNCTION let accumulatedFunctionCost: TrustedExecutionCost | undefined @@ -563,7 +570,11 @@ export class BlockExecutor { for (;;) { tries++ try { - const output = await invoke() + const output = await invoke({ + attempt: tries, + maxTries: policy.maxTries, + isFinalTry: tries >= policy.maxTries, + }) if (!shouldAccumulateFunctionCost || !accumulatedFunctionCost || !isRecordLike(output)) { return output } diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 7a55a12fc85..33bd0db13d2 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -616,6 +616,73 @@ describe('AgentBlockHandler', () => { expect(blockLog).toMatchObject({ modelFallbacks: ['gpt-4o'] }) }) + it('holds the fallbacks on a try the executor will replay', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + const blockLog = openLog() + blockLog.modelFallbacks = ['stale-from-earlier-run'] + + await expect( + handler.execute( + { ...mockContext, blockLogs: [blockLog] }, + mockBlock, + { ...baseInputs, fallbackModels: [{ model: 'claude-sonnet-5' }] }, + { nodeId: mockBlock.id, retry: { attempt: 1, maxTries: 3, isFinalTry: false } } + ) + ).rejects.toThrow('overloaded') + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + expect(mockAgentLogger.info).toHaveBeenCalledWith('Fallback models held for the final try', { + blockId: mockBlock.id, + attempt: 1, + maxTries: 3, + }) + expect(mockAgentLogger.warn).not.toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.anything() + ) + expect(blockLog.modelFallbacks).toBeUndefined() + }) + + it('walks the chain on the final try, and on a block that never retries', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + const inputs = { ...baseInputs, fallbackModels: [{ model: 'claude-sonnet-5' }] } + + const onFinalTry = await handler.execute(mockContext, mockBlock, inputs, { + nodeId: mockBlock.id, + retry: { attempt: 3, maxTries: 3, isFinalTry: true }, + }) + const withoutPolicy = await handler.execute(mockContext, mockBlock, inputs, { + nodeId: mockBlock.id, + }) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(4) + expect((onFinalTry as { model: string }).model).toBe('claude-sonnet-5') + expect((withoutPolicy as { model: string }).model).toBe('claude-sonnet-5') + expect(mockAgentLogger.info).not.toHaveBeenCalledWith( + 'Fallback models held for the final try', + expect.anything() + ) + }) + + it('says nothing about held fallbacks when none are configured', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + + await expect( + handler.execute(mockContext, mockBlock, baseInputs, { + nodeId: mockBlock.id, + retry: { attempt: 1, maxTries: 2, isFinalTry: false }, + }) + ).rejects.toThrow('overloaded') + expect(mockAgentLogger.info).not.toHaveBeenCalledWith( + 'Fallback models held for the final try', + expect.anything() + ) + }) + it('never falls back on a deep-research follow-up turn', async () => { mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 990a75f048b..1f9e5f57271 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -75,7 +75,13 @@ import type { ToolInput, } from '@/executor/handlers/agent/types' import { parseResponseFormat } from '@/executor/handlers/shared/response-format' -import type { BlockHandler, ExecutionContext, StreamingExecution, UserFile } from '@/executor/types' +import type { + BlockHandler, + BlockNodeMetadata, + ExecutionContext, + StreamingExecution, + UserFile, +} from '@/executor/types' import { collectBlockData } from '@/executor/utils/block-data' import { stringifyJSON } from '@/executor/utils/json' import { projectResolvedSecretDiagnosticContent } from '@/executor/utils/resolved-secret-content-projection' @@ -283,7 +289,8 @@ export class AgentBlockHandler implements BlockHandler { async execute( ctx: ExecutionContext, block: SerializedBlock, - inputs: AgentInputs + inputs: AgentInputs, + nodeMetadata?: BlockNodeMetadata ): Promise { ctx.mcpBlockId = block.id const providerErrorRegistry = ctx.resolvedSecretTraceRegistry?.forkForInputPaths( @@ -483,19 +490,33 @@ export class AgentBlockHandler implements BlockHandler { } /** + * Retry on fail retries the selected model; the fallbacks join only on the + * try after which the executor promises no other. Until then a failure of + * the primary is left to escape, so the executor's policy can replay it. + * * A follow-up turn of a deep-research interaction lives on the primary's * provider; another model has none of that conversation, so a green answer * from it would be built on a fresh context. Such a request never falls back. */ - const fallbackCandidates = modelInputs.previousInteractionId - ? [] - : normalizeFallbackModels(filteredInputs.fallbackModels).filter( - (candidate) => candidate.model.toLowerCase() !== model.toLowerCase() - ) - if (modelInputs.previousInteractionId && filteredInputs.fallbackModels?.length) { + const configuredFallbacks = normalizeFallbackModels(filteredInputs.fallbackModels) + const retry = nodeMetadata?.retry + const fallbacksHeld = retry !== undefined && !retry.isFinalTry + const fallbackCandidates = + modelInputs.previousInteractionId || fallbacksHeld + ? [] + : configuredFallbacks.filter( + (candidate) => candidate.model.toLowerCase() !== model.toLowerCase() + ) + if (configuredFallbacks.length > 0 && modelInputs.previousInteractionId) { logger.info('Fallback models skipped for a deep-research follow-up turn', { blockId: block.id, }) + } else if (configuredFallbacks.length > 0 && fallbacksHeld) { + logger.info('Fallback models held for the final try', { + blockId: block.id, + attempt: retry.attempt, + maxTries: retry.maxTries, + }) } const candidates: ModelCandidate[] = [ { model, apiKey: modelInputs.apiKey, isPrimary: true }, @@ -2397,8 +2418,9 @@ export class AgentBlockHandler implements BlockHandler { * * Which model serves the request is decided here, inside one handler * invocation. How many invocations the block gets is the executor's retry - * policy, which wraps this whole chain: with retry on, every try walks the - * chain again from the primary. + * policy, and the caller keeps the fallbacks out of the candidate list until + * the final try, so with retry on the block runs the primary alone on every + * earlier try and walks the whole chain once. * * Falling through is deliberately as indiscriminate as block retry * (`isRetryableBlockError`): a provider error carries no status, so an @@ -2678,7 +2700,7 @@ export class AgentBlockHandler implements BlockHandler { * the executor pushes the entry before running the handler with `endedAt` * still empty, which is what tells it apart from earlier runs of the same * block in a loop or an earlier retry. An empty list clears the field, since - * a retry that succeeds on the primary reuses the entry a failed try wrote. + * every try reuses one entry and only the final try can write failed models. * * A model id can itself come from a resolved reference, so the names are * projected through the same secret registry as every other diagnostic and diff --git a/apps/sim/executor/types.ts b/apps/sim/executor/types.ts index 073ac1e2bb7..1d6ed9229d8 100644 --- a/apps/sim/executor/types.ts +++ b/apps/sim/executor/types.ts @@ -756,6 +756,22 @@ export interface BlockNodeMetadata { originalBlockId?: string isLoopNode?: boolean executionOrder?: number + /** Where this invocation sits in the block's retry policy; absent when the block has none. */ + retry?: BlockRetryAttempt +} + +/** + * One try of a block under its retry policy, told to the handler so it can hold + * work for the last try. `isFinalTry` is the executor's own judgment, not + * `attempt >= maxTries` recomputed by the handler: what makes a try final is the + * policy's business, and a handler that fails on a non-final try is promised + * another invocation for any retryable error. + */ +export interface BlockRetryAttempt { + /** 1-based. */ + attempt: number + maxTries: number + isFinalTry: boolean } export interface BlockHandler { From 93abc4ce58b73a77b49f87b559d8dad84dda8906 Mon Sep 17 00:00:00 2001 From: Marcus Chandra Date: Fri, 18 Sep 2026 04:22:34 -0700 Subject: [PATCH 09/14] fix(agent): address the pre-landing review of retry-then-fallback Review fixes: the executor judges the final try once per iteration; the skip-warn is one helper; the fallback warn names the candidate position rather than reusing `attempt`; the BlockLog.modelFallbacks doc matches the final-try semantics; the viability check resolves a provider once through the new providerRequiresFamilyCredentials; editor handlers read rows via a ref so a keystroke in one row no longer re-renders every row, and tuning options keep their identity across renders. Two consistency fixes from the red-team pass: a sim-auto fallback takes the projected system prompt rather than the raw input, and a row key is honoured at runtime only when the block stored it as a whole {{NAME}} reference, the one form the editor, the validator, and an export agree on. Tests now cover the composed executor-plus-handler sequence (three tries on the selected model, then each fallback once), the node-taking handler signature, a fallback whose provider cannot take the attachments, the per-provider hydration cache, a stop during a skipped candidate, the un-primed stream when no candidate follows, a non-retryable failure on a non-final try, a numeric tuning value, and an unresolved temperature. --- .../model-fallback-list.tsx | 73 ++++-- apps/sim/blocks/utils.test.ts | 10 + apps/sim/blocks/utils.ts | 10 +- .../execution/block-executor.retry.test.ts | 24 ++ apps/sim/executor/execution/block-executor.ts | 8 +- .../handlers/agent/agent-handler.test.ts | 247 +++++++++++++++++- .../executor/handlers/agent/agent-handler.ts | 89 +++++-- apps/sim/executor/types.ts | 5 +- .../workflows/blocks/fallback-models.test.ts | 13 +- .../lib/workflows/blocks/fallback-models.ts | 10 +- .../lib/workflows/editing/validation.test.ts | 4 + 11 files changed, 425 insertions(+), 68 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx index 502835b8361..2950e6511c8 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx @@ -1,6 +1,6 @@ 'use client' -import { memo, useCallback, useMemo } from 'react' +import { memo, useCallback, useEffect, useMemo, useRef } from 'react' import { Button, Combobox, type ComboboxOption, Label, Tooltip } from '@sim/emcn' import { ChevronDown, ChevronUp, Plus, Trash } from '@sim/emcn/icons' import { generateShortId } from '@sim/utils/id' @@ -51,7 +51,10 @@ interface ViableModelOption { interface FallbackRowProps { row: FallbackModelEntry index: number - count: number + /** The row can move down only while another follows it. */ + isLast: boolean + /** Move controls render only once a second row exists. */ + canMove: boolean primaryModel: string primaryTuning: Partial> viableOptions: ViableModelOption[] @@ -69,7 +72,8 @@ interface FallbackRowProps { const FallbackRow = memo(function FallbackRow({ row, index, - count, + isLast, + canMove, primaryModel, primaryTuning, viableOptions, @@ -96,7 +100,12 @@ const FallbackRow = memo(function FallbackRow({ return { needsApiKey: fallbackRowNeedsApiKey(row.model, primaryModel), tuningFields: getFallbackTuningKnobsToShow(row.model, primaryModel, primaryTuning).map( - (knob) => ({ knob, options: getTuningOptionsForModel(row.model, knob) ?? [] }) + (knob) => ({ + knob, + options: (getTuningOptionsForModel(row.model, knob) ?? []).map( + (value): ComboboxOption => ({ label: value, value }) + ), + }) ), } }, [row.model, primaryModel, primaryTuning]) @@ -112,7 +121,7 @@ const FallbackRow = memo(function FallbackRow({
{ordinalChoiceLabel(index)}
- {count > 1 && ( + {canMove && ( <> @@ -133,7 +142,7 @@ const FallbackRow = memo(function FallbackRow({ ), - Combobox: ({ + ChipCombobox: ({ options, value, placeholder, @@ -52,6 +60,21 @@ vi.mock('@sim/emcn', () => ({ ))}
), + ChipDropdown: ({ + options, + value, + placeholder, + }: { + options: Array<{ value: string; label: string }> + value?: string + placeholder?: string + }) => ( +
+ {options.map((option) => ( + {option.label} + ))} +
+ ), Label: ({ children }: { children?: React.ReactNode }) => {children}, Tooltip: { Root: ({ children }: { children?: React.ReactNode }) => <>{children}, @@ -102,6 +125,8 @@ vi.mock('@/stores/providers/store', () => ({ })) vi.mock('@/blocks/utils', () => ({ + shouldRequireApiKeyForModel: (model: string) => + model.startsWith('openrouter/') || (model.startsWith('gpt') && !getDeploymentShape().hosted), getModelOptions: () => [ { id: 'claude-sonnet-5', label: 'claude-sonnet-5' }, { id: 'gpt-5', label: 'gpt-5' }, @@ -117,7 +142,6 @@ vi.mock('@/lib/workflows/blocks/fallback-models', async (importOriginal) => { ...actual, isViableFallbackModel: (model: string, primary: string) => model !== 'sim-auto' && model !== primary, - fallbackRowNeedsApiKey: (model: string) => model.startsWith('openrouter/'), getFallbackTuningKnobsToShow: (model: string) => (model === 'gpt-5' ? ['reasoningEffort'] : []), getTuningOptionsForModel: (model: string, knob: string) => model === 'gpt-5' && knob === 'reasoningEffort' ? ['auto', 'low', 'high'] : null, @@ -126,6 +150,16 @@ vi.mock('@/lib/workflows/blocks/fallback-models', async (importOriginal) => { import { ModelFallbackList } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list' +beforeEach(() => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + seedDeploymentShape({ ...resolveDeploymentShape(), hosted: true }) +}) + +afterEach(() => { + resetDeploymentShape() + vi.unstubAllGlobals() +}) + function render(extra: Partial> = {}) { return renderToStaticMarkup( @@ -186,6 +220,27 @@ describe('ModelFallbackList', () => { expect(html).toContain('Create Secret') }) + it('updates key visibility when hosted context arrives after mount, without rewriting the rows', async () => { + seedDeploymentShape({ ...resolveDeploymentShape(), hosted: false }) + subBlockValues.fallbackModels = [{ id: 'r1', model: 'gpt-5' }] + const container = document.createElement('div') + const root = createRoot(container) + try { + await act(async () => { + root.render() + }) + expect(container.querySelector('[data-combobox="Select a secret"]')).not.toBeNull() + + await act(async () => { + seedDeploymentShape({ ...resolveDeploymentShape(), hosted: true }) + }) + expect(container.querySelector('[data-combobox="Select a secret"]')).toBeNull() + expect(mockSetValue).not.toHaveBeenCalled() + } finally { + await act(async () => root.unmount()) + } + }) + it('shows a tuning field only for the knobs the helper says need one', () => { subBlockValues.fallbackModels = [ { id: 'r1', model: 'gpt-5', reasoningEffort: 'low' }, diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx index 011988e9913..15b7bcefdd1 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx @@ -1,10 +1,11 @@ 'use client' import { memo, useCallback, useEffect, useMemo, useRef } from 'react' -import { Button, Combobox, type ComboboxOption, Label, Tooltip } from '@sim/emcn' +import { Chip, ChipCombobox, ChipDropdown, type ComboboxOption, Label, Tooltip } from '@sim/emcn' import { ChevronDown, ChevronUp, Plus, Trash } from '@sim/emcn/icons' import { generateShortId } from '@sim/utils/id' import { useParams } from 'next/navigation' +import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import { writePendingCredentialCreateRequest } from '@/lib/credentials/client-state' import { addFallbackRow, @@ -96,6 +97,8 @@ const FallbackRow = memo(function FallbackRow({ onMove, onRemove, }: FallbackRowProps) { + /** Credential visibility follows the server-resolved shape, including late hydration. */ + useDeploymentShape() const modelOptions = useMemo( (): ComboboxOption[] => viableOptions.map((option) => ({ @@ -105,20 +108,16 @@ const FallbackRow = memo(function FallbackRow({ [viableOptions, takenModels, row.model] ) - const { needsApiKey, tuningFields } = useMemo(() => { - if (!row.model) return { needsApiKey: false, tuningFields: [] } - return { - needsApiKey: fallbackRowNeedsApiKey(row.model, primaryModel), - tuningFields: getFallbackTuningKnobsToShow(row.model, primaryModel, primaryTuning).map( - (knob) => ({ - knob, - options: (getTuningOptionsForModel(row.model, knob) ?? []).map( - (value): ComboboxOption => ({ label: value, value }) - ), - }) - ), - } - }, [row.model, primaryModel, primaryTuning]) + const needsApiKey = fallbackRowNeedsApiKey(row.model, primaryModel) + const tuningFields = getFallbackTuningKnobsToShow(row.model, primaryModel, primaryTuning).map( + (knob) => ({ + knob, + options: (getTuningOptionsForModel(row.model, knob) ?? []).map((value) => ({ + label: value, + value, + })), + }) + ) /** Only a reference is ever shown; anything else that reached the store reads as unset. */ const apiKeyValue = isWholeEnvVarReference(row.apiKey) ? row.apiKey : '' @@ -126,38 +125,32 @@ const FallbackRow = memo(function FallbackRow({ return (
-
- {ordinalChoiceLabel(index)} -
+
+ {ordinalChoiceLabel(index)} +
{canMove && ( <> - + /> Move up - + /> Move down @@ -165,44 +158,40 @@ const FallbackRow = memo(function FallbackRow({ )} - + /> Remove
-
-
- - onChangeModel(row.id, model)} - placeholder='Select a model' - disabled={readOnly} - searchable - searchPlaceholder='Search models...' - maxHeight={240} - emptyMessage='No models available' - /> -
+
+ onChangeModel(row.id, model)} + placeholder='Select a model' + aria-label={`${ordinalChoiceLabel(index)} model`} + disabled={readOnly} + searchable + searchPlaceholder='Search models...' + maxHeight={240} + emptyMessage='No models available' + /> {needsApiKey && (
- onChangeApiKey(row.id, apiKey)} placeholder='Select a secret' + aria-label={`${ordinalChoiceLabel(index)} API key`} disabled={readOnly} searchable searchPlaceholder='Search secrets...' @@ -214,12 +203,14 @@ const FallbackRow = memo(function FallbackRow({ {tuningFields.map(({ knob, options }) => (
- onChangeTuning(row.id, knob, value)} placeholder={`Select ${FALLBACK_TUNING_LABELS[knob].toLowerCase()}`} + aria-label={`${ordinalChoiceLabel(index)} ${FALLBACK_TUNING_LABELS[knob].toLowerCase()}`} disabled={readOnly} + className='w-full' />
))} @@ -250,6 +241,7 @@ export function ModelFallbackList({ const workspaceId = typeof params?.workspaceId === 'string' ? params.workspaceId : '' const { navigateToSettings } = useSettingsNavigation() const { isModelUsable } = usePermissionConfig() + const deploymentShape = useDeploymentShape() const providers = useProvidersStore((state) => state.providers) const [storeValue, setStoreValue] = useSubBlockValue(blockId, subBlockId) const [primaryModelValue] = useSubBlockValue(blockId, 'model') @@ -311,7 +303,7 @@ export function ModelFallbackList({ value: option.id, ...(option.icon ? { icon: option.icon } : {}), })), - [primaryModel, isModelUsable, providers] + [primaryModel, isModelUsable, providers, deploymentShape] ) const takenModels = useMemo(() => new Set(rows.map((row) => row.model).filter(Boolean)), [rows]) @@ -417,15 +409,15 @@ export function ModelFallbackList({ /> ))} {!readOnly && ( - + )}
) diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index a331e4c2963..82464855edd 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -1,10 +1,8 @@ import { createLogger } from '@sim/logger' import { omit } from '@sim/utils/object' import { AgentIcon } from '@/components/icons' -import { - MAX_FALLBACK_MODELS, - normalizeFallbackModels, -} from '@/lib/workflows/blocks/fallback-models' +import { normalizeFallbackModels } from '@/lib/workflows/blocks/fallback-models' +import { getModelFallbackSubBlock, MODEL_FALLBACK_INPUTS } from '@/blocks/model-fallbacks' import type { BlockConfig } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' import { @@ -37,7 +35,6 @@ const logger = createLogger('AgentBlock') /** Model the agent block falls back to when `model` is unset or the auto pseudo-model. */ const AGENT_FALLBACK_MODEL = 'claude-sonnet-5' -const FALLBACK_MODELS_DESCRIPTION = `Ordered models tried in sequence, once each, when the request to the selected model fails; with Retry on fail, after the selected model's tries run out. Each row is { model, apiKey?, reasoningEffort?, thinkingLevel?, verbosity? }; apiKey, when present, must be a whole {{ENV_VAR}} reference, and a tuning value must be one the row model declares. sim-auto is not allowed. Max ${MAX_FALLBACK_MODELS}.` const MODELS_WITH_REASONING_EFFORT = getModelsWithReasoningEffort() const MODELS_WITH_VERBOSITY = getModelsWithVerbosity() const MODELS_WITH_THINKING = getModelsWithThinking() @@ -436,13 +433,7 @@ Return ONLY the JSON array.`, value: MODELS_WITH_DEEP_RESEARCH, }, }, - { - id: 'fallbackModels', - title: 'Fallback models', - type: 'model-fallback-list', - mode: 'advanced', - description: FALLBACK_MODELS_DESCRIPTION, - }, + getModelFallbackSubBlock(), ], tools: { access: [ @@ -605,7 +596,7 @@ Return ONLY the JSON array.`, type: 'boolean', description: 'Cache the system prompt and tool definitions on models that support it', }, - fallbackModels: { type: 'json', description: FALLBACK_MODELS_DESCRIPTION }, + ...MODEL_FALLBACK_INPUTS, tools: { type: 'json', description: 'Available tools configuration' }, skills: { type: 'json', description: 'Selected skills configuration' }, }, diff --git a/apps/sim/blocks/blocks/evaluator.ts b/apps/sim/blocks/blocks/evaluator.ts index 194183149cc..75fed5e1c06 100644 --- a/apps/sim/blocks/blocks/evaluator.ts +++ b/apps/sim/blocks/blocks/evaluator.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { ChartBarIcon } from '@/components/icons' +import { getModelFallbackSubBlock, MODEL_FALLBACK_INPUTS } from '@/blocks/model-fallbacks' import type { BlockConfig, ParamType } from '@/blocks/types' import { getModelOptions, @@ -189,6 +190,7 @@ export const EvaluatorBlock: BlockConfig = { options: getModelOptions, }, ...getProviderCredentialSubBlocks(), + getModelFallbackSubBlock(), { id: 'temperature', title: 'Temperature', @@ -294,6 +296,7 @@ export const EvaluatorBlock: BlockConfig = { }, model: { type: 'string' as ParamType, description: 'AI model to use' }, ...PROVIDER_CREDENTIAL_INPUTS, + ...MODEL_FALLBACK_INPUTS, temperature: { type: 'number' as ParamType, description: 'Response randomness level (low for consistent evaluation)', diff --git a/apps/sim/blocks/blocks/router.ts b/apps/sim/blocks/blocks/router.ts index 34ba119712e..8d9bccb9f9c 100644 --- a/apps/sim/blocks/blocks/router.ts +++ b/apps/sim/blocks/blocks/router.ts @@ -1,4 +1,5 @@ import { ConnectIcon } from '@/components/icons' +import { getModelFallbackSubBlock, MODEL_FALLBACK_INPUTS } from '@/blocks/model-fallbacks' import { AuthMode, type BlockConfig } from '@/blocks/types' import { getModelOptions, @@ -186,6 +187,7 @@ export const RouterBlock: BlockConfig = { options: getModelOptions, }, ...getProviderCredentialSubBlocks(), + getModelFallbackSubBlock(), { id: 'temperature', title: 'Temperature', @@ -221,6 +223,7 @@ export const RouterBlock: BlockConfig = { prompt: { type: 'string', description: 'Routing prompt content' }, model: { type: 'string', description: 'AI model to use' }, ...PROVIDER_CREDENTIAL_INPUTS, + ...MODEL_FALLBACK_INPUTS, temperature: { type: 'number', description: 'Response randomness level (low for consistent routing)', @@ -303,6 +306,7 @@ export const RouterV2Block: BlockConfig = { options: getModelOptions, }, ...getProviderCredentialSubBlocks(), + getModelFallbackSubBlock(), ], tools: { access: [ @@ -322,6 +326,7 @@ export const RouterV2Block: BlockConfig = { routes: { type: 'json', description: 'Route definitions with descriptions' }, model: { type: 'string', description: 'AI model to use' }, ...PROVIDER_CREDENTIAL_INPUTS, + ...MODEL_FALLBACK_INPUTS, }, outputs: { context: { type: 'string', description: 'Context used for routing' }, diff --git a/apps/sim/blocks/model-fallbacks.ts b/apps/sim/blocks/model-fallbacks.ts new file mode 100644 index 00000000000..4f449356d8d --- /dev/null +++ b/apps/sim/blocks/model-fallbacks.ts @@ -0,0 +1,19 @@ +import { MAX_FALLBACK_MODELS } from '@/lib/workflows/blocks/fallback-models' +import type { SubBlockConfig } from '@/blocks/types' + +const DESCRIPTION = `Ordered models tried once each when the selected model's request fails; with Retry on fail, after its tries run out. Each row is { model, apiKey?, reasoningEffort?, thinkingLevel?, verbosity? }. API keys must be whole {{ENV_VAR}} references and apply only when the row asks for a key. Tuning must be supported by the row model. sim-auto is not allowed. Max ${MAX_FALLBACK_MODELS}.` + +/** Shared advanced field for blocks that execute model requests. */ +export function getModelFallbackSubBlock(): SubBlockConfig { + return { + id: 'fallbackModels', + title: 'Fallback models', + type: 'model-fallback-list', + mode: 'advanced', + description: DESCRIPTION, + } +} + +export const MODEL_FALLBACK_INPUTS = { + fallbackModels: { type: 'json', description: DESCRIPTION }, +} as const diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 702c247702d..799281abcd2 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -18,6 +18,7 @@ import { type Mock, vi, } from 'vitest' +import { resetDeploymentShape } from '@/lib/core/config/deployment-shape' import type { AutoRoutingSignals } from '@/lib/model-router/resolve' import * as userFileBase64 from '@/lib/uploads/utils/user-file-base64.server' import { getAllBlocks } from '@/blocks' @@ -572,6 +573,8 @@ describe('AgentBlockHandler', () => { } beforeEach(() => { + setEnvFlags({ isHosted: false }) + resetDeploymentShape() mockGetProviderFromModel.mockImplementation(providerFor) mockValidateModelProvider.mockResolvedValue(undefined) }) @@ -622,6 +625,120 @@ describe('AgentBlockHandler', () => { expect(blockLog).toMatchObject({ modelFallbacks: ['gpt-4o'] }) }) + it.each(['flat', 'memory-block'] as const)( + 'preserves injected %s memories when switching providers', + async (shape) => { + const history: Message[] = [ + { role: 'user', content: 'My name is Ada.' }, + { role: 'assistant', content: 'Hello Ada.' }, + ] + const inputs: AgentInputs = { + ...baseInputs, + systemPrompt: 'Use the conversation history.', + userPrompt: 'What is my name?', + memories: + shape === 'flat' ? history : { memories: [{ key: 'conversation-1', data: history }] }, + fallbackModels: [{ model: 'claude-sonnet-5' }], + } + const original = structuredClone(inputs) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5', 'Your name is Ada.')) + + await handler.execute(mockContext, mockBlock, inputs) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + const expectedMessages = [ + { role: 'system', content: 'Use the conversation history.' }, + ...history, + { role: 'user', content: 'What is my name?' }, + ] + for (const [, request] of mockExecuteProviderRequest.mock.calls) { + expect(request.messages).toEqual(expectedMessages) + } + expect(mockExecuteProviderRequest.mock.calls[1][0]).toBe('anthropic') + expect(inputs).toEqual(original) + } + ) + + it.each([ + { memoryType: 'conversation', streaming: false }, + { memoryType: 'conversation', streaming: true }, + { memoryType: 'sliding_window', streaming: false }, + { memoryType: 'sliding_window', streaming: true }, + ] as const)( + 'preserves $memoryType history through fallback and saves each turn once (streaming=$streaming)', + async ({ memoryType, streaming }) => { + dbChainMockFns.returning.mockResolvedValue([{ id: 'memory-1' }]) + const history: Message[] = [ + { role: 'user', content: 'Hello.' }, + { role: 'assistant', content: 'How can I help?' }, + { role: 'user', content: 'My name is Ada.' }, + { role: 'assistant', content: 'Hello Ada.' }, + ] + queueTableRows(schemaMock.memory, [{ secretProvenanceVersion: null, data: history }]) + const ctx = { ...mockContext, executionId: 'memory-fallback-execution' } + const inputs: AgentInputs = { + ...baseInputs, + userPrompt: undefined, + memoryType, + slidingWindowSize: '2', + conversationId: 'conversation-1', + messages: [ + { role: 'system', content: 'Use the conversation history.' }, + { role: 'user', content: 'What is my name?' }, + ], + fallbackModels: [{ model: 'claude-sonnet-5' }], + } + const original = structuredClone(inputs) + if (streaming) { + mockExecuteProviderRequest + .mockResolvedValueOnce( + streamingResponse([], { failBeforeFirstChunk: new Error('overloaded') }) + ) + .mockResolvedValueOnce(streamingResponse(['Your name is Ada.'])) + } else { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5', 'Your name is Ada.')) + } + + const result = await handler.execute(ctx, mockBlock, inputs) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + const currentUserMessage = { + role: 'user', + content: 'What is my name?', + executionId: ctx.executionId, + } + const expectedMessages = [ + { role: 'system', content: 'Use the conversation history.' }, + ...(memoryType === 'sliding_window' ? history.slice(-2) : history), + { role: 'user', content: 'What is my name?' }, + ] + for (const [, request] of mockExecuteProviderRequest.mock.calls) { + expect(request.messages).toEqual(expectedMessages) + } + expect(mockExecuteProviderRequest.mock.calls[1][0]).toBe('anthropic') + if (streaming) { + const streamedResult = result as StreamingExecution + expect(await drain(streamedResult.stream)).toEqual(['Your name is Ada.']) + expect(streamedResult.onFullContent).toBeTypeOf('function') + await streamedResult.onFullContent?.('Your name is Ada.') + } + + const memoryWrites = dbChainMockFns.values.mock.calls + .map(([row]) => row) + .filter((row) => Array.isArray(row.data)) + expect(memoryWrites.map((row) => row.data)).toEqual([ + [currentUserMessage], + [{ role: 'assistant', content: 'Your name is Ada.' }], + ]) + expect(memoryWrites.every((row) => row.key === inputs.conversationId)).toBe(true) + expect(inputs).toEqual(original) + } + ) + it('holds the fallbacks on a try the executor will replay', async () => { mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) const blockLog = openLog() @@ -757,6 +874,58 @@ describe('AgentBlockHandler', () => { ) }) + it.each([ + { hosted: false, primary: 'gpt-4o', fallback: 'gpt-4o-mini', expectedKey: 'primary-key' }, + { hosted: true, primary: 'gpt-4o', fallback: 'claude-sonnet-5', expectedKey: undefined }, + ])( + 'ignores a hidden row key for $fallback with hosted=$hosted', + async ({ hosted, primary, fallback, expectedKey }) => { + setEnvFlags({ isHosted: hosted }) + resetDeploymentShape() + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse(fallback)) + const block = { + ...mockBlock, + config: { + ...mockBlock.config, + params: { fallbackModels: [{ model: fallback, apiKey: '{{OLD_KEY}}' }] }, + }, + } + + await handler.execute(mockContext, block, { + ...baseInputs, + model: primary, + fallbackModels: [{ model: fallback, apiKey: 'old-row-key' }], + }) + + expect(mockExecuteProviderRequest.mock.calls[1][1].apiKey).toBe(expectedKey) + } + ) + + it('lets the executor retry a stream startup failure while fallbacks are held', async () => { + mockExecuteProviderRequest.mockResolvedValueOnce( + streamingResponse([], { failBeforeFirstChunk: new Error('429 at stream start') }) + ) + + await expect( + handler.execute( + mockContext, + mockBlock, + { + ...baseInputs, + stream: true, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }, + { + nodeId: mockBlock.id, + retry: { attempt: 1, maxTries: 3, isFinalTry: false }, + } + ) + ).rejects.toThrow('429 at stream start') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + it('leaves provider-family credentials off a fallback on another provider', async () => { mockExecuteProviderRequest .mockRejectedValueOnce(new Error('one')) diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 3bc87c25312..cbc182f271a 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -41,8 +41,6 @@ import { import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' import { type FallbackModelCandidate, - isWholeEnvVarReference, - normalizeFallbackModels, resolveFallbackTuning, } from '@/lib/workflows/blocks/fallback-models' import { resolveCustomBlockToolBinding } from '@/lib/workflows/custom-blocks/operations' @@ -84,6 +82,12 @@ import type { } from '@/executor/types' import { collectBlockData } from '@/executor/utils/block-data' import { stringifyJSON } from '@/executor/utils/json' +import { + getModelFallbacks, + PROVIDER_FAMILY_CREDENTIAL_FIELDS, + recordModelFallbacks, + resolveFallbackApiKey, +} from '@/executor/utils/model-fallbacks' import { projectResolvedSecretDiagnosticContent } from '@/executor/utils/resolved-secret-content-projection' import { prepareResolvedSecretProjectedInputs } from '@/executor/utils/resolved-secret-input-projection' import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal' @@ -170,22 +174,6 @@ function stripAutoPreamble(messages: Message[] | undefined): Message[] | undefin }) } -/** - * Block fields that only a provider family reads. A fallback on another provider - * never needs them, so they are left off its request rather than handed to a - * provider that has no use for a Bedrock secret or a Vertex credential. - */ -const PROVIDER_FAMILY_CREDENTIAL_FIELDS = [ - 'azureEndpoint', - 'azureApiVersion', - 'vertexProject', - 'vertexLocation', - 'vertexCredential', - 'bedrockAccessKeyId', - 'bedrockSecretKey', - 'bedrockRegion', -] as const satisfies ReadonlyArray - /** One model in the order the block tries them; the primary carries the block's own key. */ interface ModelCandidate extends FallbackModelCandidate { isPrimary: boolean @@ -199,6 +187,7 @@ interface ModelCandidate extends FallbackModelCandidate { interface ExecuteAcrossModelsConfig { candidates: ModelCandidate[] + retryPrimaryOnStreamStart: boolean primaryModel: string /** * The model the builder configured, which is what the editor showed the @@ -528,8 +517,11 @@ export class AgentBlockHandler implements BlockHandler { * provider; another model has none of that conversation, so a green answer * from it would be built on a fresh context. Such a request never falls back. */ - const configuredFallbacks = normalizeFallbackModels( - this.keepReferenceRowKeys(ctx, block, filteredInputs.fallbackModels) + const configuredFallbacks = getModelFallbacks( + ctx, + block, + filteredInputs.fallbackModels, + logger ) const retry = nodeMetadata?.retry const fallbacksHeld = retry !== undefined && !retry.isFinalTry @@ -565,6 +557,8 @@ export class AgentBlockHandler implements BlockHandler { resultRegistry: servedRegistry, } = await this.executeAcrossModels(ctx, block, { candidates, + retryPrimaryOnStreamStart: + fallbacksHeld && configuredFallbacks.length > 0 && !modelInputs.previousInteractionId, primaryModel: model, configuredModel: autoRouting ? SIM_AUTO_MODEL_ID : model, primaryProviderId: providerId, @@ -2557,8 +2551,8 @@ export class AgentBlockHandler implements BlockHandler { } /** - * A fallback's own key wins; without one it may reuse the block's key only - * on the primary's provider. Otherwise the provider layer resolves BYOK or + * A fallback's own key applies only while its key field is visible. On the + * primary's provider it reuses the block's key; otherwise the provider layer resolves BYOK or * the platform key, or reports that a key is required, which counts as * this candidate failing. A previous interaction id belongs to the primary's * provider alone. Tuning is re-resolved against the fallback's own @@ -2571,27 +2565,19 @@ export class AgentBlockHandler implements BlockHandler { config.configuredModel, config.modelInputs ) - /** - * A row key still in `{{NAME}}` form was never resolved: the variable is - * not set for the principal running this workflow. Sending the literal - * would only replace a platform or BYOK key with garbage, so it counts - * as no key at all. - */ - let rowKey = candidate.apiKey - if (rowKey && isWholeEnvVarReference(rowKey)) { - logger.warn('Fallback key variable is not set for this run', { - blockId: block.id, - model: candidate.model, - variable: rowKey, - }) - rowKey = undefined - } const sameProvider = candidateProviderId === config.primaryProviderId inputs = { ...(sameProvider ? config.modelInputs - : omit(config.modelInputs, [...PROVIDER_FAMILY_CREDENTIAL_FIELDS])), - apiKey: rowKey ?? (sameProvider ? config.modelInputs.apiKey : undefined), + : omit(config.modelInputs, [...PROVIDER_FAMILY_CREDENTIAL_FIELDS, 'vertexCredential'])), + apiKey: resolveFallbackApiKey({ + candidate, + configuredModel: config.configuredModel, + sameProvider, + primaryApiKey: config.modelInputs.apiKey, + blockId: block.id, + logger, + }), previousInteractionId: undefined, ...(config.fallbackSystemPrompt !== undefined ? { systemPrompt: config.fallbackSystemPrompt } @@ -2633,16 +2619,16 @@ export class AgentBlockHandler implements BlockHandler { resultRegistry, config.providerErrorRegistry ) - if (hasNext && this.isStreamingExecution(result)) { + if ((hasNext || config.retryPrimaryOnStreamStart) && this.isStreamingExecution(result)) { result = await this.primeStreamingExecution(result as StreamingExecution) } - this.recordModelFallbacks(ctx, block, failedModels) + recordModelFallbacks(ctx, block, failedModels) return { result, servedModel: candidate.model, resultRegistry } } catch (error) { lastError = error failedModels.push(candidate.traceName ?? candidate.model) if (!hasNext || ctx.abortSignal?.aborted || !isRetryableBlockError(error)) { - this.recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) + recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) throw error } @@ -2682,43 +2668,10 @@ export class AgentBlockHandler implements BlockHandler { ctx.errorResolvedSecretTraceRegistry = lastErrorRegistries.error ctx.resolvedSecretTraceRegistry = lastErrorRegistries.resolved } - this.recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) + recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) throw lastError } - /** - * Keeps a fallback row's key only when the block stored it as a whole - * `{{NAME}}` reference, which is the one form the editor offers, the validator - * accepts, and an export preserves. A raw value written through the realtime - * subblock op would otherwise reach the provider while the row shows no key - * at all; it is dropped here so every surface agrees, and named in a warn so - * the builder can find and clear it. - */ - private keepReferenceRowKeys( - ctx: ExecutionContext, - block: SerializedBlock, - rows: AgentInputs['fallbackModels'] - ): AgentInputs['fallbackModels'] { - if (!Array.isArray(rows)) return rows - const storedRows: unknown = block.config?.params?.fallbackModels - return rows.map((row, index) => { - if (!row || typeof row !== 'object' || row.apiKey === undefined) return row - const stored = Array.isArray(storedRows) ? storedRows[index] : undefined - const storedKey = - stored && typeof stored === 'object' ? (stored as { apiKey?: unknown }).apiKey : undefined - if (isWholeEnvVarReference(storedKey)) return row - logger.warn( - 'Fallback row key ignored; only an environment variable reference is accepted', - projectAgentDiagnosticMetadata( - ctx, - { blockId: block.id, model: row.model, row: index + 1 }, - { blockId: block.id, row: index + 1 } - ) - ) - return { ...row, apiKey: undefined } - }) - } - /** * Warns that a fallback candidate was passed over, projected like every other * diagnostic so a model id resolved from a reference never reaches the log. @@ -2783,42 +2736,6 @@ export class AgentBlockHandler implements BlockHandler { return { ...result, stream } } - /** - * Writes the models that failed onto the block's open log entry so the trace - * can show them beside the model that answered. Handlers get no log handle; - * the executor pushes the entry before running the handler with `endedAt` - * still empty, which is what tells it apart from earlier runs of the same - * block in a loop or an earlier retry. An empty list clears the field, since - * every try reuses one entry and only the final try can write failed models. - * - * A model id can itself come from a resolved reference, so the names are - * projected through the same secret registry as every other diagnostic and - * left off the log entirely when the projection is not safe. - */ - private recordModelFallbacks( - ctx: ExecutionContext, - block: SerializedBlock, - failedModels: string[] - ): void { - const logs = ctx.blockLogs ?? [] - for (let index = logs.length - 1; index >= 0; index--) { - const entry = logs[index] - if (entry.blockId !== block.id || entry.endedAt !== '') continue - if (failedModels.length === 0) { - entry.modelFallbacks = undefined - return - } - const registry = ctx.errorResolvedSecretTraceRegistry ?? ctx.resolvedSecretTraceRegistry - const projection = projectResolvedSecretDiagnosticContent({ models: failedModels }, registry) - const models = projection.safe ? (projection.value as { models?: unknown }).models : undefined - entry.modelFallbacks = - Array.isArray(models) && models.every((model) => typeof model === 'string') - ? [...models] - : undefined - return - } - } - private buildProviderRequest(config: { ctx: ExecutionContext providerId: string diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts index 22a337b3812..47932eb9470 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts @@ -160,6 +160,53 @@ describe('EvaluatorBlockHandler', () => { apiKey: 'test-api-key', } + it('preserves metric scores and Auto routing cost when a fallback answers', async () => { + mockGetProviderFromModel.mockImplementation((model: string) => + model.startsWith('claude') ? 'anthropic' : 'fireworks' + ) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce({ + content: '{"score1":7}', + model: 'claude-sonnet-5', + tokens: { input: 12, output: 3, total: 15 }, + cost: { input: 0.003, output: 0.001, total: 0.004 }, + }) + const output = await handler.execute(mockContext, mockBlock, { + ...admissionInputs, + model: 'sim-auto', + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + expect(output).toMatchObject({ + content: admissionInputs.content, + model: 'claude-sonnet-5', + score1: 7, + tokens: { total: 15 }, + cost: { total: 0.006 }, + }) + const first = mockExecuteProviderRequest.mock.calls[0][1] + const fallback = mockExecuteProviderRequest.mock.calls[1][1] + expect(fallback.responseFormat).toEqual(first.responseFormat) + expect(fallback.systemPrompt).not.toContain('Sim auto system preamble') + expect(fallback.apiKey).toBeUndefined() + }) + + it('forwards retry metadata so fallbacks wait for the final primary attempt', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + await expect( + handler.execute( + mockContext, + mockBlock, + { + ...admissionInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }, + { nodeId: mockBlock.id, retry: { attempt: 1, maxTries: 2, isFinalTry: false } } + ) + ).rejects.toThrow('overloaded') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + it('refuses to reach the provider without an execution subject', async () => { mockContext.userId = undefined diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts index 52f0859f88a..71bf795b96d 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts @@ -9,9 +9,9 @@ import { import type { BlockOutput } from '@/blocks/types' import { validateModelProvider } from '@/ee/access-control/utils/permission-check' import { BlockType, DEFAULTS, EVALUATOR } from '@/executor/constants' -import type { BlockHandler, ExecutionContext } from '@/executor/types' +import type { BlockHandler, BlockNodeMetadata, ExecutionContext } from '@/executor/types' import { isJSONString, parseJSON, stringifyJSON } from '@/executor/utils/json' -import { executeBlockProviderRequest } from '@/executor/utils/provider-request' +import { executeModelRequestWithFallbacks } from '@/executor/utils/model-fallback-request' import { projectResolvedSecretDiagnosticError } from '@/executor/utils/resolved-secret-content-projection' import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal' import type { @@ -38,7 +38,8 @@ export class EvaluatorBlockHandler implements BlockHandler { async execute( ctx: ExecutionContext, block: SerializedBlock, - inputs: Record + inputs: Record, + nodeMetadata?: BlockNodeMetadata ): Promise { const evaluatorConfig = { model: inputs.model || EVALUATOR.DEFAULT_MODEL, @@ -142,6 +143,7 @@ export class EvaluatorBlockHandler implements BlockHandler { 'Evaluate the content and provide scores for each metric as JSON.' } + const fallbackSystemPrompt = systemPromptObj.systemPrompt let model = evaluatorConfig.model let autoRouting: AutoRoutingResult | null = null if (isAutoModel(model)) { @@ -213,7 +215,12 @@ export class EvaluatorBlockHandler implements BlockHandler { workspaceId: ctx.workspaceId, } - const result = await executeBlockProviderRequest({ + const { result, usedFallback } = await executeModelRequestWithFallbacks({ + block, + configuredModel: evaluatorConfig.model, + fallbackModels: inputs.fallbackModels, + fallbackSystemPrompt, + retry: nodeMetadata?.retry, ctx, providerId, request: providerRequest, @@ -237,7 +244,7 @@ export class EvaluatorBlockHandler implements BlockHandler { return { content: inputs.content, - model: autoRouting ? SIM_AUTO_MODEL_ID : result.model, + model: autoRouting && !usedFallback ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: inputTokens, output: outputTokens, diff --git a/apps/sim/executor/handlers/router/router-handler.test.ts b/apps/sim/executor/handlers/router/router-handler.test.ts index b69d6f4cf02..3f0fdbb4e28 100644 --- a/apps/sim/executor/handlers/router/router-handler.test.ts +++ b/apps/sim/executor/handlers/router/router-handler.test.ts @@ -187,6 +187,27 @@ describe('RouterBlockHandler', () => { expect(handler.canHandle(nonRouterBlock)).toBe(false) }) + it('selects the same legacy destination when a fallback provider answers', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce({ + content: 'target-block-1', + model: 'claude-sonnet-5', + tokens: { input: 10, output: 2, total: 12 }, + cost: 0.001, + }) + const output = await handler.execute(mockContext, mockBlock, { + prompt: 'Pick a destination', + model: 'gpt-4o', + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + expect(output).toMatchObject({ + model: 'claude-sonnet-5', + selectedPath: { blockId: 'target-block-1' }, + }) + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + }) + it('should execute router block correctly and select a path', async () => { const inputs = { prompt: 'Choose the best option.', @@ -788,6 +809,69 @@ describe('RouterBlockHandler V2', () => { expect(handler.canHandle(mockRouterV2Block)).toBe(true) }) + it('preserves route selection and reasoning when an Auto request falls back', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce({ + content: '{"route":"route-support","reasoning":"Needs assistance"}', + model: 'claude-sonnet-5', + tokens: { input: 10, output: 2, total: 12 }, + cost: { input: 0.0008, output: 0.0002, total: 0.001 }, + }) + const output = await handler.execute(mockContext, mockRouterV2Block, { + context: 'Help me', + model: 'sim-auto', + routes: [{ id: 'route-support', title: 'Support', value: 'Needs help' }], + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + expect(output).toMatchObject({ + model: 'claude-sonnet-5', + selectedRoute: 'route-support', + reasoning: 'Needs assistance', + cost: { total: 0.003 }, + }) + expect(mockExecuteProviderRequest.mock.calls[1][1].systemPrompt).toBe( + 'Generated V2 System Prompt' + ) + expect(mockExecuteProviderRequest.mock.calls[1][1].responseFormat).toEqual( + mockExecuteProviderRequest.mock.calls[0][1].responseFormat + ) + }) + + it('waits for the final retry before using a Router V2 fallback', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + await expect( + handler.execute( + mockContext, + mockRouterV2Block, + { + context: 'Help me', + model: 'gpt-4o', + routes: [{ id: 'route-support', title: 'Support', value: 'Needs help' }], + fallbackModels: [{ model: 'claude-sonnet-5' }], + }, + { nodeId: mockRouterV2Block.id, retry: { attempt: 1, maxTries: 2, isFinalTry: false } } + ) + ).rejects.toThrow('overloaded') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + + it('does not ask a fallback to override a NO_MATCH decision', async () => { + mockExecuteProviderRequest.mockResolvedValueOnce({ + content: '{"route":"NO_MATCH","reasoning":"Unrelated"}', + model: 'gpt-4o', + }) + await expect( + handler.execute(mockContext, mockRouterV2Block, { + context: 'Unrelated', + model: 'gpt-4o', + routes: [{ id: 'route-support', title: 'Support', value: 'Needs help' }], + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toThrow('Router could not determine a matching route') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + it('should execute router V2 and return reasoning', async () => { const inputs = { context: 'I need help with a billing issue', diff --git a/apps/sim/executor/handlers/router/router-handler.ts b/apps/sim/executor/handlers/router/router-handler.ts index 613ff0ce239..1b3ad1d9711 100644 --- a/apps/sim/executor/handlers/router/router-handler.ts +++ b/apps/sim/executor/handlers/router/router-handler.ts @@ -16,8 +16,8 @@ import { isRouterV2BlockType, ROUTER, } from '@/executor/constants' -import type { BlockHandler, ExecutionContext } from '@/executor/types' -import { executeBlockProviderRequest } from '@/executor/utils/provider-request' +import type { BlockHandler, BlockNodeMetadata, ExecutionContext } from '@/executor/types' +import { executeModelRequestWithFallbacks } from '@/executor/utils/model-fallback-request' import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal' import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry' import { resolveVertexCredential } from '@/executor/utils/vertex-credential' @@ -49,15 +49,16 @@ export class RouterBlockHandler implements BlockHandler { async execute( ctx: ExecutionContext, block: SerializedBlock, - inputs: Record + inputs: Record, + nodeMetadata?: BlockNodeMetadata ): Promise { const isV2 = isRouterV2BlockType(block.metadata?.id) if (isV2) { - return this.executeV2(ctx, block, inputs) + return this.executeV2(ctx, block, inputs, nodeMetadata) } - return this.executeLegacy(ctx, block, inputs) + return this.executeLegacy(ctx, block, inputs, nodeMetadata) } /** @@ -66,7 +67,8 @@ export class RouterBlockHandler implements BlockHandler { private async executeLegacy( ctx: ExecutionContext, block: SerializedBlock, - inputs: Record + inputs: Record, + nodeMetadata?: BlockNodeMetadata ): Promise { const promptModelInputPaths: ResolvedSecretInputPath[] = [['prompt']] const modelInputProjection = projectResolvedModelInput( @@ -139,7 +141,12 @@ export class RouterBlockHandler implements BlockHandler { workspaceId: ctx.workspaceId, } - const result = await executeBlockProviderRequest({ + const { result, usedFallback } = await executeModelRequestWithFallbacks({ + block, + configuredModel: routerConfig.model, + fallbackModels: inputs.fallbackModels, + fallbackSystemPrompt: systemPrompt, + retry: nodeMetadata?.retry, ctx, providerId, request: providerRequest, @@ -172,7 +179,7 @@ export class RouterBlockHandler implements BlockHandler { return { prompt: inputs.prompt, - model: resolved.autoRouting ? SIM_AUTO_MODEL_ID : result.model, + model: resolved.autoRouting && !usedFallback ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: tokens.input || DEFAULTS.TOKENS.PROMPT, output: tokens.output || DEFAULTS.TOKENS.COMPLETION, @@ -206,7 +213,8 @@ export class RouterBlockHandler implements BlockHandler { private async executeV2( ctx: ExecutionContext, block: SerializedBlock, - inputs: Record + inputs: Record, + nodeMetadata?: BlockNodeMetadata ): Promise { const routes = this.parseRoutes(inputs.routes) @@ -321,7 +329,12 @@ export class RouterBlockHandler implements BlockHandler { }, } - const result = await executeBlockProviderRequest({ + const { result, usedFallback } = await executeModelRequestWithFallbacks({ + block, + configuredModel: routerConfig.model, + fallbackModels: inputs.fallbackModels, + fallbackSystemPrompt: systemPrompt, + retry: nodeMetadata?.retry, ctx, providerId, request: providerRequest, @@ -389,7 +402,7 @@ export class RouterBlockHandler implements BlockHandler { return { context: inputs.context, - model: resolved.autoRouting ? SIM_AUTO_MODEL_ID : result.model, + model: resolved.autoRouting && !usedFallback ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: tokens.input || DEFAULTS.TOKENS.PROMPT, output: tokens.output || DEFAULTS.TOKENS.COMPLETION, diff --git a/apps/sim/executor/utils/model-fallback-request.test.ts b/apps/sim/executor/utils/model-fallback-request.test.ts new file mode 100644 index 00000000000..6bbcc93e0c7 --- /dev/null +++ b/apps/sim/executor/utils/model-fallback-request.test.ts @@ -0,0 +1,279 @@ +/** + * @vitest-environment node + */ +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionContext } from '@/executor/types' +import { executeModelRequestWithFallbacks } from '@/executor/utils/model-fallback-request' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import type { SerializedBlock } from '@/serializer/types' + +const { request, validateModel } = vi.hoisted(() => ({ + request: vi.fn(), + validateModel: vi.fn(), +})) + +vi.mock('@/executor/utils/provider-request', () => ({ executeBlockProviderRequest: request })) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + validateModelProvider: validateModel, +})) +vi.mock('@/providers/utils', () => ({ + getProviderFromModel: (model: string) => { + if (model.startsWith('claude')) return 'anthropic' + if (model.startsWith('vertex/')) return 'vertex' + return 'openai' + }, +})) + +function context(): ExecutionContext { + return { + workflowId: 'workflow-1', + userId: 'user-1', + workspaceId: 'workspace-1', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), + blockStates: new Map(), + blockLogs: [ + { + blockId: 'block-1', + startedAt: '', + endedAt: '', + durationMs: 0, + success: false, + executionOrder: 1, + }, + ], + metadata: { duration: 0 }, + environmentVariables: {}, + decisions: { router: new Map(), condition: new Map() }, + loopExecutions: new Map(), + completedLoops: new Set(), + executedBlocks: new Set(), + activeExecutionPath: new Set(), + } +} + +const block: SerializedBlock = { + id: 'block-1', + metadata: { id: 'evaluator' }, + position: { x: 0, y: 0 }, + config: { tool: 'evaluator', params: {} }, + inputs: {}, + outputs: {}, + enabled: true, +} + +function input() { + return { + ctx: context(), + block, + providerId: 'openai', + configuredModel: 'gpt-4o', + request: { + model: 'gpt-4o', + apiKey: 'primary-key', + systemPrompt: 'Score the content', + temperature: 0.3, + }, + fallbackSystemPrompt: 'Score the content', + fallbackModels: [{ model: 'claude-sonnet-5' }, { model: 'gpt-4o-mini' }], + resolvedSecretTraceRegistry: undefined, + } +} + +beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isHosted: false }) + request.mockReset().mockImplementation(async ({ request: candidate }) => ({ + content: '{}', + model: candidate.model, + })) + validateModel.mockReset().mockResolvedValue(undefined) +}) +afterEach(resetEnvFlagsMock) + +describe('executeModelRequestWithFallbacks', () => { + it('keeps a successful primary request unchanged and clears earlier fallback metadata', async () => { + const options = input() + options.ctx.blockLogs[0].modelFallbacks = ['old-model'] + const output = await executeModelRequestWithFallbacks(options) + expect(output).toMatchObject({ result: { model: 'gpt-4o' }, usedFallback: false }) + expect(request).toHaveBeenCalledTimes(1) + expect(request.mock.calls[0][0].request).toBe(options.request) + expect(options.ctx.blockLogs[0].modelFallbacks).toBeUndefined() + }) + + it('walks the chain in order, preserving the last error and recording earlier failed models', async () => { + const options = input() + const last = new Error('last provider failed') + request + .mockRejectedValueOnce(new Error('first')) + .mockRejectedValueOnce(new Error('second')) + .mockRejectedValueOnce(last) + await expect(executeModelRequestWithFallbacks(options)).rejects.toBe(last) + expect(request.mock.calls.map(([args]) => args.request.model)).toEqual([ + 'gpt-4o', + 'claude-sonnet-5', + 'gpt-4o-mini', + ]) + expect(options.ctx.blockLogs[0].modelFallbacks).toEqual(['gpt-4o', 'claude-sonnet-5']) + }) + + it.each([1, 2])('leaves primary attempt %i to the executor retry policy', async (attempt) => { + const error = new Error('overloaded') + request.mockRejectedValueOnce(error) + await expect( + executeModelRequestWithFallbacks({ + ...input(), + retry: { attempt, maxTries: 3, isFinalTry: false }, + }) + ).rejects.toBe(error) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('uses fallbacks on the final primary try', async () => { + request.mockRejectedValueOnce(new Error('overloaded')) + expect( + await executeModelRequestWithFallbacks({ + ...input(), + retry: { attempt: 3, maxTries: 3, isFinalTry: true }, + }) + ).toMatchObject({ usedFallback: true, result: { model: 'claude-sonnet-5' } }) + }) + + it('skips denied and incompatible provider families without sending them a request', async () => { + request.mockRejectedValueOnce(new Error('overloaded')) + validateModel.mockImplementation(async (_user, _workspace, model) => { + if (model === 'claude-sonnet-5') throw new Error('not permitted') + }) + const options = { + ...input(), + fallbackModels: [ + { model: 'claude-sonnet-5' }, + { model: 'vertex/gemini-3.1-pro' }, + { model: 'gpt-4o-mini' }, + ], + } + const output = await executeModelRequestWithFallbacks(options) + expect(output.result.model).toBe('gpt-4o-mini') + expect(request).toHaveBeenCalledTimes(2) + expect(options.ctx.blockLogs[0].modelFallbacks).toEqual(['gpt-4o']) + }) + + it('returns the original error when all remaining candidates are skipped', async () => { + const error = new Error('overloaded') + request.mockRejectedValueOnce(error) + validateModel.mockRejectedValue(new Error('not permitted')) + await expect(executeModelRequestWithFallbacks(input())).rejects.toBe(error) + expect(request).toHaveBeenCalledTimes(1) + }) + + it.each([ + Object.assign(new Error('stopped'), { name: 'AbortError' }), + Object.assign(new Error('permanent'), { retryable: false }), + ])('does not fall back after a non-retryable error', async (error) => { + request.mockRejectedValueOnce(error) + await expect(executeModelRequestWithFallbacks(input())).rejects.toBe(error) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('stops when cancelled between attempts', async () => { + const controller = new AbortController() + const error = new Error('provider failed during cancellation') + request.mockImplementationOnce(async () => { + controller.abort() + throw error + }) + const options = input() + options.ctx.abortSignal = controller.signal + await expect(executeModelRequestWithFallbacks(options)).rejects.toBe(error) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('only sends a cross-provider key that was stored as a reference, and drops family credentials', async () => { + const options = input() + const rows = [{ model: 'claude-sonnet-5', apiKey: '{{ANTHROPIC_KEY}}', thinkingLevel: 'high' }] + options.block = { ...block, config: { ...block.config, params: { fallbackModels: rows } } } + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ + ...options, + request: { + ...options.request, + vertexProject: 'private-project', + bedrockSecretKey: 'private-key', + responseFormat: { name: 'scores', schema: { type: 'object' }, strict: true }, + }, + fallbackModels: [{ ...rows[0], apiKey: 'resolved-key' }], + }) + const fallback = request.mock.calls[1][0] + expect(fallback).toMatchObject({ + providerId: 'anthropic', + request: { + apiKey: 'resolved-key', + thinkingLevel: 'high', + temperature: 0.3, + responseFormat: { name: 'scores' }, + }, + }) + expect(fallback.request.vertexProject).toBeUndefined() + expect(fallback.request.bedrockSecretKey).toBeUndefined() + }) + + it.each(['raw-key', '{{MISSING_KEY}}'])( + 'does not send an unsafe or unresolved key: %s', + async (key) => { + const options = input() + const rows = [{ model: 'claude-sonnet-5', apiKey: key }] + options.block = { ...block, config: { ...block.config, params: { fallbackModels: rows } } } + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ ...options, fallbackModels: rows }) + expect(request.mock.calls[1][0].request.apiKey).toBeUndefined() + } + ) + + it('uses the primary key on the same provider even if a row retained an old key', async () => { + const options = input() + options.block = { + ...block, + config: { + ...block.config, + params: { fallbackModels: [{ model: 'gpt-4o-mini', apiKey: '{{OLD_KEY}}' }] }, + }, + } + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ + ...options, + fallbackModels: [{ model: 'gpt-4o-mini', apiKey: 'old-key' }], + }) + expect(request.mock.calls[1][0].request.apiKey).toBe('primary-key') + }) + + it('lets hosted fallbacks resolve platform or BYOK credentials instead of a stale row key', async () => { + setEnvFlags({ isHosted: true }) + const options = input() + options.block = { + ...block, + config: { + ...block.config, + params: { fallbackModels: [{ model: 'claude-sonnet-5', apiKey: '{{OLD_KEY}}' }] }, + }, + } + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ + ...options, + fallbackModels: [{ model: 'claude-sonnet-5', apiKey: 'old-key' }], + }) + expect(request.mock.calls[1][0].request.apiKey).toBeUndefined() + }) + + it('strips the Auto identity prompt from fallbacks and keeps the pool model out of the trace', async () => { + const options = input() + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ + ...options, + configuredModel: 'sim-auto', + request: { ...options.request, systemPrompt: 'Auto identity\n\nScore the content' }, + }) + expect(request.mock.calls[1][0].request.systemPrompt).toBe('Score the content') + expect(options.ctx.blockLogs[0].modelFallbacks).toEqual(['sim-auto']) + }) +}) diff --git a/apps/sim/executor/utils/model-fallback-request.ts b/apps/sim/executor/utils/model-fallback-request.ts new file mode 100644 index 00000000000..17b26763a67 --- /dev/null +++ b/apps/sim/executor/utils/model-fallback-request.ts @@ -0,0 +1,150 @@ +import { createLogger } from '@sim/logger' +import { omit } from '@sim/utils/object' +import { resolveFallbackTuning } from '@/lib/workflows/blocks/fallback-models' +import { providerRequiresFamilyCredentials } from '@/blocks/utils' +import { validateModelProvider } from '@/ee/access-control/utils/permission-check' +import { isRetryableBlockError } from '@/executor/execution/block-retry' +import type { BlockRetryAttempt, ExecutionContext } from '@/executor/types' +import { + getModelFallbacks, + PROVIDER_FAMILY_CREDENTIAL_FIELDS, + recordModelFallbacks, + resolveFallbackApiKey, +} from '@/executor/utils/model-fallbacks' +import { executeBlockProviderRequest } from '@/executor/utils/provider-request' +import { projectResolvedSecretDiagnosticError } from '@/executor/utils/resolved-secret-content-projection' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models' +import type { ProviderRequest, ProviderResponse } from '@/providers/types' +import { getProviderFromModel } from '@/providers/utils' +import type { SerializedBlock } from '@/serializer/types' + +const logger = createLogger('BlockModelFallbacks') + +interface ModelFallbackRequestInput { + ctx: ExecutionContext + block: SerializedBlock + providerId: string + request: ProviderRequest + configuredModel: string + fallbackModels: unknown + /** The original system prompt, before an Auto identity preamble was added. */ + fallbackSystemPrompt: string + retry?: BlockRetryAttempt + resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined +} + +/** + * Shared non-streaming model execution for Router and Evaluator. Block retries + * exhaust the primary first; the final try walks the ordered fallback models. + * Parsing a successful response remains the handler's job, so a routing decision + * such as NO_MATCH is never silently replaced by another model's decision. + */ +export async function executeModelRequestWithFallbacks({ + ctx, + block, + providerId, + request, + configuredModel, + fallbackModels, + fallbackSystemPrompt, + retry, + resolvedSecretTraceRegistry, +}: ModelFallbackRequestInput): Promise<{ result: ProviderResponse; usedFallback: boolean }> { + const fallbacks = getModelFallbacks(ctx, block, fallbackModels, logger) + const candidates = [ + { model: request.model }, + ...(retry && !retry.isFinalTry + ? [] + : fallbacks.filter( + (candidate) => candidate.model.toLowerCase() !== request.model.toLowerCase() + )), + ] + const failedModels: string[] = [] + let lastError: unknown + + for (const [index, candidate] of candidates.entries()) { + const isPrimary = index === 0 + if (!isPrimary && ctx.abortSignal?.aborted) break + let candidateProviderId = providerId + if (!isPrimary) { + try { + candidateProviderId = getProviderFromModel(candidate.model) + await validateModelProvider(ctx.userId, ctx.workspaceId, candidate.model, ctx) + if ( + candidateProviderId !== providerId && + providerRequiresFamilyCredentials(candidateProviderId) + ) { + throw new Error('Fallback requires credentials from a different provider family') + } + } catch (error) { + logger.warn( + 'Fallback model unusable; skipping', + projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry, { + blockId: block.id, + model: candidate.model, + }) + ) + continue + } + } + + let candidateRequest = request + if (!isPrimary) { + const sameProvider = candidateProviderId === providerId + const { adjustments: _adjustments, ...tuning } = resolveFallbackTuning( + candidate, + configuredModel, + request + ) + candidateRequest = { + ...(sameProvider ? request : omit(request, [...PROVIDER_FAMILY_CREDENTIAL_FIELDS])), + ...tuning, + temperature: tuning.temperature === undefined ? undefined : Number(tuning.temperature), + maxTokens: tuning.maxTokens === undefined ? undefined : Number(tuning.maxTokens), + model: candidate.model, + apiKey: resolveFallbackApiKey({ + candidate, + configuredModel, + sameProvider, + primaryApiKey: request.apiKey, + blockId: block.id, + logger, + }), + systemPrompt: fallbackSystemPrompt, + } + } + + try { + const result = await executeBlockProviderRequest({ + ctx, + providerId: candidateProviderId, + request: candidateRequest, + resolvedSecretTraceRegistry, + }) + recordModelFallbacks(ctx, block, failedModels) + return { result, usedFallback: !isPrimary } + } catch (error) { + lastError = error + failedModels.push( + isPrimary && isAutoModel(configuredModel) ? SIM_AUTO_MODEL_ID : candidate.model + ) + if ( + index === candidates.length - 1 || + ctx.abortSignal?.aborted || + !isRetryableBlockError(error) + ) + break + logger.warn( + 'Model request failed; trying fallback', + projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry, { + blockId: block.id, + failedModel: candidate.model, + }) + ) + } + } + + recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) + throw lastError +} diff --git a/apps/sim/executor/utils/model-fallbacks.ts b/apps/sim/executor/utils/model-fallbacks.ts new file mode 100644 index 00000000000..84e29e366b6 --- /dev/null +++ b/apps/sim/executor/utils/model-fallbacks.ts @@ -0,0 +1,109 @@ +import type { createLogger } from '@sim/logger' +import { isPlainRecord } from '@sim/utils/object' +import { + type FallbackModelCandidate, + fallbackRowNeedsApiKey, + isWholeEnvVarReference, + normalizeFallbackModels, +} from '@/lib/workflows/blocks/fallback-models' +import type { ExecutionContext } from '@/executor/types' +import { projectResolvedSecretDiagnosticContent } from '@/executor/utils/resolved-secret-content-projection' +import type { SerializedBlock } from '@/serializer/types' + +type Logger = ReturnType + +/** Credentials scoped to one provider family, never forwarded across providers. */ +export const PROVIDER_FAMILY_CREDENTIAL_FIELDS = [ + 'azureEndpoint', + 'azureApiVersion', + 'vertexProject', + 'vertexLocation', + 'bedrockAccessKeyId', + 'bedrockSecretKey', + 'bedrockRegion', +] as const + +/** Validates stored key references before admitting their resolved values to a provider. */ +export function getModelFallbacks( + ctx: ExecutionContext, + block: SerializedBlock, + rows: unknown, + logger: Logger +): FallbackModelCandidate[] { + if (!Array.isArray(rows)) return [] + const storedRows: unknown = block.config?.params?.fallbackModels + return normalizeFallbackModels( + rows.map((row, index) => { + if (!isPlainRecord(row) || row.apiKey === undefined) return row + const stored = Array.isArray(storedRows) ? storedRows[index] : undefined + if (isPlainRecord(stored) && isWholeEnvVarReference(stored.apiKey)) return row + const projection = projectResolvedSecretDiagnosticContent( + { blockId: block.id, model: row.model, row: index + 1 }, + ctx.resolvedSecretTraceRegistry + ) + logger.warn( + 'Fallback row key ignored; only an environment variable reference is accepted', + projection.safe && isPlainRecord(projection.value) + ? projection.value + : { blockId: block.id, row: index + 1 } + ) + return { ...row, apiKey: undefined } + }) + ) +} + +/** A hidden or unresolved row key cannot override the primary, BYOK, or platform key. */ +export function resolveFallbackApiKey({ + candidate, + configuredModel, + sameProvider, + primaryApiKey, + blockId, + logger, +}: { + candidate: FallbackModelCandidate + configuredModel: string + sameProvider: boolean + primaryApiKey: string | undefined + blockId: string + logger: Logger +}): string | undefined { + let rowKey = fallbackRowNeedsApiKey(candidate.model, configuredModel) + ? candidate.apiKey + : undefined + if (rowKey && isWholeEnvVarReference(rowKey)) { + logger.warn('Fallback key variable is not set for this run', { + blockId, + model: candidate.model, + variable: rowKey, + }) + rowKey = undefined + } + return rowKey ?? (sameProvider ? primaryApiKey : undefined) +} + +/** Records failed candidates on the current attempt's log, projecting secret-derived model names. */ +export function recordModelFallbacks( + ctx: ExecutionContext, + block: SerializedBlock, + failedModels: string[] +): void { + const logs = ctx.blockLogs ?? [] + for (let index = logs.length - 1; index >= 0; index--) { + const entry = logs[index] + if (entry.blockId !== block.id || entry.endedAt !== '') continue + if (failedModels.length === 0) { + entry.modelFallbacks = undefined + return + } + const registry = ctx.errorResolvedSecretTraceRegistry ?? ctx.resolvedSecretTraceRegistry + const projection = projectResolvedSecretDiagnosticContent({ models: failedModels }, registry) + const models = + projection.safe && isPlainRecord(projection.value) ? projection.value.models : undefined + entry.modelFallbacks = + Array.isArray(models) && models.every((model) => typeof model === 'string') + ? [...models] + : undefined + return + } +}