diff --git a/apps/docs/content/docs/workflows/blocks/agent.mdx b/apps/docs/content/docs/workflows/blocks/agent.mdx index dde3a58689c..0b28b7feb88 100644 --- a/apps/docs/content/docs/workflows/blocks/agent.mdx +++ b/apps/docs/content/docs/workflows/blocks/agent.mdx @@ -25,7 +25,7 @@ Answer in two sentences, cite the doc you used, and never guess a price. ### Model -The model that runs the step. Defaults to `claude-sonnet-4-6`. Type or pick any model from OpenAI, Anthropic, Google, xAI, Groq, Cerebras, DeepSeek, Azure, AWS Bedrock, Google Vertex, or OpenRouter, or a local model through Ollama or VLLM. +The model that runs the step. Defaults to `claude-sonnet-5`. Type or pick any model from OpenAI, Anthropic, Google, xAI, Groq, Cerebras, DeepSeek, Azure, AWS Bedrock, Google Vertex, or OpenRouter, or a local model through Ollama or VLLM. For a custom cloud deployment, enter its provider prefix and model ID: `azure/my-deployment`, `azure-anthropic/my-deployment`, `bedrock/my-inference-profile`, or `vertex/my-gemini-model`. The prefix selects the provider and shows its credential fields even when the ID is absent from the catalog. Bedrock accepts full inference profile ARNs after `bedrock/`; Vertex uses the Gemini API and accepts Google model resource names. The deployment must support the selected provider's API. Custom IDs have no catalog pricing or token limits. @@ -87,6 +87,8 @@ Some settings live under advanced, or appear only for models that support them: - **Reasoning effort / Thinking level.** For models with extended reasoning, how much the model thinks before answering. Higher is more thorough but slower and costs more tokens. - **Prompt caching.** For Anthropic Claude models, reuses the system prompt and tool definitions between runs instead of re-reading them every time. Cached input costs a tenth of the normal rate, but writing the cache costs 1.25x, so leave it off for one-off runs and turn it on when the same agent runs repeatedly. The cache covers a prefix only if it reaches 1,024 tokens (2,048 on Haiku) — below that Anthropic ignores it and nothing changes. Entries expire after five minutes of no use. - **API key.** Your key for the chosen provider. Hidden on hosted Sim, which supplies one. +- **Fallback models.** An ordered list of up to five models to try when the request to the selected model fails, whether the provider is overloaded, rate-limited, or down. Sim tries the 2nd choice, then the 3rd, and so on, once each, and `` reports the model that answered. On hosted Sim, hosted models use your workspace's BYOK or platform credentials; local and self-hosted installations may still require a key. A model that needs its own key takes it from a workspace environment variable you pick on the row; a model on the same provider as the selected model reuses the block's key. A stored row key stops applying when its key field is hidden. Providers that require family-specific credentials, such as Vertex, can only be fallbacks for a selected model of the same family. The Auto model cannot be a fallback. A fallback runs with the selected model's settings where its provider accepts them: temperature and max output tokens are clamped to the fallback's limits, and when the fallback has a reasoning effort, thinking level, or verbosity setting that the selected model's value does not fit, the row shows that field so you can pick a value for it; leave it empty and the provider's default applies. +- **Retry on fail.** Retries the selected model after a failure, up to a maximum number of tries with a wait between them. When its tries run out, the fallback models are tried in order, once each, with no wait before the first of them. A fallback is never retried. A failure that happens after the model already called a tool runs that conversation again on the next try or the next model, so keep fallbacks and retry off for agents whose tools must not repeat. OpenAI and Gemini cache automatically at no extra cost and need no setting; their discount is already reflected in what you are charged. @@ -148,4 +150,5 @@ The Agent reads the message from Start with `` and returns a result diff --git a/apps/docs/content/docs/workflows/blocks/evaluator.mdx b/apps/docs/content/docs/workflows/blocks/evaluator.mdx index db480409d4d..7b5716cd5bc 100644 --- a/apps/docs/content/docs/workflows/blocks/evaluator.mdx +++ b/apps/docs/content/docs/workflows/blocks/evaluator.mdx @@ -32,6 +32,12 @@ The content to score. Usually an earlier output like ``. Structur The model that does the scoring, defaulting to `claude-sonnet-4-6`. Stronger reasoning models give more consistent scores. Type or pick any supported model. **Temperature** and a **System Prompt** are available under advanced, and on hosted Sim the API key is supplied for you. +### Fallback models + +Under **Additional fields**, add up to five models to try in order when a model request fails. With **Retry on fail** enabled, the selected model exhausts its tries first; each fallback is then tried once. Every attempt uses the same content, metrics, and response schema. `` reports the model that answered. + +Hosted models on hosted Sim use workspace BYOK or platform credentials. On local or self-hosted installations, a fallback on another provider may need a secret selected on its row; same-provider fallbacks reuse the selected model's key. The picker shows supported tuning fields when the selected model's settings cannot be inherited. Auto cannot be a fallback. + ## Outputs The Evaluator returns a number for each metric, read by the metric's lowercase name: diff --git a/apps/docs/content/docs/workflows/blocks/router.mdx b/apps/docs/content/docs/workflows/blocks/router.mdx index 06d183f369e..1f6fead37ba 100644 --- a/apps/docs/content/docs/workflows/blocks/router.mdx +++ b/apps/docs/content/docs/workflows/blocks/router.mdx @@ -35,6 +35,12 @@ Each route is a **title** and a **description** of when to choose it ("Route her The model that makes the decision, defaulting to `claude-sonnet-4-6`. Stronger reasoning models route more accurately; a faster, cheaper model is fine when the routes are clearly distinct. Type or pick any supported model, or a local one through Ollama or VLLM. On hosted Sim the API key is supplied for you. +### Fallback models + +Under **Additional fields**, add up to five models to try in order when a model request fails. With **Retry on fail** enabled, the selected model exhausts its tries first; each fallback is then tried once. Every attempt uses the same context and route definitions. `` reports the model that answered. This also works for existing legacy Router blocks. + +Hosted models on hosted Sim use workspace BYOK or platform credentials. On local or self-hosted installations, a fallback on another provider may need a secret selected on its row; same-provider fallbacks reuse the selected model's key. Auto cannot be a fallback. A completed `NO_MATCH` decision still takes the error path; it does not trigger another model request. + ## Outputs | Output | What it is | diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx index e97cc186c3f..38a774317b5 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx @@ -694,6 +694,9 @@ const TraceDetailPane = memo(function TraceDetailPane({ span }: { span: TraceSpa if (span.tries !== undefined) metaEntries.push({ label: 'Tries', value: String(span.tries) }) if (span.provider) metaEntries.push({ label: 'Provider', value: span.provider }) if (span.model) metaEntries.push({ label: 'Model', value: span.model }) + for (const failedModel of span.modelFallbacks ?? []) { + metaEntries.push({ label: 'Failed model', value: failedModel }) + } if (span.finishReason) metaEntries.push({ label: 'Finish reason', value: span.finishReason }) const ttftFormatted = formatTtft(span.ttft) if (ttftFormatted) metaEntries.push({ label: 'TTFT', value: ttftFormatted }) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts index 921e0c15285..0ec29554681 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts @@ -18,6 +18,7 @@ export { LongInput } from './long-input' export { McpDynamicArgs } from './mcp-dynamic-args' export { McpServerSelector, McpToolSelector } from './mcp-server-modal' export { MessagesInput } from './messages-input' +export { ModelFallbackList } from './model-fallback-list' export { maskSecretText, PASSWORD_MASKED_SUBBLOCK_TYPES } from './password-mask' export { ResponseFormat } from './response' export { ScheduleInfo } from './schedule-info' diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/index.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/index.ts new file mode 100644 index 00000000000..630453e439a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/index.ts @@ -0,0 +1 @@ +export { ModelFallbackList } from './model-fallback-list' diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx new file mode 100644 index 00000000000..701db95d900 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.test.tsx @@ -0,0 +1,275 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot } from 'react-dom/client' +import { renderToStaticMarkup } from 'react-dom/server' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + getDeploymentShape, + resetDeploymentShape, + resolveDeploymentShape, + seedDeploymentShape, +} from '@/lib/core/config/deployment-shape' + +const { subBlockValues, mockSetValue } = vi.hoisted(() => ({ + subBlockValues: { + model: 'claude-sonnet-5' as string, + fallbackModels: [] as Array<{ + id: string + model: string + apiKey?: string + reasoningEffort?: string + }>, + }, + mockSetValue: vi.fn(), +})) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) + +vi.mock('@sim/emcn', () => ({ + Chip: ({ + children, + disabled, + 'aria-label': ariaLabel, + }: { + children?: React.ReactNode + disabled?: boolean + 'aria-label'?: string + }) => ( + + ), + ChipCombobox: ({ + options, + value, + placeholder, + }: { + options: Array<{ value: string; label: string; disabled?: boolean }> + value?: string + placeholder?: string + }) => ( +
+ {options.map((option) => ( + + {option.label} + + ))} +
+ ), + ChipDropdown: ({ + options, + value, + placeholder, + }: { + options: Array<{ value: string; label: string }> + value?: string + placeholder?: string + }) => ( +
+ {options.map((option) => ( + {option.label} + ))} +
+ ), + Label: ({ children }: { children?: React.ReactNode }) => {children}, + Tooltip: { + Root: ({ children }: { children?: React.ReactNode }) => <>{children}, + Trigger: ({ children }: { children?: React.ReactNode }) => <>{children}, + Content: () => null, + }, +})) + +vi.mock('@sim/emcn/icons', () => ({ + ChevronDown: () => null, + ChevronUp: () => null, + Plus: () => null, + Trash: () => null, +})) + +vi.mock( + '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value', + () => ({ + useSubBlockValue: (_blockId: string, subBlockId: string) => [ + subBlockId === 'model' || subBlockId === 'fallbackModels' ? subBlockValues[subBlockId] : null, + mockSetValue, + ], + }) +) + +vi.mock('@/hooks/queries/environment', () => ({ + usePersonalEnvironment: () => ({ data: { PERSONAL_KEY: 'x' } }), + useWorkspaceEnvironment: () => ({ + data: { workspace: { OPENROUTER_API_KEY: 'x' }, personal: {}, conflicts: [] }, + }), +})) + +vi.mock('@/hooks/use-permission-config', () => ({ + usePermissionConfig: () => ({ isModelUsable: (model: string) => model !== 'denied-model' }), +})) + +vi.mock('@/hooks/use-settings-navigation', () => ({ + useSettingsNavigation: () => ({ navigateToSettings: vi.fn() }), +})) + +vi.mock('@/lib/credentials/client-state', () => ({ + writePendingCredentialCreateRequest: vi.fn(), +})) + +vi.mock('@/stores/providers/store', () => ({ + useProvidersStore: (selector: (state: { providers: object }) => unknown) => + selector({ providers: {} }), +})) + +vi.mock('@/blocks/utils', () => ({ + shouldRequireApiKeyForModel: (model: string) => + model.startsWith('openrouter/') || (model.startsWith('gpt') && !getDeploymentShape().hosted), + getModelOptions: () => [ + { id: 'claude-sonnet-5', label: 'claude-sonnet-5' }, + { id: 'gpt-5', label: 'gpt-5' }, + { id: 'denied-model', label: 'denied-model' }, + { id: 'openrouter/x', label: 'openrouter/x' }, + { id: 'sim-auto', label: 'Auto' }, + ], +})) + +vi.mock('@/lib/workflows/blocks/fallback-models', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + isViableFallbackModel: (model: string, primary: string) => + model !== 'sim-auto' && model !== primary, + getFallbackTuningKnobsToShow: (model: string) => (model === 'gpt-5' ? ['reasoningEffort'] : []), + getTuningOptionsForModel: (model: string, knob: string) => + model === 'gpt-5' && knob === 'reasoningEffort' ? ['auto', 'low', 'high'] : null, + } +}) + +import { ModelFallbackList } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list' + +beforeEach(() => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + seedDeploymentShape({ ...resolveDeploymentShape(), hosted: true }) +}) + +afterEach(() => { + resetDeploymentShape() + vi.unstubAllGlobals() +}) + +function render(extra: Partial> = {}) { + return renderToStaticMarkup( + + ) +} + +describe('ModelFallbackList', () => { + beforeEach(() => { + subBlockValues.model = 'claude-sonnet-5' + subBlockValues.fallbackModels = [] + mockSetValue.mockReset() + }) + + it('renders only the add affordance when nothing is configured', () => { + const html = render() + expect(html).toContain('Add fallback model') + expect(html).not.toContain('choice') + }) + + it('labels rows as ordinal choices and offers viable, permitted models', () => { + subBlockValues.fallbackModels = [ + { id: 'r1', model: 'gpt-5' }, + { id: 'r2', model: '' }, + ] + const html = render() + expect(html).toContain('2nd choice') + expect(html).toContain('3rd choice') + expect(html).not.toContain('Auto') + expect(html).not.toContain('denied-model') + /** The primary is never offered. A model another row holds is disabled there, never in its own row. */ + expect(html).not.toContain('>claude-sonnet-5<') + expect(html.match(/data-disabled="true">gpt-5gpt-5 { + subBlockValues.fallbackModels = [ + { id: 'r1', model: 'openrouter/x', apiKey: 'sk-raw-through-socket' }, + ] + const html = render() + expect(html).not.toContain('aria-label="Move up"') + expect(html).not.toContain('sk-raw-through-socket') + expect(html).toContain('data-combobox="Select a secret" data-value=""') + }) + + it('asks for an environment variable only when the row model needs its own key', () => { + subBlockValues.fallbackModels = [{ id: 'r1', model: 'gpt-5' }] + expect(render()).not.toContain('data-combobox="Select a secret"') + + subBlockValues.fallbackModels = [ + { id: 'r1', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + ] + const html = render() + expect(html).toContain('data-combobox="Select a secret"') + expect(html).toContain('data-value="{{OPENROUTER_API_KEY}}"') + expect(html).toContain('OPENROUTER_API_KEY') + expect(html).toContain('Create Secret') + }) + + it('updates key visibility when hosted context arrives after mount, without rewriting the rows', async () => { + seedDeploymentShape({ ...resolveDeploymentShape(), hosted: false }) + subBlockValues.fallbackModels = [{ id: 'r1', model: 'gpt-5' }] + const container = document.createElement('div') + const root = createRoot(container) + try { + await act(async () => { + root.render() + }) + expect(container.querySelector('[data-combobox="Select a secret"]')).not.toBeNull() + + await act(async () => { + seedDeploymentShape({ ...resolveDeploymentShape(), hosted: true }) + }) + expect(container.querySelector('[data-combobox="Select a secret"]')).toBeNull() + expect(mockSetValue).not.toHaveBeenCalled() + } finally { + await act(async () => root.unmount()) + } + }) + + it('shows a tuning field only for the knobs the helper says need one', () => { + subBlockValues.fallbackModels = [ + { id: 'r1', model: 'gpt-5', reasoningEffort: 'low' }, + { id: 'r2', model: 'openrouter/x' }, + ] + const html = render() + expect(html).toContain('data-combobox="Select reasoning effort" data-value="low"') + expect(html.match(/Select reasoning effort/g)).toHaveLength(1) + expect(html).not.toContain('Thinking level') + }) + + it('gates a preview against the previewed primary, not the live block', () => { + /** The live block selects claude-sonnet-5; the previewed version selected gpt-5. */ + const html = render({ + isPreview: true, + previewValue: [{ id: 'r1', model: 'openrouter/x' }], + previewPrimary: { model: 'gpt-5' }, + }) + expect(html).not.toContain('>gpt-5<') + expect(html).toContain('>claude-sonnet-5<') + expect(html).not.toContain('Add fallback model') + }) + + it('disables the add affordance at the cap', () => { + subBlockValues.fallbackModels = Array.from({ length: 5 }, (_, i) => ({ + id: `r${i}`, + model: `m-${i}`, + })) + const html = render() + expect(html).toMatch(/]*disabled=""[^>]*>Add fallback model/) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx new file mode 100644 index 00000000000..15b7bcefdd1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/model-fallback-list/model-fallback-list.tsx @@ -0,0 +1,424 @@ +'use client' + +import { memo, useCallback, useEffect, useMemo, useRef } from 'react' +import { Chip, ChipCombobox, ChipDropdown, type ComboboxOption, Label, Tooltip } from '@sim/emcn' +import { ChevronDown, ChevronUp, Plus, Trash } from '@sim/emcn/icons' +import { generateShortId } from '@sim/utils/id' +import { useParams } from 'next/navigation' +import { useDeploymentShape } from '@/lib/core/config/deployment-shape' +import { writePendingCredentialCreateRequest } from '@/lib/credentials/client-state' +import { + addFallbackRow, + changeFallbackRowApiKey, + changeFallbackRowModel, + changeFallbackRowTuning, + FALLBACK_TUNING_LABELS, + type FallbackModelEntry, + type FallbackTuningKnob, + fallbackRowNeedsApiKey, + getFallbackTuningKnobsToShow, + getTuningOptionsForModel, + isViableFallbackModel, + isWholeEnvVarReference, + MAX_FALLBACK_MODELS, + moveFallbackRow, + ordinalChoiceLabel, + removeFallbackRow, +} from '@/lib/workflows/blocks/fallback-models' +import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' +import { getModelOptions } from '@/blocks/utils' +import { usePersonalEnvironment, useWorkspaceEnvironment } from '@/hooks/queries/environment' +import { usePermissionConfig } from '@/hooks/use-permission-config' +import { useSettingsNavigation } from '@/hooks/use-settings-navigation' +import { useProvidersStore } from '@/stores/providers/store' + +const CREATE_SECRET_VALUE = 'action-create-secret' + +/** The sibling values a preview renders against, since the store holds the live block's. */ +export interface FallbackListPreviewPrimary { + model?: unknown + reasoningEffort?: unknown + thinkingLevel?: unknown + verbosity?: unknown +} + +interface ModelFallbackListProps { + blockId: string + subBlockId: string + isPreview?: boolean + previewValue?: FallbackModelEntry[] | null + /** Required for a faithful preview; ignored outside preview mode. */ + previewPrimary?: FallbackListPreviewPrimary + disabled?: boolean +} + +/** A viable model before the per-row `disabled` flag is stamped on it. */ +interface ViableModelOption { + label: string + value: string + icon?: React.ComponentType<{ className?: string }> +} + +interface FallbackRowProps { + row: FallbackModelEntry + index: number + /** The row can move down only while another follows it. */ + isLast: boolean + /** Move controls render only once a second row exists. */ + canMove: boolean + primaryModel: string + primaryTuning: Partial> + viableOptions: ViableModelOption[] + /** Models any row holds; a row's own model is exempted when its options are built. */ + takenModels: ReadonlySet + envVarOptions: ComboboxOption[] + readOnly: boolean + onChangeModel: (id: string, model: string) => void + onChangeApiKey: (id: string, apiKey: string) => void + onChangeTuning: (id: string, knob: FallbackTuningKnob, value: string) => void + onMove: (id: string, direction: -1 | 1) => void + onRemove: (id: string) => void +} + +const FallbackRow = memo(function FallbackRow({ + row, + index, + isLast, + canMove, + primaryModel, + primaryTuning, + viableOptions, + takenModels, + envVarOptions, + readOnly, + onChangeModel, + onChangeApiKey, + onChangeTuning, + onMove, + onRemove, +}: FallbackRowProps) { + /** Credential visibility follows the server-resolved shape, including late hydration. */ + useDeploymentShape() + const modelOptions = useMemo( + (): ComboboxOption[] => + viableOptions.map((option) => ({ + ...option, + disabled: option.value !== row.model && takenModels.has(option.value), + })), + [viableOptions, takenModels, row.model] + ) + + const needsApiKey = fallbackRowNeedsApiKey(row.model, primaryModel) + const tuningFields = getFallbackTuningKnobsToShow(row.model, primaryModel, primaryTuning).map( + (knob) => ({ + knob, + options: (getTuningOptionsForModel(row.model, knob) ?? []).map((value) => ({ + label: value, + value, + })), + }) + ) + + /** Only a reference is ever shown; anything else that reached the store reads as unset. */ + const apiKeyValue = isWholeEnvVarReference(row.apiKey) ? row.apiKey : '' + + return ( +
+
+ {ordinalChoiceLabel(index)} +
+ {canMove && ( + <> + + + onMove(row.id, -1)} + disabled={readOnly || index === 0} + aria-label='Move up' + /> + + Move up + + + + onMove(row.id, 1)} + disabled={readOnly || isLast} + aria-label='Move down' + /> + + Move down + + + )} + + + onRemove(row.id)} + disabled={readOnly} + aria-label='Remove fallback model' + /> + + Remove + +
+
+ +
+ onChangeModel(row.id, model)} + placeholder='Select a model' + aria-label={`${ordinalChoiceLabel(index)} model`} + disabled={readOnly} + searchable + searchPlaceholder='Search models...' + maxHeight={240} + emptyMessage='No models available' + /> + {needsApiKey && ( +
+ + onChangeApiKey(row.id, apiKey)} + placeholder='Select a secret' + aria-label={`${ordinalChoiceLabel(index)} API key`} + disabled={readOnly} + searchable + searchPlaceholder='Search secrets...' + maxHeight={240} + emptyMessage='No secrets' + /> +
+ )} + {tuningFields.map(({ knob, options }) => ( +
+ + onChangeTuning(row.id, knob, value)} + placeholder={`Select ${FALLBACK_TUNING_LABELS[knob].toLowerCase()}`} + aria-label={`${ordinalChoiceLabel(index)} ${FALLBACK_TUNING_LABELS[knob].toLowerCase()}`} + disabled={readOnly} + className='w-full' + /> +
+ ))} +
+
+ ) +}) + +/** + * Ordered fallback models for a model-driven block: the 2nd, 3rd, ... choice + * tried in sequence when the request to the block's own model fails. + * + * Every write is the whole array, so a collaborator's concurrent edit and an + * undo both flow straight through the store. A row's key is stored only as a + * `{{ENV_VAR}}` reference: the picker offers the workspace's secret names and + * nothing else, which is what keeps a raw secret out of the list value (see + * `FallbackModelEntry`). The row transforms live in `fallback-models.ts`. + */ +export function ModelFallbackList({ + blockId, + subBlockId, + isPreview = false, + previewValue, + previewPrimary, + disabled = false, +}: ModelFallbackListProps) { + const params = useParams() + const workspaceId = typeof params?.workspaceId === 'string' ? params.workspaceId : '' + const { navigateToSettings } = useSettingsNavigation() + const { isModelUsable } = usePermissionConfig() + const deploymentShape = useDeploymentShape() + const providers = useProvidersStore((state) => state.providers) + const [storeValue, setStoreValue] = useSubBlockValue(blockId, subBlockId) + const [primaryModelValue] = useSubBlockValue(blockId, 'model') + const [primaryReasoningEffort] = useSubBlockValue(blockId, 'reasoningEffort') + const [primaryThinkingLevel] = useSubBlockValue(blockId, 'thinkingLevel') + const [primaryVerbosity] = useSubBlockValue(blockId, 'verbosity') + const { data: personalEnv = {} } = usePersonalEnvironment() + const { data: workspaceEnv } = useWorkspaceEnvironment(workspaceId, { + enabled: Boolean(workspaceId), + }) + + const readOnly = isPreview || disabled + /** A preview shows another version's rows, so its gates read that version's primary, not the live one. */ + const primarySource = isPreview + ? { + model: previewPrimary?.model, + reasoningEffort: previewPrimary?.reasoningEffort, + thinkingLevel: previewPrimary?.thinkingLevel, + verbosity: previewPrimary?.verbosity, + } + : { + model: primaryModelValue, + reasoningEffort: primaryReasoningEffort, + thinkingLevel: primaryThinkingLevel, + verbosity: primaryVerbosity, + } + const primaryModel = typeof primarySource.model === 'string' ? primarySource.model : '' + const { + reasoningEffort: sourceReasoningEffort, + thinkingLevel: sourceThinkingLevel, + verbosity: sourceVerbosity, + } = primarySource + const primaryTuning = useMemo( + () => ({ + reasoningEffort: sourceReasoningEffort, + thinkingLevel: sourceThinkingLevel, + verbosity: sourceVerbosity, + }), + [sourceReasoningEffort, sourceThinkingLevel, sourceVerbosity] + ) + const rows: FallbackModelEntry[] = useMemo(() => { + const value = isPreview ? previewValue : storeValue + return Array.isArray(value) ? value : [] + }, [isPreview, previewValue, storeValue]) + + /** + * `getModelOptions` reads the providers store itself; subscribing to + * `providers` here is what recomputes the list when a dynamic provider's + * models finish loading. + */ + const viableOptions = useMemo( + (): ViableModelOption[] => + getModelOptions() + .filter( + (option) => isModelUsable(option.id) && isViableFallbackModel(option.id, primaryModel) + ) + .map((option) => ({ + label: option.label, + value: option.id, + ...(option.icon ? { icon: option.icon } : {}), + })), + [primaryModel, isModelUsable, providers, deploymentShape] + ) + + const takenModels = useMemo(() => new Set(rows.map((row) => row.model).filter(Boolean)), [rows]) + + const envVarOptions = useMemo((): ComboboxOption[] => { + const names = workspaceId + ? [ + ...Object.keys(workspaceEnv?.workspace ?? {}), + ...Object.keys(workspaceEnv?.personal ?? {}), + ] + : Object.keys(personalEnv) + const options: ComboboxOption[] = [...new Set(names)].map((name) => ({ + label: name, + value: `{{${name}}}`, + })) + options.push({ + label: 'Create Secret', + value: CREATE_SECRET_VALUE, + icon: Plus, + onSelect: () => { + if (workspaceId) { + writePendingCredentialCreateRequest({ + workspaceId, + type: 'env_personal', + requestedAt: Date.now(), + }) + } + navigateToSettings({ section: 'secrets' }) + }, + }) + return options + }, [workspaceId, workspaceEnv, personalEnv, navigateToSettings]) + + /** + * Handlers read the latest rows through a ref so their identity survives an + * edit; otherwise every keystroke in one row would re-render all of them. + */ + const rowsRef = useRef(rows) + useEffect(() => { + rowsRef.current = rows + }, [rows]) + + const write = useCallback( + (transform: (current: FallbackModelEntry[]) => FallbackModelEntry[]) => { + if (readOnly) return + const current = rowsRef.current + const next = transform(current) + if (next !== current) setStoreValue(next) + }, + [readOnly, setStoreValue] + ) + + const handleAdd = useCallback( + () => write((current) => addFallbackRow(current, generateShortId())), + [write] + ) + const handleRemove = useCallback( + (id: string) => write((current) => removeFallbackRow(current, id)), + [write] + ) + const handleMove = useCallback( + (id: string, direction: -1 | 1) => write((current) => moveFallbackRow(current, id, direction)), + [write] + ) + const handleChangeModel = useCallback( + (id: string, model: string) => + write((current) => changeFallbackRowModel(current, id, model, primaryModel)), + [primaryModel, write] + ) + const handleChangeTuning = useCallback( + (id: string, knob: FallbackTuningKnob, value: string) => + write((current) => changeFallbackRowTuning(current, id, knob, value)), + [write] + ) + const handleChangeApiKey = useCallback( + (id: string, apiKey: string) => { + if (apiKey === CREATE_SECRET_VALUE) return + write((current) => changeFallbackRowApiKey(current, id, apiKey)) + }, + [write] + ) + + return ( +
+ {rows.map((row, index) => ( + 1} + primaryModel={primaryModel} + primaryTuning={primaryTuning} + viableOptions={viableOptions} + takenModels={takenModels} + envVarOptions={envVarOptions} + readOnly={readOnly} + onChangeModel={handleChangeModel} + onChangeApiKey={handleChangeApiKey} + onChangeTuning={handleChangeTuning} + onMove={handleMove} + onRemove={handleRemove} + /> + ))} + {!readOnly && ( + = MAX_FALLBACK_MODELS} + > + Add fallback model + + )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx index 2fd31809eac..17045ccc159 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx @@ -11,6 +11,7 @@ import { import { isEqual } from 'es-toolkit' import { useParams } from 'next/navigation' import type { FilterRule, SortRule } from '@/lib/table/query-builder/constants' +import type { FallbackModelEntry } from '@/lib/workflows/blocks/fallback-models' import { CheckboxList, Code, @@ -32,6 +33,7 @@ import { McpServerSelector, McpToolSelector, MessagesInput, + ModelFallbackList, ResponseFormat, ScheduleInfo, SelectorInput, @@ -1196,6 +1198,27 @@ function SubBlockComponent({ } return } + case 'model-fallback-list': + return ( + + ) + case 'messages-input': return ( = { 'messages-input': MessageSquareText, 'tool-input': Wrench, 'skill-input': Sparkles, + 'model-fallback-list': ArrowLeftRight, 'oauth-input': Key, switch: ToggleLeft, 'file-upload': Paperclip, @@ -574,6 +576,11 @@ const SubBlockRow = memo(function SubBlockRow({ [subBlock, rawValue, workspaceSkills] ) + const fallbackModelsDisplayValue = useMemo( + () => resolveFallbackModelsLabel(subBlock, rawValue), + [subBlock, rawValue] + ) + /** * Hydrates the Function block's sandbox id to its name. Deliberately scoped to * the sandbox row: this row is memoized per subblock, and the shared list query @@ -605,6 +612,7 @@ const SubBlockRow = memo(function SubBlockRow({ filterDisplayValue || toolsDisplayValue || skillsDisplayValue || + fallbackModelsDisplayValue || sandboxDisplayValue || knowledgeBaseDisplayName || workflowSelectionName || diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx index b3935655903..04dfea0021a 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx @@ -23,6 +23,7 @@ import { getDisplayValue, hasDisplayableRowValue, resolveDropdownLabel, + resolveFallbackModelsLabel, resolveFolderPathLabel, resolveSkillsLabel, resolveToolsLabel, @@ -151,6 +152,7 @@ function resolvePreviewDisplayValue( // schema/registry fallbacks rather than the API. const toolsDisplay = resolveToolsLabel(subBlock, rawValue, []) const skillsDisplay = resolveSkillsLabel(subBlock, rawValue, []) + const fallbackModelsDisplay = resolveFallbackModelsLabel(subBlock, rawValue) const workflowName = resolveWorkflowSelectionLabel(subBlock, rawValue, workflowLookup) const workflowMultiSelectionNames = resolveWorkflowMultiSelectLabel( subBlock, @@ -165,6 +167,7 @@ function resolvePreviewDisplayValue( variablesDisplay || toolsDisplay || skillsDisplay || + fallbackModelsDisplay || workflowName || workflowMultiSelectionNames || /* diff --git a/apps/sim/blocks/blocks.test.ts b/apps/sim/blocks/blocks.test.ts index 693b2d00bdf..15077a66fdb 100644 --- a/apps/sim/blocks/blocks.test.ts +++ b/apps/sim/blocks/blocks.test.ts @@ -604,6 +604,7 @@ describe.concurrent('Blocks Module', () => { 'text', 'router-input', 'table-selector', + 'model-fallback-list', 'column-selector', 'filter-builder', 'sort-builder', diff --git a/apps/sim/blocks/blocks/agent.test.ts b/apps/sim/blocks/blocks/agent.test.ts index 23ec48fed05..9be09f82a0a 100644 --- a/apps/sim/blocks/blocks/agent.test.ts +++ b/apps/sim/blocks/blocks/agent.test.ts @@ -30,6 +30,27 @@ describe('AgentBlock', () => { } describe('tools.config.params function', () => { + it('normalizes fallback models and drops the key when none survive', () => { + const withRows = paramsFunction({ + model: 'gpt-4o', + fallbackModels: [ + { id: 'a', model: ' claude-sonnet-5 ' }, + { id: 'b', model: 'sim-auto' }, + { id: 'c', model: 'claude-sonnet-5' }, + { id: 'd', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { id: 'e', model: 'openrouter/y', apiKey: '' }, + ], + }) + expect(withRows.fallbackModels).toEqual([ + { model: 'claude-sonnet-5' }, + { model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { model: 'openrouter/y' }, + ]) + + const empty = paramsFunction({ model: 'gpt-4o', fallbackModels: [{ id: 'a', model: '' }] }) + expect(empty).not.toHaveProperty('fallbackModels') + }) + it('should pass through params when no tools array is provided', () => { const params = { model: 'gpt-4o', diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index b09eae6e203..82464855edd 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -1,5 +1,8 @@ import { createLogger } from '@sim/logger' +import { omit } from '@sim/utils/object' import { AgentIcon } from '@/components/icons' +import { normalizeFallbackModels } from '@/lib/workflows/blocks/fallback-models' +import { getModelFallbackSubBlock, MODEL_FALLBACK_INPUTS } from '@/blocks/model-fallbacks' import type { BlockConfig } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' import { @@ -31,6 +34,7 @@ const logger = createLogger('AgentBlock') /** Model the agent block falls back to when `model` is unset or the auto pseudo-model. */ const AGENT_FALLBACK_MODEL = 'claude-sonnet-5' + const MODELS_WITH_REASONING_EFFORT = getModelsWithReasoningEffort() const MODELS_WITH_VERBOSITY = getModelsWithVerbosity() const MODELS_WITH_THINKING = getModelsWithThinking() @@ -429,6 +433,7 @@ Return ONLY the JSON array.`, value: MODELS_WITH_DEEP_RESEARCH, }, }, + getModelFallbackSubBlock(), ], tools: { access: [ @@ -448,7 +453,12 @@ Return ONLY the JSON array.`, }, params: (params: Record) => { const normalizedFiles = normalizeFileInput(params.files) - const baseParams = normalizedFiles ? { ...params, files: normalizedFiles } : params + const withFiles = normalizedFiles ? { ...params, files: normalizedFiles } : params + const fallbackModels = normalizeFallbackModels(params.fallbackModels) + const baseParams = + fallbackModels.length > 0 + ? { ...withFiles, fallbackModels } + : omit(withFiles, ['fallbackModels']) // If tools array is provided, handle tool usage control if (params.tools && Array.isArray(params.tools)) { @@ -586,6 +596,7 @@ Return ONLY the JSON array.`, type: 'boolean', description: 'Cache the system prompt and tool definitions on models that support it', }, + ...MODEL_FALLBACK_INPUTS, tools: { type: 'json', description: 'Available tools configuration' }, skills: { type: 'json', description: 'Selected skills configuration' }, }, diff --git a/apps/sim/blocks/blocks/evaluator.ts b/apps/sim/blocks/blocks/evaluator.ts index 194183149cc..75fed5e1c06 100644 --- a/apps/sim/blocks/blocks/evaluator.ts +++ b/apps/sim/blocks/blocks/evaluator.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { ChartBarIcon } from '@/components/icons' +import { getModelFallbackSubBlock, MODEL_FALLBACK_INPUTS } from '@/blocks/model-fallbacks' import type { BlockConfig, ParamType } from '@/blocks/types' import { getModelOptions, @@ -189,6 +190,7 @@ export const EvaluatorBlock: BlockConfig = { options: getModelOptions, }, ...getProviderCredentialSubBlocks(), + getModelFallbackSubBlock(), { id: 'temperature', title: 'Temperature', @@ -294,6 +296,7 @@ export const EvaluatorBlock: BlockConfig = { }, model: { type: 'string' as ParamType, description: 'AI model to use' }, ...PROVIDER_CREDENTIAL_INPUTS, + ...MODEL_FALLBACK_INPUTS, temperature: { type: 'number' as ParamType, description: 'Response randomness level (low for consistent evaluation)', diff --git a/apps/sim/blocks/blocks/router.ts b/apps/sim/blocks/blocks/router.ts index 34ba119712e..8d9bccb9f9c 100644 --- a/apps/sim/blocks/blocks/router.ts +++ b/apps/sim/blocks/blocks/router.ts @@ -1,4 +1,5 @@ import { ConnectIcon } from '@/components/icons' +import { getModelFallbackSubBlock, MODEL_FALLBACK_INPUTS } from '@/blocks/model-fallbacks' import { AuthMode, type BlockConfig } from '@/blocks/types' import { getModelOptions, @@ -186,6 +187,7 @@ export const RouterBlock: BlockConfig = { options: getModelOptions, }, ...getProviderCredentialSubBlocks(), + getModelFallbackSubBlock(), { id: 'temperature', title: 'Temperature', @@ -221,6 +223,7 @@ export const RouterBlock: BlockConfig = { prompt: { type: 'string', description: 'Routing prompt content' }, model: { type: 'string', description: 'AI model to use' }, ...PROVIDER_CREDENTIAL_INPUTS, + ...MODEL_FALLBACK_INPUTS, temperature: { type: 'number', description: 'Response randomness level (low for consistent routing)', @@ -303,6 +306,7 @@ export const RouterV2Block: BlockConfig = { options: getModelOptions, }, ...getProviderCredentialSubBlocks(), + getModelFallbackSubBlock(), ], tools: { access: [ @@ -322,6 +326,7 @@ export const RouterV2Block: BlockConfig = { routes: { type: 'json', description: 'Route definitions with descriptions' }, model: { type: 'string', description: 'AI model to use' }, ...PROVIDER_CREDENTIAL_INPUTS, + ...MODEL_FALLBACK_INPUTS, }, outputs: { context: { type: 'string', description: 'Context used for routing' }, diff --git a/apps/sim/blocks/model-fallbacks.ts b/apps/sim/blocks/model-fallbacks.ts new file mode 100644 index 00000000000..4f449356d8d --- /dev/null +++ b/apps/sim/blocks/model-fallbacks.ts @@ -0,0 +1,19 @@ +import { MAX_FALLBACK_MODELS } from '@/lib/workflows/blocks/fallback-models' +import type { SubBlockConfig } from '@/blocks/types' + +const DESCRIPTION = `Ordered models tried once each when the selected model's request fails; with Retry on fail, after its tries run out. Each row is { model, apiKey?, reasoningEffort?, thinkingLevel?, verbosity? }. API keys must be whole {{ENV_VAR}} references and apply only when the row asks for a key. Tuning must be supported by the row model. sim-auto is not allowed. Max ${MAX_FALLBACK_MODELS}.` + +/** Shared advanced field for blocks that execute model requests. */ +export function getModelFallbackSubBlock(): SubBlockConfig { + return { + id: 'fallbackModels', + title: 'Fallback models', + type: 'model-fallback-list', + mode: 'advanced', + description: DESCRIPTION, + } +} + +export const MODEL_FALLBACK_INPUTS = { + fallbackModels: { type: 'json', description: DESCRIPTION }, +} as const diff --git a/apps/sim/blocks/utils.test.ts b/apps/sim/blocks/utils.test.ts index c260de06e87..9c1b933eff7 100644 --- a/apps/sim/blocks/utils.test.ts +++ b/apps/sim/blocks/utils.test.ts @@ -75,6 +75,8 @@ import { parseOptionalBooleanInput, parseOptionalJsonInput, parseOptionalNumberInput, + providerRequiresFamilyCredentials, + requiresProviderFamilyCredentials, } from '@/blocks/utils' import { getProviderFromModel } from '@/providers/utils' @@ -97,6 +99,45 @@ const BASE_CLOUD_MODELS: Record = { 'mistral-large-latest': 'mistral', } +describe('providerRequiresFamilyCredentials', () => { + it('answers for a provider the caller already resolved', () => { + expect(providerRequiresFamilyCredentials('vertex')).toBe(true) + expect(providerRequiresFamilyCredentials('openai')).toBe(false) + expect(providerRequiresFamilyCredentials(null)).toBe(false) + expect(providerRequiresFamilyCredentials(undefined)).toBe(false) + }) +}) + +describe('requiresProviderFamilyCredentials', () => { + beforeEach(() => { + setEnvFlags({ isHosted: false, isAzureConfigured: false, isOllamaConfigured: false }) + }) + + it('is true for Vertex, and for Bedrock until the deployment provides default credentials', () => { + expect(requiresProviderFamilyCredentials('vertex/gemini-2.5-pro')).toBe(true) + expect(requiresProviderFamilyCredentials('bedrock/my-inference-profile')).toBe(true) + vi.stubEnv('NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS', 'true') + try { + expect(requiresProviderFamilyCredentials('bedrock/my-inference-profile')).toBe(false) + } finally { + vi.unstubAllEnvs() + } + }) + + it('is true for Azure only until the deployment configures it server-side', () => { + expect(requiresProviderFamilyCredentials('azure/my-deployment')).toBe(true) + expect(requiresProviderFamilyCredentials('azure-anthropic/my-deployment')).toBe(true) + setEnvFlags({ isAzureConfigured: true }) + expect(requiresProviderFamilyCredentials('azure/my-deployment')).toBe(false) + }) + + it('is false for API-key providers, local servers, and unknown ids', () => { + expect(requiresProviderFamilyCredentials('openrouter/anthropic/claude')).toBe(false) + expect(requiresProviderFamilyCredentials('ollama/llama3')).toBe(false) + expect(requiresProviderFamilyCredentials('')).toBe(false) + }) +}) + describe('getApiKeyCondition / shouldRequireApiKeyForModel', () => { const evaluateCondition = (model: string): boolean => { const conditionFn = getApiKeyCondition() diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index bc81c39cc2a..92b43f7b483 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -1,6 +1,7 @@ import { toError } from '@sim/utils/errors' import { SimAutoIcon } from '@/components/icons' import { getDeploymentShape } from '@/lib/core/config/deployment-shape' +import { getEnv, isTruthy } from '@/lib/core/config/env' import { isOllamaConfigured } from '@/lib/core/config/env-flags' import { getScopesForService } from '@/lib/oauth/utils' import { containsReference } from '@/lib/workflows/sanitization/references' @@ -137,7 +138,12 @@ function buildModelVisibilityCondition(model: string, shouldShow: boolean) { return shouldShow ? { field: 'model', value: model } : { field: 'model', value: model, not: true } } -function shouldRequireApiKeyForModel(model: string): boolean { +/** + * Whether the block must show an API Key field for `model` on this deployment: + * false for hosted models on hosted Sim (BYOK or the platform key serve them), + * for providers with their own credential fields, and for local servers. + */ +export function shouldRequireApiKeyForModel(model: string): boolean { const normalizedModel = model.trim().toLowerCase() if (!normalizedModel) return false @@ -278,6 +284,31 @@ export function getCohereRerankerApiKeyCondition() { } } +/** + * Whether `model` can only run with credentials that live on the block beyond an + * API key: a Vertex OAuth credential, Bedrock AWS keys, or an Azure endpoint, + * unless the deployment supplies them server-side (the same env flags that hide + * those fields). The fields render only while the block's own `model` is in + * that provider family, so nothing outside the family can inherit them. + */ +export function requiresProviderFamilyCredentials(model: string): boolean { + return providerRequiresFamilyCredentials(findProviderFromModel(model.trim())) +} + +/** + * The provider-keyed half of {@link requiresProviderFamilyCredentials}, for a + * caller that has already resolved the provider and must not pay for a second + * catalog scan. + */ +export function providerRequiresFamilyCredentials(provider: string | null | undefined): boolean { + if (provider === 'vertex') return true + if (provider === 'bedrock') return !isTruthy(getEnv('NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS')) + if (provider === 'azure-openai' || provider === 'azure-anthropic') { + return !getDeploymentShape().azureConfigured + } + return false +} + function getModelProviderCondition(...providerIds: ProviderId[]) { return (values?: Record) => { const model = typeof values?.model === 'string' ? values.model : '' diff --git a/apps/sim/executor/execution/block-executor.retry.test.ts b/apps/sim/executor/execution/block-executor.retry.test.ts index e22c93104eb..d52827b0a72 100644 --- a/apps/sim/executor/execution/block-executor.retry.test.ts +++ b/apps/sim/executor/execution/block-executor.retry.test.ts @@ -119,6 +119,60 @@ describe('BlockExecutor retry', () => { await expect(executor.execute(createContext(state), createNode(block), block)).rejects.toThrow() expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][3]).not.toHaveProperty('retry') + }) + + it('tells each try where it sits in the policy, and a block without one nothing', async () => { + const block = createBlock({ enabled: true, maxTries: 3, waitBetweenTriesMs: 0 }) + const execute = vi + .fn() + .mockRejectedValueOnce(new Error('one')) + .mockRejectedValueOnce(new Error('two')) + .mockResolvedValueOnce({ ok: true }) + const state = new ExecutionState() + const executor = buildExecutor(block, { canHandle: () => true, execute }, state) + + await executor.execute(createContext(state), createNode(block), block) + + expect(execute.mock.calls.map(([, , , metadata]) => metadata.retry)).toEqual([ + { attempt: 1, maxTries: 3, isFinalTry: false }, + { attempt: 2, maxTries: 3, isFinalTry: false }, + { attempt: 3, maxTries: 3, isFinalTry: true }, + ]) + expect(execute.mock.calls[0][3].nodeId).toBe(block.id) + + const plain = createBlock() + const executePlain = vi.fn().mockResolvedValue({ ok: true }) + const plainState = new ExecutionState() + await buildExecutor( + plain, + { canHandle: () => true, execute: executePlain }, + plainState + ).execute(createContext(plainState), createNode(plain), plain) + expect(executePlain.mock.calls[0][3]).not.toHaveProperty('retry') + }) + + it('hands the same try position to a handler that takes the node', async () => { + const block = createBlock({ enabled: true, maxTries: 2, waitBetweenTriesMs: 0 }) + const executeWithNode = vi + .fn() + .mockRejectedValueOnce(new Error('one')) + .mockResolvedValueOnce({ ok: true }) + const execute = vi.fn() + const state = new ExecutionState() + const executor = buildExecutor( + block, + { canHandle: () => true, execute, executeWithNode }, + state + ) + + await executor.execute(createContext(state), createNode(block), block) + + expect(execute).not.toHaveBeenCalled() + expect(executeWithNode.mock.calls.map(([, , , metadata]) => metadata.retry)).toEqual([ + { attempt: 1, maxTries: 2, isFinalTry: false }, + { attempt: 2, maxTries: 2, isFinalTry: true }, + ]) }) it('replays any failure and succeeds on a later try', async () => { diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index e19b7a464ac..d19349ec2d1 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -44,6 +44,7 @@ import { import { type BlockHandler, type BlockLog, + type BlockRetryAttempt, type BlockState, type ExecutionContext, getNextExecutionOrder, @@ -277,11 +278,12 @@ export class BlockExecutor { * token is drained, so a replay cannot duplicate output the client has * already seen. */ - const output = await this.runHandlerWithRetry(blockCtx, block, blockLog, () => - handler.executeWithNode - ? handler.executeWithNode(blockCtx, block, resolvedInputs, nodeMetadata) - : handler.execute(blockCtx, block, resolvedInputs, nodeMetadata) - ) + const output = await this.runHandlerWithRetry(blockCtx, block, blockLog, (retry) => { + const invocationMetadata = retry ? { ...nodeMetadata, retry } : nodeMetadata + return handler.executeWithNode + ? handler.executeWithNode(blockCtx, block, resolvedInputs, invocationMetadata) + : handler.execute(blockCtx, block, resolvedInputs, invocationMetadata) + }) completedHandlerCost = readTrustedExecutionCost(output) @@ -546,15 +548,20 @@ export class BlockExecutor { * Rethrows the final try's error so the caller's catch — and with it the error * port — behaves exactly as it does for a block that never retried. Retrying * only ever delays the existing outcome; it never changes it. + * + * Each try is told where it sits in the policy (`BlockRetryAttempt`). The + * policy stays here: a handler cannot ask for another try or skip the wait, + * it can only hold work for the try after which no other follows, the way the + * Agent block keeps its fallback models for the final try. */ private async runHandlerWithRetry( ctx: ExecutionContext, block: SerializedBlock, blockLog: BlockLog | undefined, - invoke: () => Promise + invoke: (retry: BlockRetryAttempt | undefined) => Promise ): Promise { const policy = resolveBlockRetryPolicy(block) - if (!policy) return invoke() + if (!policy) return invoke(undefined) const shouldAccumulateFunctionCost = block.metadata?.id === BlockType.FUNCTION let accumulatedFunctionCost: TrustedExecutionCost | undefined @@ -562,8 +569,9 @@ export class BlockExecutor { try { for (;;) { tries++ + const isFinalTry = tries >= policy.maxTries try { - const output = await invoke() + const output = await invoke({ attempt: tries, maxTries: policy.maxTries, isFinalTry }) if (!shouldAccumulateFunctionCost || !accumulatedFunctionCost || !isRecordLike(output)) { return output } @@ -585,7 +593,6 @@ export class BlockExecutor { ) } - const isFinalTry = tries >= policy.maxTries if (isFinalTry || ctx.abortSignal?.aborted || !isRetryableBlockError(error)) { attachTrustedExecutionCost(error, accumulatedFunctionCost) throw error diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 85d14bd6e27..799281abcd2 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -18,17 +18,22 @@ import { type Mock, vi, } from 'vitest' +import { resetDeploymentShape } from '@/lib/core/config/deployment-shape' import type { AutoRoutingSignals } from '@/lib/model-router/resolve' import * as userFileBase64 from '@/lib/uploads/utils/user-file-base64.server' import { getAllBlocks } from '@/blocks' import { AGENT, BlockType, isMcpTool } from '@/executor/constants' +import type { DAGNode } from '@/executor/dag/builder' +import { BlockExecutor } from '@/executor/execution/block-executor' +import { ExecutionState } from '@/executor/execution/state' import { AgentBlockHandler } from '@/executor/handlers/agent/agent-handler' import type { AgentInputs, Message } from '@/executor/handlers/agent/types' import type { ExecutionContext, StreamingExecution } from '@/executor/types' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { VariableResolver } from '@/executor/variables/resolver' import { executeProviderRequest } from '@/providers' import { installStreamingCostPolicy } from '@/providers/cost-policy' -import { SIM_AUTO_MODEL_ID } from '@/providers/models' +import { getModelCapabilities, SIM_AUTO_MODEL_ID } from '@/providers/models' import { getProviderToolInputProvenance, getProviderToolModelInputRegistry, @@ -43,15 +48,23 @@ process.env.NEXT_PUBLIC_APP_URL = 'http://localhost:3000' const { mockDiscoverMcpServerToolsAsExecutor, mockImportWorkspaceFileSecretProvenanceForModelView, + mockValidateModelProvider, } = vi.hoisted(() => ({ mockDiscoverMcpServerToolsAsExecutor: vi.fn().mockResolvedValue([]), mockImportWorkspaceFileSecretProvenanceForModelView: vi.fn().mockResolvedValue(true), + mockValidateModelProvider: vi.fn().mockResolvedValue(undefined), })) vi.mock('@/lib/internal/mcp/discover-tools', () => ({ discoverMcpServerToolsAsExecutor: mockDiscoverMcpServerToolsAsExecutor, })) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + assertPermissionsAllowed: vi.fn().mockResolvedValue(undefined), + validateBlockType: vi.fn().mockResolvedValue(undefined), + validateModelProvider: mockValidateModelProvider, +})) + vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ importWorkspaceFileSecretProvenanceForModelView: mockImportWorkspaceFileSecretProvenanceForModelView, @@ -85,6 +98,7 @@ vi.mock('@/providers/utils', () => ({ vi.mock('@/blocks', () => ({ getAllBlocks: vi.fn().mockReturnValue([]), + getBlock: vi.fn().mockReturnValue(undefined), })) vi.mock('@/tools', () => ({ @@ -174,6 +188,7 @@ describe('AgentBlockHandler', () => { beforeEach(() => { handler = new AgentBlockHandler() vi.clearAllMocks() + mockValidateModelProvider.mockReset().mockResolvedValue(undefined) mockDiscoverMcpServerToolsAsExecutor.mockImplementation( async ({ serverId }: { serverId: string }) => [ @@ -495,6 +510,901 @@ describe('AgentBlockHandler', () => { }) }) + describe('model fallback', () => { + const baseInputs = { + model: 'gpt-4o', + userPrompt: 'Hello', + apiKey: 'primary-key', + temperature: 0.4, + } + + const providerFor = (model: string) => { + if (model.startsWith('gpt')) return 'openai' + if (model.startsWith('claude')) return 'anthropic' + if (model === 'blacklisted-model') throw new Error('provider blacklisted') + return 'openai' + } + + const providerResponse = (model: string, content = 'ok') => ({ + content, + model, + tokens: { input: 1, output: 1, total: 2 }, + toolCalls: [], + cost: 0, + timing: { total: 1 }, + }) + + const openLog = (blockId = mockBlock.id, endedAt = '') => ({ + blockId, + startedAt: '2026-01-01T00:00:00.000Z', + endedAt, + durationMs: 0, + success: false, + executionOrder: 1, + }) + + const streamingResponse = ( + chunks: string[], + options: { failBeforeFirstChunk?: Error; failAfterFirstChunk?: Error } = {} + ) => ({ + stream: new ReadableStream({ + async pull(controller) { + if (options.failBeforeFirstChunk) throw options.failBeforeFirstChunk + const chunk = chunks.shift() + if (chunk !== undefined) { + controller.enqueue(chunk) + return + } + if (options.failAfterFirstChunk) throw options.failAfterFirstChunk + controller.close() + }, + }), + execution: { output: { content: '' } }, + }) + + const drain = async (stream: ReadableStream) => { + const reader = stream.getReader() + const chunks: string[] = [] + for (;;) { + const { done, value } = await reader.read() + if (done) return chunks + chunks.push(value) + } + } + + beforeEach(() => { + setEnvFlags({ isHosted: false }) + resetDeploymentShape() + mockGetProviderFromModel.mockImplementation(providerFor) + mockValidateModelProvider.mockResolvedValue(undefined) + }) + + it('never touches the fallbacks when the primary answers', async () => { + const log = openLog() + log.modelFallbacks = ['stale-from-earlier-try'] + await handler.execute({ ...mockContext, blockLogs: [log] }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + expect(mockExecuteProviderRequest.mock.calls[0][1].model).toBe('gpt-4o') + expect(mockAgentLogger.warn).not.toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.anything() + ) + /** A try that succeeds on the primary clears what an earlier try wrote. */ + expect(log.modelFallbacks).toBeUndefined() + }) + + it('falls through to the next model with the same request and no primary-only fields', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5', 'from fallback')) + const blockLog = openLog() + const ctx = { ...mockContext, blockLogs: [blockLog] } + + const result = await handler.execute(ctx, mockBlock, { + ...baseInputs, + fallbackModels: [{ id: 'row-1', model: 'claude-sonnet-5' }], + }) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + const [primaryProvider, primaryRequest] = mockExecuteProviderRequest.mock.calls[0] + const [fallbackProvider, fallbackRequest] = mockExecuteProviderRequest.mock.calls[1] + expect(primaryProvider).toBe('openai') + expect(fallbackProvider).toBe('anthropic') + expect(fallbackRequest.model).toBe('claude-sonnet-5') + expect(fallbackRequest.messages).toEqual(primaryRequest.messages) + expect(fallbackRequest.temperature).toBe(primaryRequest.temperature) + expect((result as { model: string }).model).toBe('claude-sonnet-5') + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.objectContaining({ failedModel: 'gpt-4o', nextModel: 'claude-sonnet-5' }) + ) + expect(blockLog).toMatchObject({ modelFallbacks: ['gpt-4o'] }) + }) + + it.each(['flat', 'memory-block'] as const)( + 'preserves injected %s memories when switching providers', + async (shape) => { + const history: Message[] = [ + { role: 'user', content: 'My name is Ada.' }, + { role: 'assistant', content: 'Hello Ada.' }, + ] + const inputs: AgentInputs = { + ...baseInputs, + systemPrompt: 'Use the conversation history.', + userPrompt: 'What is my name?', + memories: + shape === 'flat' ? history : { memories: [{ key: 'conversation-1', data: history }] }, + fallbackModels: [{ model: 'claude-sonnet-5' }], + } + const original = structuredClone(inputs) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5', 'Your name is Ada.')) + + await handler.execute(mockContext, mockBlock, inputs) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + const expectedMessages = [ + { role: 'system', content: 'Use the conversation history.' }, + ...history, + { role: 'user', content: 'What is my name?' }, + ] + for (const [, request] of mockExecuteProviderRequest.mock.calls) { + expect(request.messages).toEqual(expectedMessages) + } + expect(mockExecuteProviderRequest.mock.calls[1][0]).toBe('anthropic') + expect(inputs).toEqual(original) + } + ) + + it.each([ + { memoryType: 'conversation', streaming: false }, + { memoryType: 'conversation', streaming: true }, + { memoryType: 'sliding_window', streaming: false }, + { memoryType: 'sliding_window', streaming: true }, + ] as const)( + 'preserves $memoryType history through fallback and saves each turn once (streaming=$streaming)', + async ({ memoryType, streaming }) => { + dbChainMockFns.returning.mockResolvedValue([{ id: 'memory-1' }]) + const history: Message[] = [ + { role: 'user', content: 'Hello.' }, + { role: 'assistant', content: 'How can I help?' }, + { role: 'user', content: 'My name is Ada.' }, + { role: 'assistant', content: 'Hello Ada.' }, + ] + queueTableRows(schemaMock.memory, [{ secretProvenanceVersion: null, data: history }]) + const ctx = { ...mockContext, executionId: 'memory-fallback-execution' } + const inputs: AgentInputs = { + ...baseInputs, + userPrompt: undefined, + memoryType, + slidingWindowSize: '2', + conversationId: 'conversation-1', + messages: [ + { role: 'system', content: 'Use the conversation history.' }, + { role: 'user', content: 'What is my name?' }, + ], + fallbackModels: [{ model: 'claude-sonnet-5' }], + } + const original = structuredClone(inputs) + if (streaming) { + mockExecuteProviderRequest + .mockResolvedValueOnce( + streamingResponse([], { failBeforeFirstChunk: new Error('overloaded') }) + ) + .mockResolvedValueOnce(streamingResponse(['Your name is Ada.'])) + } else { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5', 'Your name is Ada.')) + } + + const result = await handler.execute(ctx, mockBlock, inputs) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + const currentUserMessage = { + role: 'user', + content: 'What is my name?', + executionId: ctx.executionId, + } + const expectedMessages = [ + { role: 'system', content: 'Use the conversation history.' }, + ...(memoryType === 'sliding_window' ? history.slice(-2) : history), + { role: 'user', content: 'What is my name?' }, + ] + for (const [, request] of mockExecuteProviderRequest.mock.calls) { + expect(request.messages).toEqual(expectedMessages) + } + expect(mockExecuteProviderRequest.mock.calls[1][0]).toBe('anthropic') + if (streaming) { + const streamedResult = result as StreamingExecution + expect(await drain(streamedResult.stream)).toEqual(['Your name is Ada.']) + expect(streamedResult.onFullContent).toBeTypeOf('function') + await streamedResult.onFullContent?.('Your name is Ada.') + } + + const memoryWrites = dbChainMockFns.values.mock.calls + .map(([row]) => row) + .filter((row) => Array.isArray(row.data)) + expect(memoryWrites.map((row) => row.data)).toEqual([ + [currentUserMessage], + [{ role: 'assistant', content: 'Your name is Ada.' }], + ]) + expect(memoryWrites.every((row) => row.key === inputs.conversationId)).toBe(true) + expect(inputs).toEqual(original) + } + ) + + it('holds the fallbacks on a try the executor will replay', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + const blockLog = openLog() + blockLog.modelFallbacks = ['stale-from-earlier-run'] + + await expect( + handler.execute( + { ...mockContext, blockLogs: [blockLog] }, + mockBlock, + { ...baseInputs, fallbackModels: [{ model: 'claude-sonnet-5' }] }, + { nodeId: mockBlock.id, retry: { attempt: 1, maxTries: 3, isFinalTry: false } } + ) + ).rejects.toThrow('overloaded') + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + expect(mockAgentLogger.info).toHaveBeenCalledWith('Fallback models held for the final try', { + blockId: mockBlock.id, + attempt: 1, + maxTries: 3, + }) + expect(mockAgentLogger.warn).not.toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.anything() + ) + expect(blockLog.modelFallbacks).toBeUndefined() + }) + + it('walks the chain on the final try, and on a block that never retries', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + const inputs = { ...baseInputs, fallbackModels: [{ model: 'claude-sonnet-5' }] } + + const onFinalTry = await handler.execute(mockContext, mockBlock, inputs, { + nodeId: mockBlock.id, + retry: { attempt: 3, maxTries: 3, isFinalTry: true }, + }) + const withoutPolicy = await handler.execute(mockContext, mockBlock, inputs, { + nodeId: mockBlock.id, + }) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(4) + expect((onFinalTry as { model: string }).model).toBe('claude-sonnet-5') + expect((withoutPolicy as { model: string }).model).toBe('claude-sonnet-5') + expect(mockAgentLogger.info).not.toHaveBeenCalledWith( + 'Fallback models held for the final try', + expect.anything() + ) + }) + + it('says nothing about held fallbacks when none are configured', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + + await expect( + handler.execute(mockContext, mockBlock, baseInputs, { + nodeId: mockBlock.id, + retry: { attempt: 1, maxTries: 2, isFinalTry: false }, + }) + ).rejects.toThrow('overloaded') + expect(mockAgentLogger.info).not.toHaveBeenCalledWith( + 'Fallback models held for the final try', + expect.anything() + ) + }) + + it('never falls back on a deep-research follow-up turn', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + + await expect( + handler.execute(mockContext, mockBlock, { + ...baseInputs, + previousInteractionId: 'interaction-1', + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toThrow('overloaded') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + expect(mockAgentLogger.info).toHaveBeenCalledWith( + 'Fallback models skipped for a deep-research follow-up turn', + expect.objectContaining({ blockId: mockBlock.id }) + ) + }) + + it('gives a fallback its own key, the block key on the same provider, and nothing otherwise', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('one')) + .mockRejectedValueOnce(new Error('two')) + .mockRejectedValueOnce(new Error('three')) + .mockResolvedValueOnce(providerResponse('gpt-4o-mini')) + + const storedRows = [ + { model: 'claude-sonnet-5', apiKey: '{{ANTHROPIC_KEY}}' }, + { model: 'claude-haiku-5' }, + { model: 'gpt-4o-mini' }, + ] + const block = { + ...mockBlock, + config: { ...mockBlock.config, params: { fallbackModels: storedRows } }, + } + await handler.execute(mockContext, block, { + ...baseInputs, + fallbackModels: [ + { model: 'claude-sonnet-5', apiKey: 'anthropic-row-key' }, + { model: 'claude-haiku-5' }, + { model: 'gpt-4o-mini' }, + ], + }) + + const keys = mockExecuteProviderRequest.mock.calls.map(([, request]) => request.apiKey) + expect(keys).toEqual(['primary-key', 'anthropic-row-key', undefined, 'primary-key']) + }) + + it('treats a row key that was never resolved as no key and says which variable', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('one')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + + const storedRows = [{ model: 'claude-sonnet-5', apiKey: '{{MISSING_KEY}}' }] + const block = { + ...mockBlock, + config: { ...mockBlock.config, params: { fallbackModels: storedRows } }, + } + await handler.execute(mockContext, block, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5', apiKey: '{{MISSING_KEY}}' }], + }) + + expect(mockExecuteProviderRequest.mock.calls[1][1].apiKey).toBeUndefined() + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Fallback key variable is not set for this run', + expect.objectContaining({ model: 'claude-sonnet-5', variable: '{{MISSING_KEY}}' }) + ) + }) + + it.each([ + { hosted: false, primary: 'gpt-4o', fallback: 'gpt-4o-mini', expectedKey: 'primary-key' }, + { hosted: true, primary: 'gpt-4o', fallback: 'claude-sonnet-5', expectedKey: undefined }, + ])( + 'ignores a hidden row key for $fallback with hosted=$hosted', + async ({ hosted, primary, fallback, expectedKey }) => { + setEnvFlags({ isHosted: hosted }) + resetDeploymentShape() + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce(providerResponse(fallback)) + const block = { + ...mockBlock, + config: { + ...mockBlock.config, + params: { fallbackModels: [{ model: fallback, apiKey: '{{OLD_KEY}}' }] }, + }, + } + + await handler.execute(mockContext, block, { + ...baseInputs, + model: primary, + fallbackModels: [{ model: fallback, apiKey: 'old-row-key' }], + }) + + expect(mockExecuteProviderRequest.mock.calls[1][1].apiKey).toBe(expectedKey) + } + ) + + it('lets the executor retry a stream startup failure while fallbacks are held', async () => { + mockExecuteProviderRequest.mockResolvedValueOnce( + streamingResponse([], { failBeforeFirstChunk: new Error('429 at stream start') }) + ) + + await expect( + handler.execute( + mockContext, + mockBlock, + { + ...baseInputs, + stream: true, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }, + { + nodeId: mockBlock.id, + retry: { attempt: 1, maxTries: 3, isFinalTry: false }, + } + ) + ).rejects.toThrow('429 at stream start') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + + it('leaves provider-family credentials off a fallback on another provider', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('one')) + .mockRejectedValueOnce(new Error('two')) + .mockResolvedValueOnce(providerResponse('gpt-4o-mini')) + + await handler.execute(mockContext, mockBlock, { + ...baseInputs, + vertexCredential: 'vertex-secret', + bedrockSecretKey: 'bedrock-secret', + azureEndpoint: 'https://azure.example.com', + fallbackModels: [{ model: 'claude-sonnet-5' }, { model: 'gpt-4o-mini' }], + }) + + const [, crossProvider] = mockExecuteProviderRequest.mock.calls[1] + const [, sameProvider] = mockExecuteProviderRequest.mock.calls[2] + expect(crossProvider.vertexCredential).toBeUndefined() + expect(crossProvider.bedrockSecretKey).toBeUndefined() + expect(crossProvider.azureEndpoint).toBeUndefined() + expect(JSON.stringify(crossProvider)).not.toMatch( + /vertex-secret|bedrock-secret|azure\.example/ + ) + expect(sameProvider.bedrockSecretKey).toBe('bedrock-secret') + expect(sameProvider.azureEndpoint).toBe('https://azure.example.com') + }) + + it('ignores a row key the block did not store as a reference', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('one')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + const block = { + ...mockBlock, + config: { + ...mockBlock.config, + params: { + fallbackModels: [{ model: 'claude-sonnet-5', apiKey: 'sk-raw-through-socket' }], + }, + }, + } + + await handler.execute(mockContext, block, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5', apiKey: 'sk-raw-through-socket' }], + }) + + expect(mockExecuteProviderRequest.mock.calls[1][1].apiKey).toBeUndefined() + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Fallback row key ignored; only an environment variable reference is accepted', + expect.objectContaining({ model: 'claude-sonnet-5', row: 1 }) + ) + expect(JSON.stringify(mockAgentLogger.warn.mock.calls)).not.toContain('sk-raw-through-socket') + }) + + it('re-resolves tuning for the fallback: row value wins, caps clamp, undeclared values drop', async () => { + const fallbackCap = getModelCapabilities('gpt-5.4-mini')?.maxOutputTokens + expect(fallbackCap).toEqual(expect.any(Number)) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('gpt-5.4-mini')) + + await handler.execute(mockContext, mockBlock, { + ...baseInputs, + model: 'claude-sonnet-5', + thinkingLevel: 'high', + temperature: 0.9, + maxTokens: (fallbackCap as number) + 5000, + fallbackModels: [{ model: 'gpt-5.4-mini', reasoningEffort: 'low' }], + }) + + const [, primaryRequest] = mockExecuteProviderRequest.mock.calls[0] + const [, fallbackRequest] = mockExecuteProviderRequest.mock.calls[1] + expect(primaryRequest.thinkingLevel).toBe('high') + expect(primaryRequest.maxTokens).toBe((fallbackCap as number) + 5000) + expect(fallbackRequest.reasoningEffort).toBe('low') + expect(fallbackRequest.thinkingLevel).toBeUndefined() + expect(fallbackRequest.temperature).toBe(0.9) + expect(fallbackRequest.maxTokens).toBe(fallbackCap) + expect(mockAgentLogger.info).toHaveBeenCalledWith( + 'Fallback model tuning adjusted', + expect.objectContaining({ model: 'gpt-5.4-mini' }) + ) + }) + + it('rethrows the last attempted model error unchanged when every model fails', async () => { + const first = new Error('primary down') + const last = new Error('fallback down') + mockExecuteProviderRequest.mockRejectedValueOnce(first).mockRejectedValueOnce(last) + const blockLog = openLog() + + await expect( + handler.execute({ ...mockContext, blockLogs: [blockLog] }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toBe(last) + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + expect(blockLog).toMatchObject({ modelFallbacks: ['gpt-4o'] }) + }) + + it('rethrows the primary error when every fallback was skipped as unusable', async () => { + const primaryError = new Error('primary down') + mockExecuteProviderRequest.mockRejectedValueOnce(primaryError) + const blockLog = openLog() + + await expect( + handler.execute({ ...mockContext, blockLogs: [blockLog] }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'blacklisted-model' }], + }) + ).rejects.toBe(primaryError) + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + expect(blockLog.modelFallbacks).toBeUndefined() + }) + + it('skips sim-auto, duplicates, the primary itself, and unusable providers', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + + await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [ + { model: 'sim-auto' }, + { model: 'GPT-4o' }, + { model: 'blacklisted-model' }, + { model: 'claude-sonnet-5' }, + { model: 'claude-sonnet-5' }, + ], + }) + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + expect(mockExecuteProviderRequest.mock.calls[1][1].model).toBe('claude-sonnet-5') + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Fallback model unusable; skipping', + expect.objectContaining({ model: 'blacklisted-model' }) + ) + }) + + it('skips a fallback the workspace does not permit', async () => { + mockValidateModelProvider.mockImplementation(async (_user, _workspace, model: string) => { + if (model === 'claude-sonnet-5') throw new Error('Model not permitted') + }) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('gpt-4o-mini')) + + await handler.execute( + { ...mockContext, userId: 'user-1', workspaceId: 'workspace-1' }, + mockBlock, + { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }, { model: 'gpt-4o-mini' }], + } + ) + + expect(mockExecuteProviderRequest.mock.calls.map(([, request]) => request.model)).toEqual([ + 'gpt-4o', + 'gpt-4o-mini', + ]) + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Fallback model unusable; skipping', + expect.objectContaining({ model: 'claude-sonnet-5', error: 'Model not permitted' }) + ) + }) + + it('does not fall back after a stop', async () => { + const controller = new AbortController() + mockExecuteProviderRequest.mockImplementationOnce(async () => { + controller.abort() + throw new Error('Provider request timed out') + }) + + await expect( + handler.execute({ ...mockContext, abortSignal: controller.signal }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toThrow('timed out') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + + it('does not start another candidate after a stop during a skipped one', async () => { + const controller = new AbortController() + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('primary down')) + mockValidateModelProvider.mockImplementation(async (_user, _workspace, model: string) => { + if (model !== 'claude-sonnet-5') return + controller.abort() + throw new Error('not permitted') + }) + + await expect( + handler.execute({ ...mockContext, abortSignal: controller.signal }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }, { model: 'gpt-4o-mini' }], + }) + ).rejects.toThrow('primary down') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + + it('skips a fallback whose provider cannot take the attachments and hydrates once per provider', async () => { + const file = { + id: 'file-1', + name: 'example.png', + key: 'execution/test-workspace/test-workflow/exec-1/example.png', + url: 'https://storage.example.com/example.png', + size: 8, + type: 'image/png', + context: 'execution', + } + const hydrate = vi + .spyOn(userFileBase64, 'hydrateUserFilesWithBase64') + .mockImplementation(async (value) => { + const files = value as Array + return files.map((attachment) => ({ + ...attachment, + base64: 'iVBORw0KGgo=', + })) as typeof value + }) + mockGetProviderFromModel.mockImplementation((model: string) => + model.startsWith('deepseek') ? 'deepseek' : providerFor(model) + ) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('claude-haiku-5')) + const blockLog = openLog() + + try { + await handler.execute({ ...mockContext, blockLogs: [blockLog] }, mockBlock, { + ...baseInputs, + userPrompt: 'Describe this file', + files: [file], + fallbackModels: [ + { model: 'deepseek-chat' }, + { model: 'claude-sonnet-5' }, + { model: 'claude-haiku-5' }, + ], + }) + + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Fallback model cannot take the attached files; skipping', + expect.objectContaining({ model: 'deepseek-chat' }) + ) + expect(mockExecuteProviderRequest.mock.calls.map(([, request]) => request.model)).toEqual([ + 'gpt-4o', + 'claude-sonnet-5', + 'claude-haiku-5', + ]) + /** One hydration for openai, one for anthropic; the skipped provider never hydrates. */ + expect(hydrate).toHaveBeenCalledTimes(2) + /** A skipped candidate is not a failed try. */ + expect(blockLog.modelFallbacks).toEqual(['gpt-4o', 'claude-sonnet-5']) + } finally { + hydrate.mockRestore() + } + }) + + it('does not prime a stream when no candidate follows', async () => { + mockExecuteProviderRequest.mockResolvedValueOnce( + streamingResponse([], { failBeforeFirstChunk: new Error('429 at stream start') }) + ) + + const result = (await handler.execute( + mockContext, + mockBlock, + baseInputs + )) as StreamingExecution + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + await expect(drain(result.stream as ReadableStream)).rejects.toThrow( + '429 at stream start' + ) + }) + + it('does not fall back on an explicitly non-retryable failure, on any try', async () => { + const error = Object.assign(new Error('permanent'), { retryable: false }) + mockExecuteProviderRequest.mockRejectedValue(error) + const inputs = { ...baseInputs, fallbackModels: [{ model: 'claude-sonnet-5' }] } + + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toBe(error) + await expect( + handler.execute(mockContext, mockBlock, inputs, { + nodeId: mockBlock.id, + retry: { attempt: 1, maxTries: 3, isFinalTry: false }, + }) + ).rejects.toBe(error) + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + expect(mockAgentLogger.warn).not.toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.anything() + ) + }) + + it('retries the selected model under the executor policy, then walks the fallbacks once', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('one')) + .mockRejectedValueOnce(new Error('two')) + .mockRejectedValueOnce(new Error('three')) + .mockRejectedValueOnce(new Error('four')) + .mockResolvedValueOnce(providerResponse('gpt-4o-mini', 'from the third choice')) + const block = { + ...mockBlock, + config: { + tool: 'mock-tool', + params: { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }, { model: 'gpt-4o-mini' }], + }, + }, + retry: { enabled: true, maxTries: 3, waitBetweenTriesMs: 0 }, + } as SerializedBlock + const workflow = { + version: '1', + blocks: [block], + connections: [], + loops: {}, + parallels: {}, + } as SerializedWorkflow + const state = new ExecutionState() + const executor = new BlockExecutor( + [handler], + new VariableResolver(workflow, {}, state), + { + workspaceId: 'test-workspace', + executionId: 'execution-1', + userId: 'user-1', + metadata: { + requestId: 'request-1', + executionId: 'execution-1', + workflowId: 'test-workflow', + workspaceId: 'test-workspace', + userId: 'user-1', + triggerType: 'manual', + useDraftState: false, + startTime: new Date().toISOString(), + }, + }, + state + ) + const ctx = { + ...mockContext, + executionId: 'execution-1', + userId: 'user-1', + blockStates: state.getBlockStates(), + blockLogs: [], + } as ExecutionContext + const node = { + id: block.id, + block, + incomingEdges: new Set(), + outgoingEdges: new Map(), + metadata: {}, + } as unknown as DAGNode + + const output = await executor.execute(ctx, node, block) + + /** Three tries on the selected model, then each fallback exactly once. */ + expect(mockExecuteProviderRequest.mock.calls.map(([, request]) => request.model)).toEqual([ + 'gpt-4o', + 'gpt-4o', + 'gpt-4o', + 'claude-sonnet-5', + 'gpt-4o-mini', + ]) + expect(output).toMatchObject({ model: 'gpt-4o-mini' }) + expect(ctx.blockLogs[0]).toMatchObject({ + success: true, + tries: 3, + modelFallbacks: ['gpt-4o', 'claude-sonnet-5'], + }) + expect(mockAgentLogger.info).toHaveBeenCalledTimes(2) + expect(mockAgentLogger.info).toHaveBeenNthCalledWith( + 2, + 'Fallback models held for the final try', + { blockId: mockBlock.id, attempt: 2, maxTries: 3 } + ) + }) + + it('keeps the fallback name when a routed sim-auto primary fails and a fallback answers', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('pool model down')) + .mockResolvedValueOnce(providerResponse('gpt-5.4-mini')) + const blockLog = openLog() + + const result = (await handler.execute({ ...mockContext, blockLogs: [blockLog] }, mockBlock, { + model: SIM_AUTO_MODEL_ID, + systemPrompt: 'Be brief.', + userPrompt: 'Hello!', + fallbackModels: [{ model: 'gpt-5.4-mini', reasoningEffort: 'low' }], + })) as { model: string } + + expect(result.model).toBe('gpt-5.4-mini') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + /** The trace names the auto identity, never the pool model that was routed. */ + expect(blockLog.modelFallbacks).toEqual([SIM_AUTO_MODEL_ID]) + /** The row's tuning was set against the auto id in the editor, so it applies whatever was routed. */ + expect(mockExecuteProviderRequest.mock.calls[1][1].reasoningEffort).toBe('low') + /** The auto identity preamble belongs to the pool model, not a named fallback. */ + const systemText = (request: { messages?: Array<{ role: string; content: string }> }) => + (request.messages ?? []) + .filter((message) => message.role === 'system') + .map((message) => message.content) + .join('\n') + expect(systemText(mockExecuteProviderRequest.mock.calls[0][1])).toContain('Sim auto model') + expect(systemText(mockExecuteProviderRequest.mock.calls[1][1])).not.toContain( + 'Sim auto model' + ) + }) + + it('records the failed models on the open log entry, not an earlier closed one', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('down')) + .mockResolvedValueOnce(providerResponse('claude-sonnet-5')) + const closed = openLog(mockBlock.id, '2026-01-01T00:00:01.000Z') + const open = openLog() + + await handler.execute({ ...mockContext, blockLogs: [closed, open] }, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + + expect(closed.modelFallbacks).toBeUndefined() + expect(open.modelFallbacks).toEqual(['gpt-4o']) + }) + + it('falls back when a streaming candidate closes before its first chunk', async () => { + mockExecuteProviderRequest + .mockResolvedValueOnce(streamingResponse([])) + .mockResolvedValueOnce(streamingResponse(['answer'])) + + const result = (await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + })) as StreamingExecution + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.objectContaining({ error: 'Provider stream closed before its first chunk' }) + ) + await expect(drain(result.stream as ReadableStream)).resolves.toEqual(['answer']) + }) + + it('falls back when a streaming primary fails before its first chunk, and replays the first chunk otherwise', async () => { + mockExecuteProviderRequest + .mockResolvedValueOnce( + streamingResponse([], { failBeforeFirstChunk: new Error('429 at stream start') }) + ) + .mockResolvedValueOnce(streamingResponse(['first', 'second'])) + + const result = (await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + })) as StreamingExecution + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + expect(mockAgentLogger.warn).toHaveBeenCalledWith( + 'Agent model failed; trying fallback', + expect.objectContaining({ failedModel: 'gpt-4o', error: '429 at stream start' }) + ) + expect(await drain(result.stream as ReadableStream)).toEqual(['first', 'second']) + }) + + it('leaves a failure after the first chunk to the stream, as before', async () => { + const midStream = new Error('dropped mid-stream') + mockExecuteProviderRequest.mockResolvedValueOnce( + streamingResponse(['first'], { failAfterFirstChunk: midStream }) + ) + + const result = (await handler.execute(mockContext, mockBlock, { + ...baseInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + })) as StreamingExecution + + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + await expect(drain(result.stream as ReadableStream)).rejects.toBe(midStream) + }) + }) + describe('execute', () => { it('should execute a basic agent block request', async () => { const inputs = { diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index d96aeed8f74..cbc182f271a 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' +import { getErrorMessage, toError } from '@sim/utils/errors' import { isPlainRecord, omit } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import { normalizeStringRecord, normalizeWorkflowVariables } from '@/lib/core/utils/records' @@ -39,6 +39,10 @@ import { selectModelBoundFileInputPaths, } from '@/lib/uploads/utils/model-input' import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' +import { + type FallbackModelCandidate, + resolveFallbackTuning, +} from '@/lib/workflows/blocks/fallback-models' import { resolveCustomBlockToolBinding } from '@/lib/workflows/custom-blocks/operations' import { getAgentToolUsageControlMode, @@ -54,6 +58,7 @@ import { validateModelProvider, } from '@/ee/access-control/utils/permission-check' import { AGENT, BlockType, DEFAULTS, stripCustomToolPrefix } from '@/executor/constants' +import { isRetryableBlockError } from '@/executor/execution/block-retry' import { memoryService } from '@/executor/handlers/agent/memory' import { buildLoadSkillTool, @@ -68,9 +73,21 @@ import type { ToolInput, } from '@/executor/handlers/agent/types' import { parseResponseFormat } from '@/executor/handlers/shared/response-format' -import type { BlockHandler, ExecutionContext, StreamingExecution, UserFile } from '@/executor/types' +import type { + BlockHandler, + BlockNodeMetadata, + ExecutionContext, + StreamingExecution, + UserFile, +} from '@/executor/types' import { collectBlockData } from '@/executor/utils/block-data' import { stringifyJSON } from '@/executor/utils/json' +import { + getModelFallbacks, + PROVIDER_FAMILY_CREDENTIAL_FIELDS, + recordModelFallbacks, + resolveFallbackApiKey, +} from '@/executor/utils/model-fallbacks' import { projectResolvedSecretDiagnosticContent } from '@/executor/utils/resolved-secret-content-projection' import { prepareResolvedSecretProjectedInputs } from '@/executor/utils/resolved-secret-input-projection' import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal' @@ -131,6 +148,7 @@ const AGENT_RAW_PROVIDER_ERROR_INPUT_PATHS: readonly ResolvedSecretInputPath[] = ['thinkingLevel'], ['promptCaching'], ['previousInteractionId'], + ['fallbackModels'], ] interface IndexedToolInput { @@ -138,6 +156,65 @@ interface IndexedToolInput { toolIndex: number } +/** + * Removes the sim-auto identity preamble from the system messages built for a + * routed primary. A fallback the builder named is not a pool model, so it must + * not be told to hide which model it is. Messages are built once per block run + * (building them appends to memory), which is why this strips rather than + * rebuilds. + */ +function stripAutoPreamble(messages: Message[] | undefined): Message[] | undefined { + if (!messages) return messages + const prefix = `${SIM_AUTO_SYSTEM_PREAMBLE}\n\n` + return messages.flatMap((message) => { + if (message.role !== 'system' || typeof message.content !== 'string') return [message] + if (message.content === SIM_AUTO_SYSTEM_PREAMBLE) return [] + if (!message.content.startsWith(prefix)) return [message] + return [{ ...message, content: message.content.slice(prefix.length) }] + }) +} + +/** One model in the order the block tries them; the primary carries the block's own key. */ +interface ModelCandidate extends FallbackModelCandidate { + isPrimary: boolean + /** + * What the trace calls this model when it fails. A routed sim-auto primary + * shows as the auto identity, since naming the pool model is the leak that + * `applyAutoModelLabel` exists to close. + */ + traceName?: string +} + +interface ExecuteAcrossModelsConfig { + candidates: ModelCandidate[] + retryPrimaryOnStreamStart: boolean + primaryModel: string + /** + * The model the builder configured, which is what the editor showed the + * per-row tuning fields against. Under sim-auto that is the auto id, not the + * pool model routed for this run, so a row's value applies whatever was routed. + */ + configuredModel: string + primaryProviderId: string + messages: Message[] | undefined + /** Provider id to hydrated messages; seeded with the primary, filled per fallback provider. */ + hydratedByProvider: Map + fileProjection: ReturnType + modelInputs: AgentInputs + /** + * The system prompt without the sim-auto identity preamble, present only when + * the primary was auto-routed: a fallback the builder named is not a pool + * model and must not be told to hide which model it is. + */ + fallbackSystemPrompt?: string + formattedTools: ProviderToolConfig[] + responseFormat: any + streaming: boolean + settledInputRegistry: ResolvedSecretTraceRegistry | undefined + resultRegistry: ResolvedSecretTraceRegistry | undefined + providerErrorRegistry: ResolvedSecretTraceRegistry | undefined +} + interface FormattedAgentTools { tools: ProviderToolConfig[] inputProvenance: Map> @@ -229,7 +306,8 @@ export class AgentBlockHandler implements BlockHandler { async execute( ctx: ExecutionContext, block: SerializedBlock, - inputs: AgentInputs + inputs: AgentInputs, + nodeMetadata?: BlockNodeMetadata ): Promise { ctx.mcpBlockId = block.id const providerErrorRegistry = ctx.resolvedSecretTraceRegistry?.forkForInputPaths( @@ -304,6 +382,8 @@ export class AgentBlockHandler implements BlockHandler { ...modelInputProjection.value, responseFormat: responseFormatProjection.value, } + /** The system prompt as the model may see it, before any auto-routing preamble joins it. */ + const projectedSystemPrompt = modelInputs.systemPrompt const projectedToolInputs = this.projectToolInputsForProvenance(ctx, tools) await this.validateToolPermissions(ctx, filteredInputs.tools || []) @@ -391,24 +471,24 @@ export class AgentBlockHandler implements BlockHandler { skillMetadata, fileProjection ) - const messagesWithFiles = await this.hydrateMessageFilesForProvider( - ctx, - messagesWithInputFiles, - providerId, - fileProjection.projectedNameByFile, - fileProjection.modelBoundInputPaths - ) - - const providerRequest = this.buildProviderRequest({ - ctx, - providerId, - model, - messages: messagesWithFiles, - inputs: modelInputs, - formattedTools: formatted.tools, - responseFormat, - streaming: streamingConfig.shouldUseStreaming ?? false, - }) + /** + * The primary hydrates before the registries settle and fork, as it always + * has: hydration imports file provenance into the live registry, and the + * result fork below must carry it. Fallbacks on another provider hydrate + * inside the chain and re-fork there. + */ + const hydratedByProvider = new Map([ + [ + providerId, + await this.hydrateMessageFilesForProvider( + ctx, + messagesWithInputFiles, + providerId, + fileProjection.projectedNameByFile, + fileProjection.modelBoundInputPaths + ), + ], + ]) settlePrivateAgentSelectors() @@ -427,21 +507,84 @@ export class AgentBlockHandler implements BlockHandler { }) } } - const result = await this.executeProviderRequest( + + /** + * Retry on fail retries the selected model; the fallbacks join only on the + * try after which the executor promises no other. Until then a failure of + * the primary is left to escape, so the executor's policy can replay it. + * + * A follow-up turn of a deep-research interaction lives on the primary's + * provider; another model has none of that conversation, so a green answer + * from it would be built on a fresh context. Such a request never falls back. + */ + const configuredFallbacks = getModelFallbacks( ctx, - providerRequest, block, + filteredInputs.fallbackModels, + logger + ) + const retry = nodeMetadata?.retry + const fallbacksHeld = retry !== undefined && !retry.isFinalTry + const fallbackCandidates = + modelInputs.previousInteractionId || fallbacksHeld + ? [] + : configuredFallbacks.filter( + (candidate) => candidate.model.toLowerCase() !== model.toLowerCase() + ) + if (configuredFallbacks.length > 0 && modelInputs.previousInteractionId) { + logger.info('Fallback models skipped for a deep-research follow-up turn', { + blockId: block.id, + }) + } else if (configuredFallbacks.length > 0 && fallbacksHeld) { + logger.info('Fallback models held for the final try', { + blockId: block.id, + attempt: retry.attempt, + maxTries: retry.maxTries, + }) + } + const candidates: ModelCandidate[] = [ + { + model, + apiKey: modelInputs.apiKey, + isPrimary: true, + ...(autoRouting ? { traceName: SIM_AUTO_MODEL_ID } : {}), + }, + ...fallbackCandidates.map((candidate) => ({ ...candidate, isPrimary: false })), + ] + const { + result, + servedModel, + resultRegistry: servedRegistry, + } = await this.executeAcrossModels(ctx, block, { + candidates, + retryPrimaryOnStreamStart: + fallbacksHeld && configuredFallbacks.length > 0 && !modelInputs.previousInteractionId, + primaryModel: model, + configuredModel: autoRouting ? SIM_AUTO_MODEL_ID : model, + primaryProviderId: providerId, + messages: messagesWithInputFiles, + hydratedByProvider, + fileProjection, + modelInputs, + fallbackSystemPrompt: autoRouting ? projectedSystemPrompt : undefined, + formattedTools: formatted.tools, responseFormat, + streaming: streamingConfig.shouldUseStreaming ?? false, + settledInputRegistry, resultRegistry, - providerErrorRegistry - ) - if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry + providerErrorRegistry, + }) + if (servedRegistry) ctx.resolvedSecretTraceRegistry = servedRegistry if (autoRouting && autoRouting.billableRoutingCost > 0) { this.applyRoutingCost(result, autoRouting.billableRoutingCost) } - if (autoRouting) { + /** + * A fallback the builder named explicitly is not a pool model, so it keeps + * its own name; only the routed pool model hides behind the auto label. + */ + if (autoRouting && servedModel === model) { this.applyAutoModelLabel(result, model) } @@ -2302,6 +2445,297 @@ export class AgentBlockHandler implements BlockHandler { return paths } + /** + * Runs the provider request against each candidate in order until one answers. + * + * Which model serves the request is decided here, inside one handler + * invocation. How many invocations the block gets is the executor's retry + * policy, and the caller keeps the fallbacks out of the candidate list until + * the final try, so with retry on the block runs the primary alone on every + * earlier try and walks the whole chain once. + * + * Falling through is deliberately as indiscriminate as block retry + * (`isRetryableBlockError`): a provider error carries no status, so an + * overloaded upstream cannot be told from any other failure, and a builder who + * lists fallbacks wants the block to answer. Only a stop and an explicitly + * non-retryable failure end the chain early. The abort signal is checked + * before the error itself because `handleExecutionError` rewrites a timeout + * into a plain `Error` without a cause, which the predicate would then treat + * as replayable. + * + * Messages are built once by the caller, since building them appends to + * memory. Hydration runs per provider, because attachment support and the + * inline budget differ, and is cached so two candidates on one provider do + * not download the same files twice. A fallback whose provider is + * blacklisted, not permitted, or cannot take the attachments is skipped + * rather than counted as a failed try. + * + * A streaming candidate is accepted only once its first chunk has arrived + * (`primeStreamingExecution`), so a startup failure inside the stream still + * falls through; that wait is skipped when no candidate follows, which keeps + * blocks without fallbacks on today's path. + * + * When every candidate fails, the last attempted candidate's error is thrown + * exactly as it escaped `executeProviderRequest`, with the registries that + * call installed for its projection, so error ports and the block-level + * error handling see the shapes they see today. The models that failed are + * written to the block log on every exit, cleared as well as set, because + * block retry reuses one log entry across tries. + */ + private async executeAcrossModels( + ctx: ExecutionContext, + block: SerializedBlock, + config: ExecuteAcrossModelsConfig + ): Promise<{ + result: BlockOutput | StreamingExecution + servedModel: string + resultRegistry: ResolvedSecretTraceRegistry | undefined + }> { + const { hydratedByProvider } = config + let resultRegistry = config.resultRegistry + const failedModels: string[] = [] + let lastError: unknown + let lastErrorRegistries: + | { + error: ResolvedSecretTraceRegistry | undefined + resolved: ResolvedSecretTraceRegistry | undefined + } + | undefined + + for (let index = 0; index < config.candidates.length; index++) { + const candidate = config.candidates[index] + const hasNext = index < config.candidates.length - 1 + + /** A run stopped while a candidate was being skipped must not start another. */ + if (!candidate.isPrimary && ctx.abortSignal?.aborted) break + + let candidateProviderId: string + if (candidate.isPrimary) { + candidateProviderId = config.primaryProviderId + } else { + try { + candidateProviderId = getProviderFromModel(candidate.model) + await validateModelProvider(ctx.userId, ctx.workspaceId, candidate.model, ctx) + } catch (error) { + this.warnFallbackSkipped(ctx, block, candidate.model, 'unusable', error) + continue + } + } + + let messages: Message[] | undefined + if (hydratedByProvider.has(candidateProviderId)) { + messages = hydratedByProvider.get(candidateProviderId) + } else { + try { + messages = await this.hydrateMessageFilesForProvider( + ctx, + config.messages, + candidateProviderId, + config.fileProjection.projectedNameByFile, + config.fileProjection.modelBoundInputPaths + ) + } catch (error) { + if (candidate.isPrimary) throw error + this.warnFallbackSkipped( + ctx, + block, + candidate.model, + 'cannot take the attached files', + error + ) + continue + } + hydratedByProvider.set(candidateProviderId, messages) + /** Hydration imported this provider's file provenance; the result fork must carry it. */ + resultRegistry = config.settledInputRegistry?.forkForInputPaths([]) + } + + /** + * A fallback's own key applies only while its key field is visible. On the + * primary's provider it reuses the block's key; otherwise the provider layer resolves BYOK or + * the platform key, or reports that a key is required, which counts as + * this candidate failing. A previous interaction id belongs to the primary's + * provider alone. Tuning is re-resolved against the fallback's own + * capabilities so the request is one its provider accepts. + */ + let inputs: AgentInputs = config.modelInputs + if (!candidate.isPrimary) { + const { adjustments, ...tuning } = resolveFallbackTuning( + candidate, + config.configuredModel, + config.modelInputs + ) + const sameProvider = candidateProviderId === config.primaryProviderId + inputs = { + ...(sameProvider + ? config.modelInputs + : omit(config.modelInputs, [...PROVIDER_FAMILY_CREDENTIAL_FIELDS, 'vertexCredential'])), + apiKey: resolveFallbackApiKey({ + candidate, + configuredModel: config.configuredModel, + sameProvider, + primaryApiKey: config.modelInputs.apiKey, + blockId: block.id, + logger, + }), + previousInteractionId: undefined, + ...(config.fallbackSystemPrompt !== undefined + ? { systemPrompt: config.fallbackSystemPrompt } + : {}), + ...tuning, + } + if (adjustments.length > 0) { + logger.info( + 'Fallback model tuning adjusted', + projectAgentDiagnosticMetadata( + ctx, + { blockId: block.id, model: candidate.model, adjustments }, + { blockId: block.id, adjustmentCount: adjustments.length } + ) + ) + } + } + + const providerRequest = this.buildProviderRequest({ + ctx, + providerId: candidateProviderId, + model: candidate.model, + messages: + !candidate.isPrimary && config.fallbackSystemPrompt !== undefined + ? stripAutoPreamble(messages) + : messages, + inputs, + formattedTools: config.formattedTools, + responseFormat: config.responseFormat, + streaming: config.streaming, + }) + + try { + let result = await this.executeProviderRequest( + ctx, + providerRequest, + block, + config.responseFormat, + resultRegistry, + config.providerErrorRegistry + ) + if ((hasNext || config.retryPrimaryOnStreamStart) && this.isStreamingExecution(result)) { + result = await this.primeStreamingExecution(result as StreamingExecution) + } + recordModelFallbacks(ctx, block, failedModels) + return { result, servedModel: candidate.model, resultRegistry } + } catch (error) { + lastError = error + failedModels.push(candidate.traceName ?? candidate.model) + if (!hasNext || ctx.abortSignal?.aborted || !isRetryableBlockError(error)) { + recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) + throw error + } + + /** + * `executeProviderRequest` installed the failed attempt's error + * registry, which is the only one that knows secrets a tool call + * activated; the warn is projected against it before the next candidate + * starts from the settled inputs again. The pair is kept so a rethrow + * after every remaining candidate was skipped projects the same way. + */ + const errorRegistry = ctx.errorResolvedSecretTraceRegistry + const diagnosticCtx = errorRegistry + ? { ...ctx, resolvedSecretTraceRegistry: errorRegistry } + : ctx + logger.warn( + 'Agent model failed; trying fallback', + projectAgentDiagnosticMetadata( + diagnosticCtx, + { + blockId: block.id, + failedModel: candidate.model, + nextModel: config.candidates[index + 1].model, + candidate: index + 1, + error: getErrorMessage(error), + }, + { blockId: block.id, candidate: index + 1 } + ) + ) + lastErrorRegistries = { error: errorRegistry, resolved: ctx.resolvedSecretTraceRegistry } + ctx.errorResolvedSecretTraceRegistry = config.providerErrorRegistry + ctx.resolvedSecretTraceRegistry = config.settledInputRegistry + } + } + + /** Reached only when every candidate after the last failure was skipped. */ + if (lastErrorRegistries) { + ctx.errorResolvedSecretTraceRegistry = lastErrorRegistries.error + ctx.resolvedSecretTraceRegistry = lastErrorRegistries.resolved + } + recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) + throw lastError + } + + /** + * Warns that a fallback candidate was passed over, projected like every other + * diagnostic so a model id resolved from a reference never reaches the log. + * A skipped candidate is not a failed try: it never appears in `modelFallbacks`. + */ + private warnFallbackSkipped( + ctx: ExecutionContext, + block: SerializedBlock, + model: string, + reason: 'unusable' | 'cannot take the attached files', + error: unknown + ): void { + logger.warn( + `Fallback model ${reason}; skipping`, + projectAgentDiagnosticMetadata( + ctx, + { blockId: block.id, model, error: getErrorMessage(error) }, + { blockId: block.id } + ) + ) + } + + /** + * Waits for a streaming candidate's first chunk before accepting it. + * + * With tools attached, providers open the stream first and issue the initial + * upstream request inside it, so a 429 at startup would otherwise surface + * only when the executor drains the stream, past every fallback. Reading one + * chunk moves that failure back inside the candidate loop; the chunk is + * re-emitted at the head of the returned stream, and nothing has reached the + * client yet, so the next candidate cannot duplicate output. A stream that + * fails after its first chunk stays a stream failure, as it is today. + */ + private async primeStreamingExecution(result: StreamingExecution): Promise { + const reader = result.stream.getReader() + const first = await reader.read() + /** + * A stream that closes before its first chunk answered nothing; with another + * candidate waiting that is a startup failure to fall through from, not an + * empty answer to return. The last candidate is never primed, so a block + * without fallbacks still returns such a stream as it always has. + */ + if (first.done) { + throw new Error('Provider stream closed before its first chunk') + } + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(first.value) + }, + async pull(controller) { + const next = await reader.read() + if (next.done) { + controller.close() + return + } + controller.enqueue(next.value) + }, + cancel(reason) { + return reader.cancel(reason) + }, + }) + return { ...result, stream } + } + private buildProviderRequest(config: { ctx: ExecutionContext providerId: string diff --git a/apps/sim/executor/handlers/agent/types.ts b/apps/sim/executor/handlers/agent/types.ts index d2e30314589..f752859e34b 100644 --- a/apps/sim/executor/handlers/agent/types.ts +++ b/apps/sim/executor/handlers/agent/types.ts @@ -1,4 +1,5 @@ import type { McpOperationPolicy } from '@/lib/mcp/operation-policy' +import type { FallbackModelEntry } from '@/lib/workflows/blocks/fallback-models' import type { UserFile } from '@/executor/types' import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry' @@ -32,8 +33,8 @@ export interface AgentInputs { // Deep research multi-turn previousInteractionId?: string // Interactions API previous interaction reference // LLM parameters - temperature?: string - maxTokens?: string + temperature?: string | number + maxTokens?: string | number apiKey?: string azureEndpoint?: string azureApiVersion?: string @@ -48,6 +49,8 @@ export interface AgentInputs { thinkingLevel?: string promptCaching?: boolean files?: unknown + /** Ordered models tried when the request to `model` fails; see `normalizeFallbackModels`. */ + fallbackModels?: Array & { model: string }> } /** diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts index 22a337b3812..47932eb9470 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts @@ -160,6 +160,53 @@ describe('EvaluatorBlockHandler', () => { apiKey: 'test-api-key', } + it('preserves metric scores and Auto routing cost when a fallback answers', async () => { + mockGetProviderFromModel.mockImplementation((model: string) => + model.startsWith('claude') ? 'anthropic' : 'fireworks' + ) + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce({ + content: '{"score1":7}', + model: 'claude-sonnet-5', + tokens: { input: 12, output: 3, total: 15 }, + cost: { input: 0.003, output: 0.001, total: 0.004 }, + }) + const output = await handler.execute(mockContext, mockBlock, { + ...admissionInputs, + model: 'sim-auto', + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + expect(output).toMatchObject({ + content: admissionInputs.content, + model: 'claude-sonnet-5', + score1: 7, + tokens: { total: 15 }, + cost: { total: 0.006 }, + }) + const first = mockExecuteProviderRequest.mock.calls[0][1] + const fallback = mockExecuteProviderRequest.mock.calls[1][1] + expect(fallback.responseFormat).toEqual(first.responseFormat) + expect(fallback.systemPrompt).not.toContain('Sim auto system preamble') + expect(fallback.apiKey).toBeUndefined() + }) + + it('forwards retry metadata so fallbacks wait for the final primary attempt', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + await expect( + handler.execute( + mockContext, + mockBlock, + { + ...admissionInputs, + fallbackModels: [{ model: 'claude-sonnet-5' }], + }, + { nodeId: mockBlock.id, retry: { attempt: 1, maxTries: 2, isFinalTry: false } } + ) + ).rejects.toThrow('overloaded') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + it('refuses to reach the provider without an execution subject', async () => { mockContext.userId = undefined diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts index 52f0859f88a..71bf795b96d 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts @@ -9,9 +9,9 @@ import { import type { BlockOutput } from '@/blocks/types' import { validateModelProvider } from '@/ee/access-control/utils/permission-check' import { BlockType, DEFAULTS, EVALUATOR } from '@/executor/constants' -import type { BlockHandler, ExecutionContext } from '@/executor/types' +import type { BlockHandler, BlockNodeMetadata, ExecutionContext } from '@/executor/types' import { isJSONString, parseJSON, stringifyJSON } from '@/executor/utils/json' -import { executeBlockProviderRequest } from '@/executor/utils/provider-request' +import { executeModelRequestWithFallbacks } from '@/executor/utils/model-fallback-request' import { projectResolvedSecretDiagnosticError } from '@/executor/utils/resolved-secret-content-projection' import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal' import type { @@ -38,7 +38,8 @@ export class EvaluatorBlockHandler implements BlockHandler { async execute( ctx: ExecutionContext, block: SerializedBlock, - inputs: Record + inputs: Record, + nodeMetadata?: BlockNodeMetadata ): Promise { const evaluatorConfig = { model: inputs.model || EVALUATOR.DEFAULT_MODEL, @@ -142,6 +143,7 @@ export class EvaluatorBlockHandler implements BlockHandler { 'Evaluate the content and provide scores for each metric as JSON.' } + const fallbackSystemPrompt = systemPromptObj.systemPrompt let model = evaluatorConfig.model let autoRouting: AutoRoutingResult | null = null if (isAutoModel(model)) { @@ -213,7 +215,12 @@ export class EvaluatorBlockHandler implements BlockHandler { workspaceId: ctx.workspaceId, } - const result = await executeBlockProviderRequest({ + const { result, usedFallback } = await executeModelRequestWithFallbacks({ + block, + configuredModel: evaluatorConfig.model, + fallbackModels: inputs.fallbackModels, + fallbackSystemPrompt, + retry: nodeMetadata?.retry, ctx, providerId, request: providerRequest, @@ -237,7 +244,7 @@ export class EvaluatorBlockHandler implements BlockHandler { return { content: inputs.content, - model: autoRouting ? SIM_AUTO_MODEL_ID : result.model, + model: autoRouting && !usedFallback ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: inputTokens, output: outputTokens, diff --git a/apps/sim/executor/handlers/router/router-handler.test.ts b/apps/sim/executor/handlers/router/router-handler.test.ts index b69d6f4cf02..3f0fdbb4e28 100644 --- a/apps/sim/executor/handlers/router/router-handler.test.ts +++ b/apps/sim/executor/handlers/router/router-handler.test.ts @@ -187,6 +187,27 @@ describe('RouterBlockHandler', () => { expect(handler.canHandle(nonRouterBlock)).toBe(false) }) + it('selects the same legacy destination when a fallback provider answers', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce({ + content: 'target-block-1', + model: 'claude-sonnet-5', + tokens: { input: 10, output: 2, total: 12 }, + cost: 0.001, + }) + const output = await handler.execute(mockContext, mockBlock, { + prompt: 'Pick a destination', + model: 'gpt-4o', + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + expect(output).toMatchObject({ + model: 'claude-sonnet-5', + selectedPath: { blockId: 'target-block-1' }, + }) + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2) + }) + it('should execute router block correctly and select a path', async () => { const inputs = { prompt: 'Choose the best option.', @@ -788,6 +809,69 @@ describe('RouterBlockHandler V2', () => { expect(handler.canHandle(mockRouterV2Block)).toBe(true) }) + it('preserves route selection and reasoning when an Auto request falls back', async () => { + mockExecuteProviderRequest + .mockRejectedValueOnce(new Error('overloaded')) + .mockResolvedValueOnce({ + content: '{"route":"route-support","reasoning":"Needs assistance"}', + model: 'claude-sonnet-5', + tokens: { input: 10, output: 2, total: 12 }, + cost: { input: 0.0008, output: 0.0002, total: 0.001 }, + }) + const output = await handler.execute(mockContext, mockRouterV2Block, { + context: 'Help me', + model: 'sim-auto', + routes: [{ id: 'route-support', title: 'Support', value: 'Needs help' }], + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + expect(output).toMatchObject({ + model: 'claude-sonnet-5', + selectedRoute: 'route-support', + reasoning: 'Needs assistance', + cost: { total: 0.003 }, + }) + expect(mockExecuteProviderRequest.mock.calls[1][1].systemPrompt).toBe( + 'Generated V2 System Prompt' + ) + expect(mockExecuteProviderRequest.mock.calls[1][1].responseFormat).toEqual( + mockExecuteProviderRequest.mock.calls[0][1].responseFormat + ) + }) + + it('waits for the final retry before using a Router V2 fallback', async () => { + mockExecuteProviderRequest.mockRejectedValueOnce(new Error('overloaded')) + await expect( + handler.execute( + mockContext, + mockRouterV2Block, + { + context: 'Help me', + model: 'gpt-4o', + routes: [{ id: 'route-support', title: 'Support', value: 'Needs help' }], + fallbackModels: [{ model: 'claude-sonnet-5' }], + }, + { nodeId: mockRouterV2Block.id, retry: { attempt: 1, maxTries: 2, isFinalTry: false } } + ) + ).rejects.toThrow('overloaded') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + + it('does not ask a fallback to override a NO_MATCH decision', async () => { + mockExecuteProviderRequest.mockResolvedValueOnce({ + content: '{"route":"NO_MATCH","reasoning":"Unrelated"}', + model: 'gpt-4o', + }) + await expect( + handler.execute(mockContext, mockRouterV2Block, { + context: 'Unrelated', + model: 'gpt-4o', + routes: [{ id: 'route-support', title: 'Support', value: 'Needs help' }], + fallbackModels: [{ model: 'claude-sonnet-5' }], + }) + ).rejects.toThrow('Router could not determine a matching route') + expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1) + }) + it('should execute router V2 and return reasoning', async () => { const inputs = { context: 'I need help with a billing issue', diff --git a/apps/sim/executor/handlers/router/router-handler.ts b/apps/sim/executor/handlers/router/router-handler.ts index 613ff0ce239..1b3ad1d9711 100644 --- a/apps/sim/executor/handlers/router/router-handler.ts +++ b/apps/sim/executor/handlers/router/router-handler.ts @@ -16,8 +16,8 @@ import { isRouterV2BlockType, ROUTER, } from '@/executor/constants' -import type { BlockHandler, ExecutionContext } from '@/executor/types' -import { executeBlockProviderRequest } from '@/executor/utils/provider-request' +import type { BlockHandler, BlockNodeMetadata, ExecutionContext } from '@/executor/types' +import { executeModelRequestWithFallbacks } from '@/executor/utils/model-fallback-request' import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal' import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry' import { resolveVertexCredential } from '@/executor/utils/vertex-credential' @@ -49,15 +49,16 @@ export class RouterBlockHandler implements BlockHandler { async execute( ctx: ExecutionContext, block: SerializedBlock, - inputs: Record + inputs: Record, + nodeMetadata?: BlockNodeMetadata ): Promise { const isV2 = isRouterV2BlockType(block.metadata?.id) if (isV2) { - return this.executeV2(ctx, block, inputs) + return this.executeV2(ctx, block, inputs, nodeMetadata) } - return this.executeLegacy(ctx, block, inputs) + return this.executeLegacy(ctx, block, inputs, nodeMetadata) } /** @@ -66,7 +67,8 @@ export class RouterBlockHandler implements BlockHandler { private async executeLegacy( ctx: ExecutionContext, block: SerializedBlock, - inputs: Record + inputs: Record, + nodeMetadata?: BlockNodeMetadata ): Promise { const promptModelInputPaths: ResolvedSecretInputPath[] = [['prompt']] const modelInputProjection = projectResolvedModelInput( @@ -139,7 +141,12 @@ export class RouterBlockHandler implements BlockHandler { workspaceId: ctx.workspaceId, } - const result = await executeBlockProviderRequest({ + const { result, usedFallback } = await executeModelRequestWithFallbacks({ + block, + configuredModel: routerConfig.model, + fallbackModels: inputs.fallbackModels, + fallbackSystemPrompt: systemPrompt, + retry: nodeMetadata?.retry, ctx, providerId, request: providerRequest, @@ -172,7 +179,7 @@ export class RouterBlockHandler implements BlockHandler { return { prompt: inputs.prompt, - model: resolved.autoRouting ? SIM_AUTO_MODEL_ID : result.model, + model: resolved.autoRouting && !usedFallback ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: tokens.input || DEFAULTS.TOKENS.PROMPT, output: tokens.output || DEFAULTS.TOKENS.COMPLETION, @@ -206,7 +213,8 @@ export class RouterBlockHandler implements BlockHandler { private async executeV2( ctx: ExecutionContext, block: SerializedBlock, - inputs: Record + inputs: Record, + nodeMetadata?: BlockNodeMetadata ): Promise { const routes = this.parseRoutes(inputs.routes) @@ -321,7 +329,12 @@ export class RouterBlockHandler implements BlockHandler { }, } - const result = await executeBlockProviderRequest({ + const { result, usedFallback } = await executeModelRequestWithFallbacks({ + block, + configuredModel: routerConfig.model, + fallbackModels: inputs.fallbackModels, + fallbackSystemPrompt: systemPrompt, + retry: nodeMetadata?.retry, ctx, providerId, request: providerRequest, @@ -389,7 +402,7 @@ export class RouterBlockHandler implements BlockHandler { return { context: inputs.context, - model: resolved.autoRouting ? SIM_AUTO_MODEL_ID : result.model, + model: resolved.autoRouting && !usedFallback ? SIM_AUTO_MODEL_ID : result.model, tokens: { input: tokens.input || DEFAULTS.TOKENS.PROMPT, output: tokens.output || DEFAULTS.TOKENS.COMPLETION, diff --git a/apps/sim/executor/types.ts b/apps/sim/executor/types.ts index 164d3de0874..4cc1367c032 100644 --- a/apps/sim/executor/types.ts +++ b/apps/sim/executor/types.ts @@ -286,6 +286,13 @@ export interface BlockLog { errorHandled?: boolean /** Total handler tries, present only when the block retried at least once. */ tries?: number + /** + * Models that failed before the one that answered, in the order tried. + * Present only when an Agent block fell back at least once. Under retry only + * the final try walks the fallbacks, so the field reflects that try; every + * earlier try clears it. + */ + modelFallbacks?: string[] loopId?: string parallelId?: string iterationIndex?: number @@ -750,6 +757,22 @@ export interface BlockNodeMetadata { originalBlockId?: string isLoopNode?: boolean executionOrder?: number + /** Where this invocation sits in the block's retry policy; absent when the block has none. */ + retry?: BlockRetryAttempt +} + +/** + * One try of a block under its retry policy, told to the handler so it can hold + * work for the last try. `isFinalTry` is the executor's own judgment, not + * `attempt >= maxTries` recomputed by the handler: what makes a try final is the + * policy's business, and a handler that fails on a non-final try is promised + * another invocation for any retryable error. + */ +export interface BlockRetryAttempt { + /** 1-based. */ + attempt: number + maxTries: number + isFinalTry: boolean } export interface BlockHandler { diff --git a/apps/sim/executor/utils/model-fallback-request.test.ts b/apps/sim/executor/utils/model-fallback-request.test.ts new file mode 100644 index 00000000000..6bbcc93e0c7 --- /dev/null +++ b/apps/sim/executor/utils/model-fallback-request.test.ts @@ -0,0 +1,279 @@ +/** + * @vitest-environment node + */ +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionContext } from '@/executor/types' +import { executeModelRequestWithFallbacks } from '@/executor/utils/model-fallback-request' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import type { SerializedBlock } from '@/serializer/types' + +const { request, validateModel } = vi.hoisted(() => ({ + request: vi.fn(), + validateModel: vi.fn(), +})) + +vi.mock('@/executor/utils/provider-request', () => ({ executeBlockProviderRequest: request })) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + validateModelProvider: validateModel, +})) +vi.mock('@/providers/utils', () => ({ + getProviderFromModel: (model: string) => { + if (model.startsWith('claude')) return 'anthropic' + if (model.startsWith('vertex/')) return 'vertex' + return 'openai' + }, +})) + +function context(): ExecutionContext { + return { + workflowId: 'workflow-1', + userId: 'user-1', + workspaceId: 'workspace-1', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), + blockStates: new Map(), + blockLogs: [ + { + blockId: 'block-1', + startedAt: '', + endedAt: '', + durationMs: 0, + success: false, + executionOrder: 1, + }, + ], + metadata: { duration: 0 }, + environmentVariables: {}, + decisions: { router: new Map(), condition: new Map() }, + loopExecutions: new Map(), + completedLoops: new Set(), + executedBlocks: new Set(), + activeExecutionPath: new Set(), + } +} + +const block: SerializedBlock = { + id: 'block-1', + metadata: { id: 'evaluator' }, + position: { x: 0, y: 0 }, + config: { tool: 'evaluator', params: {} }, + inputs: {}, + outputs: {}, + enabled: true, +} + +function input() { + return { + ctx: context(), + block, + providerId: 'openai', + configuredModel: 'gpt-4o', + request: { + model: 'gpt-4o', + apiKey: 'primary-key', + systemPrompt: 'Score the content', + temperature: 0.3, + }, + fallbackSystemPrompt: 'Score the content', + fallbackModels: [{ model: 'claude-sonnet-5' }, { model: 'gpt-4o-mini' }], + resolvedSecretTraceRegistry: undefined, + } +} + +beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isHosted: false }) + request.mockReset().mockImplementation(async ({ request: candidate }) => ({ + content: '{}', + model: candidate.model, + })) + validateModel.mockReset().mockResolvedValue(undefined) +}) +afterEach(resetEnvFlagsMock) + +describe('executeModelRequestWithFallbacks', () => { + it('keeps a successful primary request unchanged and clears earlier fallback metadata', async () => { + const options = input() + options.ctx.blockLogs[0].modelFallbacks = ['old-model'] + const output = await executeModelRequestWithFallbacks(options) + expect(output).toMatchObject({ result: { model: 'gpt-4o' }, usedFallback: false }) + expect(request).toHaveBeenCalledTimes(1) + expect(request.mock.calls[0][0].request).toBe(options.request) + expect(options.ctx.blockLogs[0].modelFallbacks).toBeUndefined() + }) + + it('walks the chain in order, preserving the last error and recording earlier failed models', async () => { + const options = input() + const last = new Error('last provider failed') + request + .mockRejectedValueOnce(new Error('first')) + .mockRejectedValueOnce(new Error('second')) + .mockRejectedValueOnce(last) + await expect(executeModelRequestWithFallbacks(options)).rejects.toBe(last) + expect(request.mock.calls.map(([args]) => args.request.model)).toEqual([ + 'gpt-4o', + 'claude-sonnet-5', + 'gpt-4o-mini', + ]) + expect(options.ctx.blockLogs[0].modelFallbacks).toEqual(['gpt-4o', 'claude-sonnet-5']) + }) + + it.each([1, 2])('leaves primary attempt %i to the executor retry policy', async (attempt) => { + const error = new Error('overloaded') + request.mockRejectedValueOnce(error) + await expect( + executeModelRequestWithFallbacks({ + ...input(), + retry: { attempt, maxTries: 3, isFinalTry: false }, + }) + ).rejects.toBe(error) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('uses fallbacks on the final primary try', async () => { + request.mockRejectedValueOnce(new Error('overloaded')) + expect( + await executeModelRequestWithFallbacks({ + ...input(), + retry: { attempt: 3, maxTries: 3, isFinalTry: true }, + }) + ).toMatchObject({ usedFallback: true, result: { model: 'claude-sonnet-5' } }) + }) + + it('skips denied and incompatible provider families without sending them a request', async () => { + request.mockRejectedValueOnce(new Error('overloaded')) + validateModel.mockImplementation(async (_user, _workspace, model) => { + if (model === 'claude-sonnet-5') throw new Error('not permitted') + }) + const options = { + ...input(), + fallbackModels: [ + { model: 'claude-sonnet-5' }, + { model: 'vertex/gemini-3.1-pro' }, + { model: 'gpt-4o-mini' }, + ], + } + const output = await executeModelRequestWithFallbacks(options) + expect(output.result.model).toBe('gpt-4o-mini') + expect(request).toHaveBeenCalledTimes(2) + expect(options.ctx.blockLogs[0].modelFallbacks).toEqual(['gpt-4o']) + }) + + it('returns the original error when all remaining candidates are skipped', async () => { + const error = new Error('overloaded') + request.mockRejectedValueOnce(error) + validateModel.mockRejectedValue(new Error('not permitted')) + await expect(executeModelRequestWithFallbacks(input())).rejects.toBe(error) + expect(request).toHaveBeenCalledTimes(1) + }) + + it.each([ + Object.assign(new Error('stopped'), { name: 'AbortError' }), + Object.assign(new Error('permanent'), { retryable: false }), + ])('does not fall back after a non-retryable error', async (error) => { + request.mockRejectedValueOnce(error) + await expect(executeModelRequestWithFallbacks(input())).rejects.toBe(error) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('stops when cancelled between attempts', async () => { + const controller = new AbortController() + const error = new Error('provider failed during cancellation') + request.mockImplementationOnce(async () => { + controller.abort() + throw error + }) + const options = input() + options.ctx.abortSignal = controller.signal + await expect(executeModelRequestWithFallbacks(options)).rejects.toBe(error) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('only sends a cross-provider key that was stored as a reference, and drops family credentials', async () => { + const options = input() + const rows = [{ model: 'claude-sonnet-5', apiKey: '{{ANTHROPIC_KEY}}', thinkingLevel: 'high' }] + options.block = { ...block, config: { ...block.config, params: { fallbackModels: rows } } } + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ + ...options, + request: { + ...options.request, + vertexProject: 'private-project', + bedrockSecretKey: 'private-key', + responseFormat: { name: 'scores', schema: { type: 'object' }, strict: true }, + }, + fallbackModels: [{ ...rows[0], apiKey: 'resolved-key' }], + }) + const fallback = request.mock.calls[1][0] + expect(fallback).toMatchObject({ + providerId: 'anthropic', + request: { + apiKey: 'resolved-key', + thinkingLevel: 'high', + temperature: 0.3, + responseFormat: { name: 'scores' }, + }, + }) + expect(fallback.request.vertexProject).toBeUndefined() + expect(fallback.request.bedrockSecretKey).toBeUndefined() + }) + + it.each(['raw-key', '{{MISSING_KEY}}'])( + 'does not send an unsafe or unresolved key: %s', + async (key) => { + const options = input() + const rows = [{ model: 'claude-sonnet-5', apiKey: key }] + options.block = { ...block, config: { ...block.config, params: { fallbackModels: rows } } } + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ ...options, fallbackModels: rows }) + expect(request.mock.calls[1][0].request.apiKey).toBeUndefined() + } + ) + + it('uses the primary key on the same provider even if a row retained an old key', async () => { + const options = input() + options.block = { + ...block, + config: { + ...block.config, + params: { fallbackModels: [{ model: 'gpt-4o-mini', apiKey: '{{OLD_KEY}}' }] }, + }, + } + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ + ...options, + fallbackModels: [{ model: 'gpt-4o-mini', apiKey: 'old-key' }], + }) + expect(request.mock.calls[1][0].request.apiKey).toBe('primary-key') + }) + + it('lets hosted fallbacks resolve platform or BYOK credentials instead of a stale row key', async () => { + setEnvFlags({ isHosted: true }) + const options = input() + options.block = { + ...block, + config: { + ...block.config, + params: { fallbackModels: [{ model: 'claude-sonnet-5', apiKey: '{{OLD_KEY}}' }] }, + }, + } + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ + ...options, + fallbackModels: [{ model: 'claude-sonnet-5', apiKey: 'old-key' }], + }) + expect(request.mock.calls[1][0].request.apiKey).toBeUndefined() + }) + + it('strips the Auto identity prompt from fallbacks and keeps the pool model out of the trace', async () => { + const options = input() + request.mockRejectedValueOnce(new Error('overloaded')) + await executeModelRequestWithFallbacks({ + ...options, + configuredModel: 'sim-auto', + request: { ...options.request, systemPrompt: 'Auto identity\n\nScore the content' }, + }) + expect(request.mock.calls[1][0].request.systemPrompt).toBe('Score the content') + expect(options.ctx.blockLogs[0].modelFallbacks).toEqual(['sim-auto']) + }) +}) diff --git a/apps/sim/executor/utils/model-fallback-request.ts b/apps/sim/executor/utils/model-fallback-request.ts new file mode 100644 index 00000000000..17b26763a67 --- /dev/null +++ b/apps/sim/executor/utils/model-fallback-request.ts @@ -0,0 +1,150 @@ +import { createLogger } from '@sim/logger' +import { omit } from '@sim/utils/object' +import { resolveFallbackTuning } from '@/lib/workflows/blocks/fallback-models' +import { providerRequiresFamilyCredentials } from '@/blocks/utils' +import { validateModelProvider } from '@/ee/access-control/utils/permission-check' +import { isRetryableBlockError } from '@/executor/execution/block-retry' +import type { BlockRetryAttempt, ExecutionContext } from '@/executor/types' +import { + getModelFallbacks, + PROVIDER_FAMILY_CREDENTIAL_FIELDS, + recordModelFallbacks, + resolveFallbackApiKey, +} from '@/executor/utils/model-fallbacks' +import { executeBlockProviderRequest } from '@/executor/utils/provider-request' +import { projectResolvedSecretDiagnosticError } from '@/executor/utils/resolved-secret-content-projection' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models' +import type { ProviderRequest, ProviderResponse } from '@/providers/types' +import { getProviderFromModel } from '@/providers/utils' +import type { SerializedBlock } from '@/serializer/types' + +const logger = createLogger('BlockModelFallbacks') + +interface ModelFallbackRequestInput { + ctx: ExecutionContext + block: SerializedBlock + providerId: string + request: ProviderRequest + configuredModel: string + fallbackModels: unknown + /** The original system prompt, before an Auto identity preamble was added. */ + fallbackSystemPrompt: string + retry?: BlockRetryAttempt + resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined +} + +/** + * Shared non-streaming model execution for Router and Evaluator. Block retries + * exhaust the primary first; the final try walks the ordered fallback models. + * Parsing a successful response remains the handler's job, so a routing decision + * such as NO_MATCH is never silently replaced by another model's decision. + */ +export async function executeModelRequestWithFallbacks({ + ctx, + block, + providerId, + request, + configuredModel, + fallbackModels, + fallbackSystemPrompt, + retry, + resolvedSecretTraceRegistry, +}: ModelFallbackRequestInput): Promise<{ result: ProviderResponse; usedFallback: boolean }> { + const fallbacks = getModelFallbacks(ctx, block, fallbackModels, logger) + const candidates = [ + { model: request.model }, + ...(retry && !retry.isFinalTry + ? [] + : fallbacks.filter( + (candidate) => candidate.model.toLowerCase() !== request.model.toLowerCase() + )), + ] + const failedModels: string[] = [] + let lastError: unknown + + for (const [index, candidate] of candidates.entries()) { + const isPrimary = index === 0 + if (!isPrimary && ctx.abortSignal?.aborted) break + let candidateProviderId = providerId + if (!isPrimary) { + try { + candidateProviderId = getProviderFromModel(candidate.model) + await validateModelProvider(ctx.userId, ctx.workspaceId, candidate.model, ctx) + if ( + candidateProviderId !== providerId && + providerRequiresFamilyCredentials(candidateProviderId) + ) { + throw new Error('Fallback requires credentials from a different provider family') + } + } catch (error) { + logger.warn( + 'Fallback model unusable; skipping', + projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry, { + blockId: block.id, + model: candidate.model, + }) + ) + continue + } + } + + let candidateRequest = request + if (!isPrimary) { + const sameProvider = candidateProviderId === providerId + const { adjustments: _adjustments, ...tuning } = resolveFallbackTuning( + candidate, + configuredModel, + request + ) + candidateRequest = { + ...(sameProvider ? request : omit(request, [...PROVIDER_FAMILY_CREDENTIAL_FIELDS])), + ...tuning, + temperature: tuning.temperature === undefined ? undefined : Number(tuning.temperature), + maxTokens: tuning.maxTokens === undefined ? undefined : Number(tuning.maxTokens), + model: candidate.model, + apiKey: resolveFallbackApiKey({ + candidate, + configuredModel, + sameProvider, + primaryApiKey: request.apiKey, + blockId: block.id, + logger, + }), + systemPrompt: fallbackSystemPrompt, + } + } + + try { + const result = await executeBlockProviderRequest({ + ctx, + providerId: candidateProviderId, + request: candidateRequest, + resolvedSecretTraceRegistry, + }) + recordModelFallbacks(ctx, block, failedModels) + return { result, usedFallback: !isPrimary } + } catch (error) { + lastError = error + failedModels.push( + isPrimary && isAutoModel(configuredModel) ? SIM_AUTO_MODEL_ID : candidate.model + ) + if ( + index === candidates.length - 1 || + ctx.abortSignal?.aborted || + !isRetryableBlockError(error) + ) + break + logger.warn( + 'Model request failed; trying fallback', + projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry, { + blockId: block.id, + failedModel: candidate.model, + }) + ) + } + } + + recordModelFallbacks(ctx, block, failedModels.slice(0, -1)) + throw lastError +} diff --git a/apps/sim/executor/utils/model-fallbacks.ts b/apps/sim/executor/utils/model-fallbacks.ts new file mode 100644 index 00000000000..84e29e366b6 --- /dev/null +++ b/apps/sim/executor/utils/model-fallbacks.ts @@ -0,0 +1,109 @@ +import type { createLogger } from '@sim/logger' +import { isPlainRecord } from '@sim/utils/object' +import { + type FallbackModelCandidate, + fallbackRowNeedsApiKey, + isWholeEnvVarReference, + normalizeFallbackModels, +} from '@/lib/workflows/blocks/fallback-models' +import type { ExecutionContext } from '@/executor/types' +import { projectResolvedSecretDiagnosticContent } from '@/executor/utils/resolved-secret-content-projection' +import type { SerializedBlock } from '@/serializer/types' + +type Logger = ReturnType + +/** Credentials scoped to one provider family, never forwarded across providers. */ +export const PROVIDER_FAMILY_CREDENTIAL_FIELDS = [ + 'azureEndpoint', + 'azureApiVersion', + 'vertexProject', + 'vertexLocation', + 'bedrockAccessKeyId', + 'bedrockSecretKey', + 'bedrockRegion', +] as const + +/** Validates stored key references before admitting their resolved values to a provider. */ +export function getModelFallbacks( + ctx: ExecutionContext, + block: SerializedBlock, + rows: unknown, + logger: Logger +): FallbackModelCandidate[] { + if (!Array.isArray(rows)) return [] + const storedRows: unknown = block.config?.params?.fallbackModels + return normalizeFallbackModels( + rows.map((row, index) => { + if (!isPlainRecord(row) || row.apiKey === undefined) return row + const stored = Array.isArray(storedRows) ? storedRows[index] : undefined + if (isPlainRecord(stored) && isWholeEnvVarReference(stored.apiKey)) return row + const projection = projectResolvedSecretDiagnosticContent( + { blockId: block.id, model: row.model, row: index + 1 }, + ctx.resolvedSecretTraceRegistry + ) + logger.warn( + 'Fallback row key ignored; only an environment variable reference is accepted', + projection.safe && isPlainRecord(projection.value) + ? projection.value + : { blockId: block.id, row: index + 1 } + ) + return { ...row, apiKey: undefined } + }) + ) +} + +/** A hidden or unresolved row key cannot override the primary, BYOK, or platform key. */ +export function resolveFallbackApiKey({ + candidate, + configuredModel, + sameProvider, + primaryApiKey, + blockId, + logger, +}: { + candidate: FallbackModelCandidate + configuredModel: string + sameProvider: boolean + primaryApiKey: string | undefined + blockId: string + logger: Logger +}): string | undefined { + let rowKey = fallbackRowNeedsApiKey(candidate.model, configuredModel) + ? candidate.apiKey + : undefined + if (rowKey && isWholeEnvVarReference(rowKey)) { + logger.warn('Fallback key variable is not set for this run', { + blockId, + model: candidate.model, + variable: rowKey, + }) + rowKey = undefined + } + return rowKey ?? (sameProvider ? primaryApiKey : undefined) +} + +/** Records failed candidates on the current attempt's log, projecting secret-derived model names. */ +export function recordModelFallbacks( + ctx: ExecutionContext, + block: SerializedBlock, + failedModels: string[] +): void { + const logs = ctx.blockLogs ?? [] + for (let index = logs.length - 1; index >= 0; index--) { + const entry = logs[index] + if (entry.blockId !== block.id || entry.endedAt !== '') continue + if (failedModels.length === 0) { + entry.modelFallbacks = undefined + return + } + const registry = ctx.errorResolvedSecretTraceRegistry ?? ctx.resolvedSecretTraceRegistry + const projection = projectResolvedSecretDiagnosticContent({ models: failedModels }, registry) + const models = + projection.safe && isPlainRecord(projection.value) ? projection.value.models : undefined + entry.modelFallbacks = + Array.isArray(models) && models.every((model) => typeof model === 'string') + ? [...models] + : undefined + return + } +} diff --git a/apps/sim/lib/logs/execution/trace-spans/span-factory.ts b/apps/sim/lib/logs/execution/trace-spans/span-factory.ts index e1b4a837c8d..7abe0bcbc71 100644 --- a/apps/sim/lib/logs/execution/trace-spans/span-factory.ts +++ b/apps/sim/lib/logs/execution/trace-spans/span-factory.ts @@ -115,6 +115,7 @@ function createBaseSpan(log: ValidBlockLog): TraceSpan { ...(log.childTraceDisabled ? { childTraceDisabled: true } : {}), ...(log.errorHandled && { errorHandled: true }), ...(log.tries !== undefined && { tries: log.tries }), + ...(log.modelFallbacks?.length && { modelFallbacks: log.modelFallbacks }), ...(log.loopId && { loopId: log.loopId }), ...(log.parallelId && { parallelId: log.parallelId }), ...(log.iterationIndex !== undefined && { iterationIndex: log.iterationIndex }), diff --git a/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts b/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts index 09a0c25858c..70444e6cb47 100644 --- a/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts +++ b/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts @@ -1027,6 +1027,36 @@ describe('buildTraceSpans', () => { }) }) +describe('modelFallbacks', () => { + const log = (modelFallbacks?: string[]) => ({ + blockId: 'agent-1', + blockName: 'Agent', + blockType: 'agent', + startedAt: '2024-01-01T10:00:00.000Z', + endedAt: '2024-01-01T10:00:01.000Z', + durationMs: 1000, + success: true, + output: { content: 'ok', model: 'gpt-5.4-mini' }, + executionOrder: 1, + ...(modelFallbacks ? { modelFallbacks } : {}), + }) + + it.concurrent('carries the failed models onto the span and omits the field when empty', () => { + const withFallbacks = buildTraceSpans({ + success: true, + output: {}, + logs: [log(['claude-sonnet-5'])], + }) + expect(withFallbacks.traceSpans[0].modelFallbacks).toEqual(['claude-sonnet-5']) + expect(withFallbacks.traceSpans[0].model).toBe('gpt-5.4-mini') + + const empty = buildTraceSpans({ success: true, output: {}, logs: [log([])] }) + expect(empty.traceSpans[0]).not.toHaveProperty('modelFallbacks') + const none = buildTraceSpans({ success: true, output: {}, logs: [log()] }) + expect(none.traceSpans[0]).not.toHaveProperty('modelFallbacks') + }) +}) + describe('errorHandled - handled errors should not bubble up', () => { it.concurrent('block span stays error but is marked errorHandled', () => { const result: ExecutionResult = { diff --git a/apps/sim/lib/logs/types.ts b/apps/sim/lib/logs/types.ts index 80d9f7542a1..19c9f4a5c1b 100644 --- a/apps/sim/lib/logs/types.ts +++ b/apps/sim/lib/logs/types.ts @@ -285,6 +285,8 @@ export interface TraceSpan { errorHandled?: boolean /** Total handler tries, present only when the block retried at least once. */ tries?: number + /** Models that failed before the one that answered; present only when the block fell back. */ + modelFallbacks?: string[] tokens?: TokenInfo relativeStartMs?: number blockId?: string diff --git a/apps/sim/lib/workflows/blocks/fallback-models.test.ts b/apps/sim/lib/workflows/blocks/fallback-models.test.ts new file mode 100644 index 00000000000..123b112b130 --- /dev/null +++ b/apps/sim/lib/workflows/blocks/fallback-models.test.ts @@ -0,0 +1,438 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockShouldRequireApiKey, mockRequiresFamilyCredentials } = vi.hoisted(() => ({ + mockShouldRequireApiKey: vi.fn((model: string) => false), + mockRequiresFamilyCredentials: vi.fn((provider: string | null | undefined) => false), +})) + +vi.mock('@/blocks/utils', () => ({ + shouldRequireApiKeyForModel: mockShouldRequireApiKey, + providerRequiresFamilyCredentials: mockRequiresFamilyCredentials, +})) + +vi.mock('@/providers/models', () => ({ + isAutoModel: (model: string) => model.trim().toLowerCase() === 'sim-auto', + isKnownModelId: (model: string) => model.startsWith('gpt') || model.startsWith('claude'), + findProviderFromModel: (model: string) => { + const lower = model.toLowerCase() + if (lower.startsWith('gpt')) return 'openai' + if (lower.startsWith('claude')) return 'anthropic' + if (lower.startsWith('vertex/')) return 'vertex' + if (lower.startsWith('openrouter/')) return 'openrouter' + return null + }, + getReasoningEffortValuesForModel: (model: string) => + model === 'gpt-big' + ? ['low', 'medium', 'high', 'xhigh'] + : model === 'gpt-small' + ? ['low', 'high'] + : null, + getThinkingLevelsForModel: (model: string) => + model.startsWith('claude') ? ['low', 'medium', 'high'] : null, + getVerbosityValuesForModel: (model: string) => + model.startsWith('gpt') ? ['low', 'medium', 'high'] : null, + getMaxTemperature: (model: string) => + model.startsWith('claude') ? 1 : model.startsWith('gpt') ? 2 : undefined, + getModelCapabilities: (model: string) => + model === 'gpt-small' + ? { maxOutputTokens: 4096 } + : model.startsWith('gpt') + ? { maxOutputTokens: 16000 } + : model.startsWith('claude') + ? {} + : null, +})) + +import { + addFallbackRow, + changeFallbackRowApiKey, + changeFallbackRowModel, + changeFallbackRowTuning, + fallbackRowNeedsApiKey, + getFallbackTuningKnobsToShow, + getTuningOptionsForModel, + isTuningValueValidForModel, + isViableFallbackModel, + isWholeEnvVarReference, + MAX_FALLBACK_MODELS, + moveFallbackRow, + normalizeFallbackModels, + normalizeTuningValues, + ordinalChoiceLabel, + removeFallbackRow, + resolveFallbackTuning, +} from '@/lib/workflows/blocks/fallback-models' + +beforeEach(() => { + vi.clearAllMocks() + mockShouldRequireApiKey.mockReturnValue(false) + mockRequiresFamilyCredentials.mockReturnValue(false) +}) + +describe('isWholeEnvVarReference', () => { + it('accepts exactly one braced variable name', () => { + expect(isWholeEnvVarReference('{{OPENROUTER_API_KEY}}')).toBe(true) + expect(isWholeEnvVarReference('{{ key_1 }}')).toBe(true) + }) + + it('refuses raw keys, partial references, and non-strings', () => { + expect(isWholeEnvVarReference('sk-live-abc')).toBe(false) + expect(isWholeEnvVarReference('prefix {{KEY}}')).toBe(false) + expect(isWholeEnvVarReference('{{A}}{{B}}')).toBe(false) + expect(isWholeEnvVarReference('{{1BAD}}')).toBe(false) + expect(isWholeEnvVarReference(42)).toBe(false) + expect(isWholeEnvVarReference(null)).toBe(false) + }) +}) + +describe('normalizeFallbackModels', () => { + it('returns an empty chain for anything that is not an array', () => { + expect(normalizeFallbackModels(undefined)).toEqual([]) + expect(normalizeFallbackModels('gpt-5')).toEqual([]) + expect(normalizeFallbackModels({ model: 'gpt-5' })).toEqual([]) + }) + + it('keeps order, trims, and drops rows without a model', () => { + expect( + normalizeFallbackModels([ + { id: 'a', model: ' gpt-5 ' }, + { id: 'b', model: '' }, + { id: 'c' }, + null, + { id: 'd', model: 'claude-sonnet-5' }, + ]) + ).toEqual([{ model: 'gpt-5' }, { model: 'claude-sonnet-5' }]) + }) + + it('drops sim-auto and case-insensitive duplicates, keeping the first position', () => { + expect( + normalizeFallbackModels([ + { model: 'sim-auto' }, + { model: 'gpt-5' }, + { model: 'GPT-5' }, + { model: 'claude-sonnet-5' }, + ]) + ).toEqual([{ model: 'gpt-5' }, { model: 'claude-sonnet-5' }]) + }) + + it('keeps a row tuning value, trimmed and lower-cased', () => { + expect( + normalizeFallbackModels([{ model: 'gpt-small', reasoningEffort: ' Low ', thinkingLevel: '' }]) + ).toEqual([{ model: 'gpt-small', reasoningEffort: 'low' }]) + }) + + it('keeps any non-empty key, reference or already resolved, and drops blanks', () => { + expect( + normalizeFallbackModels([ + { model: 'openrouter/a', apiKey: '{{OPENROUTER_API_KEY}}' }, + { model: 'openrouter/b', apiKey: ' sk-resolved-at-runtime ' }, + { model: 'openrouter/c', apiKey: '' }, + { model: 'openrouter/d', apiKey: 42 }, + ]) + ).toEqual([ + { model: 'openrouter/a', apiKey: '{{OPENROUTER_API_KEY}}' }, + { model: 'openrouter/b', apiKey: 'sk-resolved-at-runtime' }, + { model: 'openrouter/c' }, + { model: 'openrouter/d' }, + ]) + }) + + it('caps the chain', () => { + const rows = Array.from({ length: MAX_FALLBACK_MODELS + 3 }, (_, i) => ({ model: `m-${i}` })) + expect(normalizeFallbackModels(rows)).toHaveLength(MAX_FALLBACK_MODELS) + }) +}) + +describe('isViableFallbackModel', () => { + it('rejects empty, sim-auto, the primary itself, and unresolvable ids', () => { + expect(isViableFallbackModel('', 'gpt-5')).toBe(false) + expect(isViableFallbackModel('sim-auto', 'gpt-5')).toBe(false) + expect(isViableFallbackModel('GPT-5', 'gpt-5')).toBe(false) + expect(isViableFallbackModel('mystery-model', 'gpt-5')).toBe(false) + }) + + it('offers any resolvable model that needs no provider-family credentials', () => { + expect(isViableFallbackModel('claude-sonnet-5', 'gpt-5')).toBe(true) + mockShouldRequireApiKey.mockReturnValue(true) + expect(isViableFallbackModel('openrouter/x', 'gpt-5')).toBe(true) + }) + + it('offers a family-bound model only alongside a primary of the same family', () => { + mockRequiresFamilyCredentials.mockImplementation((provider) => provider === 'vertex') + expect(isViableFallbackModel('vertex/gemini-b', 'vertex/gemini-a')).toBe(true) + expect(isViableFallbackModel('vertex/gemini-b', 'gpt-5')).toBe(false) + }) +}) + +describe('fallbackRowNeedsApiKey', () => { + it('is false when the model needs no key at all', () => { + expect(fallbackRowNeedsApiKey('claude-sonnet-5', 'gpt-5')).toBe(false) + }) + + it('is false when the block key on the same provider can be reused', () => { + mockShouldRequireApiKey.mockReturnValue(true) + expect(fallbackRowNeedsApiKey('gpt-5-mini', 'gpt-5')).toBe(false) + }) + + it('is true for a keyed model on another provider', () => { + mockShouldRequireApiKey.mockReturnValue(true) + expect(fallbackRowNeedsApiKey('openrouter/x', 'gpt-5')).toBe(true) + }) +}) + +describe('tuning options and validity', () => { + it('offers the provider-decides entry first, then what the model declares', () => { + expect(getTuningOptionsForModel('gpt-small', 'reasoningEffort')).toEqual([ + 'auto', + 'low', + 'high', + ]) + expect(getTuningOptionsForModel('claude-sonnet-5', 'thinkingLevel')).toEqual([ + 'none', + 'low', + 'medium', + 'high', + ]) + expect(getTuningOptionsForModel('claude-sonnet-5', 'reasoningEffort')).toBeNull() + expect(getTuningOptionsForModel('', 'verbosity')).toBeNull() + }) + + it('treats unset, the sentinel, and declared values as valid, and passes uncatalogued through', () => { + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', undefined)).toBe(true) + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', 'auto')).toBe(true) + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', 'High')).toBe(true) + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', 'xhigh')).toBe(false) + expect(isTuningValueValidForModel('gpt-small', 'reasoningEffort', 42)).toBe(false) + expect(isTuningValueValidForModel('openrouter/x', 'reasoningEffort', 'anything')).toBe(true) + /** Catalogued but without the knob: nothing but unset or the sentinel is acceptable. */ + expect(isTuningValueValidForModel('claude-sonnet-5', 'reasoningEffort', 'high')).toBe(false) + expect(isTuningValueValidForModel('claude-sonnet-5', 'reasoningEffort', 'auto')).toBe(true) + }) +}) + +describe('getFallbackTuningKnobsToShow', () => { + it('shows a knob the primary lacks and one whose primary value the fallback does not declare', () => { + expect(getFallbackTuningKnobsToShow('claude-sonnet-5', 'gpt-big', {})).toEqual([ + 'thinkingLevel', + ]) + expect( + getFallbackTuningKnobsToShow('gpt-small', 'gpt-big', { reasoningEffort: 'xhigh' }) + ).toEqual(['reasoningEffort']) + }) + + it('stays bare when the primary value carries over or nothing is set', () => { + expect( + getFallbackTuningKnobsToShow('gpt-small', 'gpt-big', { reasoningEffort: 'high' }) + ).toEqual([]) + expect(getFallbackTuningKnobsToShow('gpt-small', 'gpt-big', {})).toEqual([]) + expect( + getFallbackTuningKnobsToShow('gpt-small', 'gpt-big', { reasoningEffort: 'auto' }) + ).toEqual([]) + }) +}) + +describe('resolveFallbackTuning', () => { + it('prefers the row value, inherits a declared primary value, and drops the rest', () => { + const resolved = resolveFallbackTuning( + { model: 'gpt-small', reasoningEffort: 'low' }, + 'gpt-big', + { + reasoningEffort: 'xhigh', + verbosity: 'high', + } + ) + expect(resolved.reasoningEffort).toBe('low') + expect(resolved.verbosity).toBe('high') + expect(resolved.thinkingLevel).toBeUndefined() + expect(resolved.adjustments).toEqual(['reasoningEffort: xhigh -> low']) + }) + + it('drops a primary value the fallback does not declare and says so', () => { + const resolved = resolveFallbackTuning({ model: 'gpt-small' }, 'gpt-big', { + reasoningEffort: 'xhigh', + }) + expect(resolved.reasoningEffort).toBeUndefined() + expect(resolved.adjustments).toEqual(['reasoningEffort: xhigh -> provider default']) + }) + + it('never inherits a knob the primary does not have', () => { + const resolved = resolveFallbackTuning({ model: 'claude-sonnet-5' }, 'gpt-big', { + thinkingLevel: 'high', + }) + expect(resolved.thinkingLevel).toBeUndefined() + }) + + it('treats a value stored under an uncatalogued primary as stale, and lets the row decide', () => { + /** The block never shows a graded knob for a model outside the catalog. */ + const stale = resolveFallbackTuning({ model: 'gpt-small' }, 'openrouter/custom', { + reasoningEffort: 'high', + }) + expect(stale.reasoningEffort).toBeUndefined() + expect(stale.adjustments).toEqual(['reasoningEffort: high -> provider default']) + + const own = resolveFallbackTuning( + { model: 'gpt-small', reasoningEffort: 'low' }, + 'openrouter/custom', + { reasoningEffort: 'high' } + ) + expect(own.reasoningEffort).toBe('low') + }) + + it('clamps temperature and max tokens to the fallback caps, keeping the input type', () => { + const resolved = resolveFallbackTuning({ model: 'claude-sonnet-5' }, 'gpt-big', { + temperature: '1.5', + maxTokens: 20000, + }) + expect(resolved.temperature).toBe('1') + expect(resolved.maxTokens).toBe(20000) + expect(resolved.adjustments).toEqual(['temperature: 1.5 -> 1']) + + const small = resolveFallbackTuning({ model: 'gpt-small' }, 'gpt-big', { + temperature: 0.2, + maxTokens: '20000', + }) + expect(small.temperature).toBe(0.2) + expect(small.maxTokens).toBe('4096') + + /** A value that never resolved to a number passes through untouched. */ + const unresolved = resolveFallbackTuning({ model: 'claude-sonnet-5' }, 'gpt-big', { + temperature: '{{TEMP}}', + }) + expect(unresolved.temperature).toBe('{{TEMP}}') + expect(unresolved.adjustments).toEqual([]) + }) + + it('passes everything through for an uncatalogued fallback', () => { + const resolved = resolveFallbackTuning({ model: 'openrouter/x' }, 'gpt-big', { + reasoningEffort: 'xhigh', + temperature: '1.9', + maxTokens: '99999', + }) + expect(resolved).toEqual({ + reasoningEffort: 'xhigh', + thinkingLevel: undefined, + verbosity: undefined, + temperature: '1.9', + maxTokens: '99999', + adjustments: [], + }) + }) +}) + +describe('resolveFallbackTuning hidden overrides', () => { + it('ignores a stored row value once the primary value fits and the field is no longer shown', () => { + const resolved = resolveFallbackTuning( + { model: 'gpt-small', reasoningEffort: 'low' }, + 'gpt-big', + { + reasoningEffort: 'high', + } + ) + expect(resolved.reasoningEffort).toBe('high') + expect(resolved.adjustments).toEqual([]) + }) +}) + +describe('normalizeTuningValues', () => { + it('keeps trimmed lower-cased strings and drops blanks and non-strings', () => { + expect( + normalizeTuningValues({ + reasoningEffort: ' Low ', + thinkingLevel: '', + verbosity: 3, + model: 'x', + }) + ).toEqual({ reasoningEffort: 'low' }) + }) + + it('stores the provider-decides entry as absence, as the editor does', () => { + expect( + normalizeTuningValues({ reasoningEffort: 'auto', thinkingLevel: 'NONE', verbosity: 'low' }) + ).toEqual({ verbosity: 'low' }) + }) +}) + +describe('row transforms', () => { + const rows = [ + { id: 'a', model: 'gpt-big' }, + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}', reasoningEffort: 'low' }, + ] + + it('adds a blank row until the cap and never past it', () => { + expect(addFallbackRow(rows, 'c')).toEqual([...rows, { id: 'c', model: '' }]) + const full = Array.from({ length: MAX_FALLBACK_MODELS }, (_, i) => ({ + id: `r${i}`, + model: 'm', + })) + expect(addFallbackRow(full, 'extra')).toBe(full) + }) + + it('removes by id and moves within bounds', () => { + expect(removeFallbackRow(rows, 'a')).toEqual([rows[1]]) + expect(moveFallbackRow(rows, 'b', -1)).toEqual([rows[1], rows[0]]) + expect(moveFallbackRow(rows, 'a', -1)).toBe(rows) + expect(moveFallbackRow(rows, 'b', 1)).toBe(rows) + expect(moveFallbackRow(rows, 'missing', 1)).toBe(rows) + }) + + it('clears tuning on a model change and keeps the key only for the same keyed provider', () => { + mockShouldRequireApiKey.mockReturnValue(true) + expect(changeFallbackRowModel(rows, 'b', 'openrouter/y', 'claude-sonnet-5')[1]).toEqual({ + id: 'b', + model: 'openrouter/y', + apiKey: '{{OPENROUTER_API_KEY}}', + }) + /** Another provider must never receive the previous provider's credential. */ + expect(changeFallbackRowModel(rows, 'b', 'gpt-small', 'claude-sonnet-5')[1]).toEqual({ + id: 'b', + model: 'gpt-small', + }) + mockShouldRequireApiKey.mockReturnValue(false) + expect(changeFallbackRowModel(rows, 'b', 'openrouter/y', 'claude-sonnet-5')[1]).toEqual({ + id: 'b', + model: 'openrouter/y', + }) + /** A key that is not a reference never survives an edit, even on the same provider. */ + mockShouldRequireApiKey.mockReturnValue(true) + const raw = [{ id: 'r', model: 'openrouter/x', apiKey: 'sk-raw' }] + expect(changeFallbackRowModel(raw, 'r', 'openrouter/y', 'claude-sonnet-5')[0]).toEqual({ + id: 'r', + model: 'openrouter/y', + }) + }) + + it('stores a tuning value, and the provider-decides entry as absence', () => { + expect(changeFallbackRowTuning(rows, 'a', 'reasoningEffort', 'high')[0]).toEqual({ + id: 'a', + model: 'gpt-big', + reasoningEffort: 'high', + }) + expect(changeFallbackRowTuning(rows, 'b', 'reasoningEffort', 'auto')[1]).toEqual({ + id: 'b', + model: 'openrouter/x', + apiKey: '{{OPENROUTER_API_KEY}}', + }) + expect(changeFallbackRowApiKey(rows, 'a', '{{K}}')[0]).toEqual({ + id: 'a', + model: 'gpt-big', + apiKey: '{{K}}', + }) + }) +}) + +describe('ordinalChoiceLabel', () => { + it('starts at the 2nd choice and handles English ordinals', () => { + expect([0, 1, 2, 3, 9, 10, 11].map(ordinalChoiceLabel)).toEqual([ + '2nd choice', + '3rd choice', + '4th choice', + '5th choice', + '11th choice', + '12th choice', + '13th choice', + ]) + }) +}) diff --git a/apps/sim/lib/workflows/blocks/fallback-models.ts b/apps/sim/lib/workflows/blocks/fallback-models.ts new file mode 100644 index 00000000000..712cde89036 --- /dev/null +++ b/apps/sim/lib/workflows/blocks/fallback-models.ts @@ -0,0 +1,372 @@ +import { providerRequiresFamilyCredentials, shouldRequireApiKeyForModel } from '@/blocks/utils' +import { + findProviderFromModel, + getMaxTemperature, + getModelCapabilities, + getReasoningEffortValuesForModel, + getThinkingLevelsForModel, + getVerbosityValuesForModel, + isAutoModel, + isKnownModelId, +} from '@/providers/models' + +/** Upper bound on fallback rows; enough for a full provider spread without an unbounded chain. */ +export const MAX_FALLBACK_MODELS = 5 + +/** The graded tuning knobs a fallback row may set for itself. */ +export const FALLBACK_TUNING_KNOBS = ['reasoningEffort', 'thinkingLevel', 'verbosity'] as const +export type FallbackTuningKnob = (typeof FALLBACK_TUNING_KNOBS)[number] + +/** The "let the provider decide" entry each knob's field offers first, as the block's own fields do. */ +const KNOB_SENTINEL: Record = { + reasoningEffort: 'auto', + thinkingLevel: 'none', + verbosity: 'auto', +} + +export const FALLBACK_TUNING_LABELS: Record = { + reasoningEffort: 'Reasoning effort', + thinkingLevel: 'Thinking level', + verbosity: 'Verbosity', +} + +/** Per-row tuning: present only when the builder chose a value for that knob. */ +export type FallbackTuningValues = Partial> + +/** + * One stored fallback row. `id` is a React key only. `apiKey`, when present, is + * always a whole `{{ENV_VAR}}` reference: a raw secret nested inside a list + * value would bypass every redaction path that keys on a top-level + * `password: true` field, so the picker cannot produce one and the copilot + * validator refuses one. The tuning knobs hold a value only when the primary's + * setting could not be carried over (see `getFallbackTuningKnobsToShow`). + */ +export interface FallbackModelEntry extends FallbackTuningValues { + id: string + model: string + apiKey?: string +} + +/** A row the executor acts on: the React key is gone, and `apiKey` is a validated reference. */ +export interface FallbackModelCandidate extends FallbackTuningValues { + model: string + apiKey?: string +} + +const WHOLE_ENV_VAR_REFERENCE = /^\{\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}\}$/ + +/** Whether a value is exactly one `{{ENV_VAR}}` reference and nothing else. */ +export function isWholeEnvVarReference(value: unknown): value is string { + return typeof value === 'string' && WHOLE_ENV_VAR_REFERENCE.test(value.trim()) +} + +/** + * Normalizes a stored fallback list into the ordered candidates execution walks. + * + * Tolerant rather than strict because it runs on every execution: rows the + * editor could not have written (missing model, sim-auto) are dropped instead + * of failing the block, and duplicates keep their first position so the order + * the builder chose is preserved. + * + * A row's `apiKey` is kept as any non-empty string. By the time this runs the + * input resolver has already turned the stored `{{ENV_VAR}}` reference into the + * key itself, exactly as it does for the block's own API Key field; the + * reference-only rule is enforced where rows are written (the picker, the + * copilot validator) and where they leave the workspace (the export sanitizer). + */ +export function normalizeFallbackModels(raw: unknown): FallbackModelCandidate[] { + if (!Array.isArray(raw)) return [] + + const seen = new Set() + const candidates: FallbackModelCandidate[] = [] + for (const row of raw) { + if (!row || typeof row !== 'object') continue + const { model, apiKey } = row as { model?: unknown; apiKey?: unknown } + if (typeof model !== 'string') continue + const trimmed = model.trim() + if (!trimmed || isAutoModel(trimmed)) continue + const key = trimmed.toLowerCase() + if (seen.has(key)) continue + seen.add(key) + const resolvedKey = typeof apiKey === 'string' ? apiKey.trim() : '' + candidates.push({ + model: trimmed, + ...(resolvedKey ? { apiKey: resolvedKey } : {}), + ...normalizeTuningValues(row as Record), + }) + if (candidates.length >= MAX_FALLBACK_MODELS) break + } + return candidates +} + +/** The tuning knobs a row carries, trimmed and lower-cased; blanks and non-strings are dropped. */ +export function normalizeTuningValues(row: Record): FallbackTuningValues { + const values: FallbackTuningValues = {} + for (const knob of FALLBACK_TUNING_KNOBS) { + const value = row[knob] + const level = typeof value === 'string' ? value.trim().toLowerCase() : '' + /** The provider-decides entry is stored as absence, the same way the editor stores it. */ + if (level && level !== KNOB_SENTINEL[knob]) values[knob] = level + } + return values +} + +/** + * The edits the editor makes to a fallback list, as pure transforms so the + * component stays a thin binding and the rules are unit-testable. + */ +export function addFallbackRow(rows: FallbackModelEntry[], id: string): FallbackModelEntry[] { + if (rows.length >= MAX_FALLBACK_MODELS) return rows + return [...rows, { id, model: '' }] +} + +export function removeFallbackRow(rows: FallbackModelEntry[], id: string): FallbackModelEntry[] { + return rows.filter((row) => row.id !== id) +} + +export function moveFallbackRow( + rows: FallbackModelEntry[], + id: string, + direction: -1 | 1 +): FallbackModelEntry[] { + const index = rows.findIndex((row) => row.id === id) + const target = index + direction + if (index === -1 || target < 0 || target >= rows.length) return rows + const next = [...rows] + ;[next[index], next[target]] = [next[target], next[index]] + return next +} + +/** + * A new model gets a clean row. Tuning always goes, since the new model may not + * declare it. The key survives only when the new model still needs one and sits + * on the same provider as the old one: a key reference is a credential for one + * provider, and carrying it to another would send that provider's secret to an + * unrelated service. + */ +export function changeFallbackRowModel( + rows: FallbackModelEntry[], + id: string, + model: string, + primaryModel: string +): FallbackModelEntry[] { + return rows.map((row) => { + if (row.id !== id) return row + const keepKey = + isWholeEnvVarReference(row.apiKey) && + fallbackRowNeedsApiKey(model, primaryModel) && + findProviderFromModel(model.trim()) === findProviderFromModel(row.model.trim()) + return { id: row.id, model, ...(keepKey ? { apiKey: row.apiKey } : {}) } + }) +} + +export function changeFallbackRowApiKey( + rows: FallbackModelEntry[], + id: string, + apiKey: string +): FallbackModelEntry[] { + return rows.map((row) => (row.id === id ? { ...row, apiKey } : row)) +} + +/** The provider-decides entry is the field's default, so it is stored as absence. */ +export function changeFallbackRowTuning( + rows: FallbackModelEntry[], + id: string, + knob: FallbackTuningKnob, + value: string +): FallbackModelEntry[] { + return rows.map((row) => { + if (row.id !== id) return row + const { [knob]: _previous, ...rest } = row + return value && value !== KNOB_SENTINEL[knob] ? { ...rest, [knob]: value } : rest + }) +} + +/** + * Whether a model can serve as a fallback for `primaryModel` with the + * credentials the block can actually give it. + * + * A fallback resolves its key the way the primary does, through workspace BYOK, + * the platform key, or the block's own field, with one addition: a row may name + * a workspace variable holding its key. What it can never do is inherit a Vertex + * credential, Bedrock keys, or an Azure endpoint from a primary in another + * family, because those fields only render for the primary's own provider. + */ +export function isViableFallbackModel(model: string, primaryModel: string): boolean { + const trimmed = model.trim() + if (!trimmed || isAutoModel(trimmed)) return false + if (trimmed.toLowerCase() === primaryModel.trim().toLowerCase()) return false + + const provider = findProviderFromModel(trimmed) + if (!provider) return false + + if (providerRequiresFamilyCredentials(provider)) { + return provider === findProviderFromModel(primaryModel.trim()) + } + return true +} + +/** + * Whether a fallback row must name a workspace variable for its key. + * + * A model that needs a key and shares the primary's provider reuses the + * block's own API Key field, so only a cross-provider fallback asks for one. + */ +export function fallbackRowNeedsApiKey(model: string, primaryModel: string): boolean { + const trimmed = model.trim() + if (!trimmed || !shouldRequireApiKeyForModel(trimmed)) return false + const provider = findProviderFromModel(trimmed) + return provider === null || provider !== findProviderFromModel(primaryModel.trim()) +} + +/** + * The values `model` accepts for `knob`, with the provider-decides entry first, + * exactly as the block's own field offers them. Null when the model lacks the + * knob or is not in the catalog, which is also how the block's field decides + * whether to render. + */ +export function getTuningOptionsForModel(model: string, knob: FallbackTuningKnob): string[] | null { + const trimmed = model.trim() + if (!trimmed) return null + const declared = + knob === 'reasoningEffort' + ? getReasoningEffortValuesForModel(trimmed) + : knob === 'thinkingLevel' + ? getThinkingLevelsForModel(trimmed) + : getVerbosityValuesForModel(trimmed) + if (!declared) return null + const sentinel = KNOB_SENTINEL[knob] + return [sentinel, ...declared.filter((value) => value !== sentinel)] +} + +/** + * Whether `value` can be sent to `model` for `knob`. Unset and the sentinel + * always can. Otherwise the model must declare it; a model the catalog does not + * know declares nothing, so anything passes through, as it does for the primary, + * while a catalogued model without the knob accepts nothing for it. + */ +export function isTuningValueValidForModel( + model: string, + knob: FallbackTuningKnob, + value: unknown +): boolean { + if (typeof value !== 'string') return value === undefined || value === null + const normalized = value.trim().toLowerCase() + if (!normalized || normalized === KNOB_SENTINEL[knob]) return true + const options = getTuningOptionsForModel(model, knob) + if (options === null) return !isKnownModelId(model.trim()) + return options.includes(normalized) +} + +/** + * The knobs a fallback row has to ask about: those the fallback model has that + * the primary's current setting cannot fill, either because the primary lacks + * the knob or because its value is not one the fallback declares. A value the + * fallback accepts is inherited silently, so a same-family row stays bare. + */ +export function getFallbackTuningKnobsToShow( + fallbackModel: string, + primaryModel: string, + primaryValues: Partial> +): FallbackTuningKnob[] { + return FALLBACK_TUNING_KNOBS.filter((knob) => { + if (getTuningOptionsForModel(fallbackModel, knob) === null) return false + if (getTuningOptionsForModel(primaryModel, knob) === null) return true + return !isTuningValueValidForModel(fallbackModel, knob, primaryValues[knob]) + }) +} + +export interface PrimaryTuningInputs extends Partial> { + temperature?: string | number + maxTokens?: string | number +} + +export interface ResolvedFallbackTuning extends Partial> { + temperature?: string | number + maxTokens?: string | number + /** Human-readable notes on every value that differs from the primary's, for the run log. */ + adjustments: string[] +} + +function clampToCap( + value: string | number | undefined, + cap: number | undefined +): string | number | undefined { + if (value === undefined || value === null || value === '' || cap === undefined) return value + const parsed = Number(value) + if (!Number.isFinite(parsed) || parsed <= cap) return value + return typeof value === 'number' ? cap : String(cap) +} + +/** + * The tuning a fallback candidate runs with. + * + * Graded knobs: the row's own value wins, but only for a knob the row is + * currently asked about (`getFallbackTuningKnobsToShow`), so a value stored + * while the primary was incompatible stops applying once the primary's own + * value fits and the field is no longer shown. Otherwise the primary's value is + * carried over only when the fallback declares it, and dropped to the + * provider's default otherwise, which is what the row field exists to override. + * Temperature and max output tokens are caps in the primary's terms, so they + * are clamped to what the fallback allows rather than dropped: a low + * temperature chosen for repeatability must survive, and "no more than N" still + * holds under a smaller ceiling. A fallback the catalog does not know has no + * caps and no lists, so everything passes through to it unchanged. A primary + * the catalog does not know never showed a graded knob in the editor, so a + * value stored under it is stale and is not inherited; the row shows the field + * instead, and its own value is what applies. + */ +export function resolveFallbackTuning( + candidate: FallbackModelCandidate, + primaryModel: string, + primary: PrimaryTuningInputs +): ResolvedFallbackTuning { + const adjustments: string[] = [] + const resolved: ResolvedFallbackTuning = { adjustments } + const overridable = new Set(getFallbackTuningKnobsToShow(candidate.model, primaryModel, primary)) + + for (const knob of FALLBACK_TUNING_KNOBS) { + const own = overridable.has(knob) ? candidate[knob] : undefined + if (own) { + resolved[knob] = own + if (own !== primary[knob]) adjustments.push(`${knob}: ${primary[knob] ?? 'unset'} -> ${own}`) + continue + } + const inherit = + getTuningOptionsForModel(primaryModel, knob) !== null && + isTuningValueValidForModel(candidate.model, knob, primary[knob]) + resolved[knob] = inherit ? primary[knob] : undefined + if (!inherit && primary[knob]) adjustments.push(`${knob}: ${primary[knob]} -> provider default`) + } + + resolved.temperature = clampToCap(primary.temperature, getMaxTemperature(candidate.model)) + if (resolved.temperature !== primary.temperature) { + adjustments.push(`temperature: ${primary.temperature} -> ${resolved.temperature}`) + } + resolved.maxTokens = clampToCap( + primary.maxTokens, + getModelCapabilities(candidate.model)?.maxOutputTokens + ) + if (resolved.maxTokens !== primary.maxTokens) { + adjustments.push(`maxTokens: ${primary.maxTokens} -> ${resolved.maxTokens}`) + } + + return resolved +} + +/** "2nd choice", "3rd choice", ... for the row at `index` (0-based) below the primary. */ +export function ordinalChoiceLabel(index: number): string { + const n = index + 2 + const mod100 = n % 100 + const suffix = + mod100 >= 11 && mod100 <= 13 + ? 'th' + : n % 10 === 1 + ? 'st' + : n % 10 === 2 + ? 'nd' + : n % 10 === 3 + ? 'rd' + : 'th' + return `${n}${suffix} choice` +} diff --git a/apps/sim/lib/workflows/credentials/credential-extractor.test.ts b/apps/sim/lib/workflows/credentials/credential-extractor.test.ts index ccdd1be166d..c4f5a7c15e8 100644 --- a/apps/sim/lib/workflows/credentials/credential-extractor.test.ts +++ b/apps/sim/lib/workflows/credentials/credential-extractor.test.ts @@ -102,6 +102,43 @@ describe('export sanitizer resource coverage', () => { expect(sanitizedValue('oauth-input', 'cred-123')).toBeNull() }) + it('keeps fallback models and only whole env-var-referenced row keys', () => { + expect( + sanitizedValue('model-fallback-list', [ + { id: 'a', model: 'gpt-5' }, + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { id: 'c', model: 'openrouter/y', apiKey: 'sk-raw-secret' }, + { id: 'd', model: 'openrouter/z', apiKey: '{{A}} sk-raw {{B}}' }, + 'not-a-row', + ]) + ).toEqual([ + { id: 'a', model: 'gpt-5' }, + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { id: 'c', model: 'openrouter/y' }, + { id: 'd', model: 'openrouter/z' }, + 'not-a-row', + ]) + expect(sanitizedValue('model-fallback-list', 'opaque')).toBe('opaque') + }) + + it('drops even referenced fallback row keys when env vars are not preserved', () => { + vi.mocked(getBlock).mockReturnValue({ + name: 'Test', + description: '', + subBlocks: [{ id: 'field', title: 'Field', type: 'model-fallback-list' }], + outputs: {}, + } as never) + const sanitized = sanitizeWorkflowForSharing( + stateWithSubBlock('model-fallback-list', [ + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + ]), + { preserveEnvVars: false, redactOpaqueCredentialInputs: true } + ) + expect(sanitized.blocks?.b1?.subBlocks?.field?.value).toEqual([ + { id: 'b', model: 'openrouter/x' }, + ]) + }) + it('leaves an ordinary field untouched', () => { expect(sanitizedValue('short-input', 'plain text')).toBe('plain text') }) diff --git a/apps/sim/lib/workflows/credentials/credential-extractor.ts b/apps/sim/lib/workflows/credentials/credential-extractor.ts index d9e3aa86fa7..436125ea952 100644 --- a/apps/sim/lib/workflows/credentials/credential-extractor.ts +++ b/apps/sim/lib/workflows/credentials/credential-extractor.ts @@ -1,4 +1,5 @@ import { isPlainRecord } from '@sim/utils/object' +import { isWholeEnvVarReference } from '@/lib/workflows/blocks/fallback-models' import { coerceObjectArray } from '@/lib/workflows/persistence/remap-internal-ids' import { getToolInputParamConfigs } from '@/lib/workflows/search-replace/indexer' import { WORKFLOW_SEARCH_SUBBLOCK_RESOURCE_TYPES } from '@/lib/workflows/search-replace/resources/registry' @@ -170,6 +171,24 @@ function isEnvironmentVariableReference(value: unknown): value is string { return typeof value === 'string' && value.startsWith('{{') && value.endsWith('}}') } +/** + * Keeps a fallback list's models and drops every row key that is not a whole + * environment-variable reference. The editor only ever writes references, but the + * realtime subblock-value op runs no validator, so this is what guarantees a raw + * key can never leave the workspace in an export or template. + */ +function sanitizeFallbackModelsValue( + value: unknown, + options: WorkflowSanitizationOptions +): unknown { + if (!Array.isArray(value)) return value + return value.map((row) => { + if (!row || typeof row !== 'object' || Array.isArray(row)) return row + const { apiKey, ...rest } = row as Record + return options.preserveEnvVars && isWholeEnvVarReference(apiKey) ? { ...rest, apiKey } : rest + }) +} + /** * Sanitizes nested tool parameters using the same codecs as workflow search and fork remapping. * Only parameters resolved from a registered definition retain non-sensitive values. Custom, MCP, @@ -245,6 +264,9 @@ function sanitizeConfiguredSubBlockValue( if (config.password === true) { return options.preserveEnvVars && isEnvironmentVariableReference(value) ? value : null } + if (config.type === 'model-fallback-list') { + return sanitizeFallbackModelsValue(value, options) + } if ( WORKSPACE_SPECIFIC_TYPES.has(config.type) || WORKSPACE_SPECIFIC_FIELDS.has(config.id) || diff --git a/apps/sim/lib/workflows/editing/validation.test.ts b/apps/sim/lib/workflows/editing/validation.test.ts index 1f192cb1721..8ee23f3b87c 100644 --- a/apps/sim/lib/workflows/editing/validation.test.ts +++ b/apps/sim/lib/workflows/editing/validation.test.ts @@ -3,6 +3,8 @@ */ import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getTuningOptionsForModel } from '@/lib/workflows/blocks/fallback-models' +import { getThinkingLevelsForModel } from '@/providers/models' import { normalizeConditionRouterIds } from './builders' const { @@ -338,6 +340,77 @@ describe('validateInputsForBlock', () => { ).toBe(false) }) + describe('model-fallback-list', () => { + const config = { id: 'fallbackModels', type: 'model-fallback-list' as const } + const validate = (value: unknown) => + validateValueForSubBlockType(config, value, 'fallbackModels', 'agent', 'agent-1') + + it('accepts known models with env-var-referenced keys and fills missing row ids', () => { + const result = validate([ + { id: 'row-1', model: ' claude-sonnet-5 ' }, + { model: 'openrouter/anthropic/claude', apiKey: '{{OPENROUTER_API_KEY}}' }, + ]) + expect(result.valid).toBe(true) + const rows = (result as { value: Array<{ id: string; model: string; apiKey?: string }> }) + .value + expect(rows[0]).toEqual({ id: 'row-1', model: 'claude-sonnet-5' }) + expect(rows[1].id).toEqual(expect.any(String)) + expect(rows[1]).toMatchObject({ + model: 'openrouter/anthropic/claude', + apiKey: '{{OPENROUTER_API_KEY}}', + }) + }) + + it('refuses a raw key rather than repairing it', () => { + const result = validate([{ model: 'claude-sonnet-5', apiKey: 'sk-live-raw' }]) + expect(result.valid).toBe(false) + expect((result as { error: { error: string } }).error.error).toContain( + 'apiKey must be a whole {{ENV_VAR}} reference' + ) + }) + + it('refuses sim-auto, unknown models, missing models, and non-arrays', () => { + expect(validate([{ model: 'sim-auto' }]).valid).toBe(false) + expect(validate([{ model: 'definitely-not-a-model-9000' }]).valid).toBe(false) + expect(validate([{ apiKey: '{{KEY}}' }]).valid).toBe(false) + expect(validate({ model: 'claude-sonnet-5' }).valid).toBe(false) + }) + + it('accepts a row tuning value the model declares and refuses one it does not', () => { + const levels = getThinkingLevelsForModel('claude-sonnet-5') + expect(levels?.length).toBeGreaterThan(0) + const ok = validate([ + { model: 'claude-sonnet-5', thinkingLevel: ` ${levels![0].toUpperCase()} ` }, + ]) + expect(ok.valid).toBe(true) + expect((ok as { value: Array<{ thinkingLevel?: string }> }).value[0].thinkingLevel).toBe( + levels![0] + ) + + const bad = validate([{ model: 'claude-sonnet-5', thinkingLevel: 'bogus' }]) + expect(bad.valid).toBe(false) + expect((bad as { error: { error: string } }).error.error).toContain('thinking level option') + + const notAString = validate([{ model: 'claude-sonnet-5', thinkingLevel: 42 }]) + expect(notAString.valid).toBe(false) + expect((notAString as { error: { error: string } }).error.error).toContain('"42"') + + const undeclared = (['reasoningEffort', 'verbosity', 'thinkingLevel'] as const).find( + (knob) => getTuningOptionsForModel('claude-sonnet-5', knob) === null + ) + expect(undeclared).toBeDefined() + const missingKnob = validate([{ model: 'claude-sonnet-5', [undeclared as string]: 'low' }]) + expect(missingKnob.valid).toBe(false) + }) + + it('refuses more rows than the cap', () => { + const rows = Array.from({ length: 6 }, () => ({ model: 'claude-sonnet-5' })) + const result = validate(rows) + expect(result.valid).toBe(false) + expect((result as { error: { error: string } }).error.error).toContain('at most 5') + }) + }) + it('accepts condition-input arrays with arbitrary item ids', () => { const result = validateInputsForBlock( 'condition', diff --git a/apps/sim/lib/workflows/editing/validation.ts b/apps/sim/lib/workflows/editing/validation.ts index e02ca16ef02..6d85d8a0999 100644 --- a/apps/sim/lib/workflows/editing/validation.ts +++ b/apps/sim/lib/workflows/editing/validation.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { generateShortId } from '@sim/utils/id' import { omit } from '@sim/utils/object' import { isHosted as isHostedDeployment } from '@/lib/core/config/env-flags' import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' @@ -8,6 +9,15 @@ import { MCP_SERVER_ADVANCED_TOOL_TYPE } from '@/lib/mcp/shared' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' import { resolveAccessControlBlockType } from '@/lib/permission-groups/integration-allowlist' +import { + FALLBACK_TUNING_KNOBS, + FALLBACK_TUNING_LABELS, + getTuningOptionsForModel, + isTuningValueValidForModel, + isWholeEnvVarReference, + MAX_FALLBACK_MODELS, + normalizeTuningValues, +} from '@/lib/workflows/blocks/fallback-models' import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' import { validateSelectorIds } from '@/lib/workflows/editing/selector-validator' import { containsReference } from '@/lib/workflows/sanitization/references' @@ -377,6 +387,50 @@ function validateAgentSkillEntry(item: any, index: number): string | null { return null } +/** + * Validates one fallback-model row. Returns an error string or null when valid. + * + * Refuses rather than repairs: an unknown model, sim-auto, or a raw key is an + * authoring mistake the caller must see. A missing React-key `id` is the one + * thing filled in, since it carries no meaning. + */ +function validateFallbackModelEntry(item: any, index: number): string | null { + const where = `fallbackModels[${index}]` + if (item === null || typeof item !== 'object' || Array.isArray(item)) { + return `${where} must be an object { model, apiKey? }` + } + const model = typeof item.model === 'string' ? item.model.trim() : '' + if (model === '') { + return `${where} is missing a string "model"` + } + if (isAutoModel(model)) { + return `${where}: sim-auto cannot be a fallback model; it already routes and falls back on its own` + } + if (!isKnownModelId(model) && !isCustomModelId(model)) { + const suggestions = suggestModelIdsForUnknownModel(model) + const suggestionText = + suggestions.length > 0 ? ` Valid options include: ${suggestions.join(', ')}.` : '' + return `${where}: unknown model id "${model}".${suggestionText}` + } + if (item.apiKey !== undefined && item.apiKey !== null && item.apiKey !== '') { + if (!isWholeEnvVarReference(item.apiKey)) { + return `${where}.apiKey must be a whole {{ENV_VAR}} reference; put the key in an environment variable instead of pasting it` + } + } + for (const knob of FALLBACK_TUNING_KNOBS) { + const value = item[knob] + if (value === undefined || value === null || value === '') continue + if (typeof value !== 'string' || !isTuningValueValidForModel(model, knob, value)) { + const options = getTuningOptionsForModel(model, knob) + const hint = options + ? ` Valid options: ${options.join(', ')}.` + : ` ${model} has no such setting.` + return `${where}.${knob}: "${String(value)}" is not a ${FALLBACK_TUNING_LABELS[knob].toLowerCase()} option for ${model}.${hint}` + } + } + return null +} + /** * Validates a value against its expected subBlock type * Returns validation result with the value or an error @@ -581,6 +635,57 @@ export function validateValueForSubBlockType( return { valid: true, value } } + case 'model-fallback-list': { + if (!Array.isArray(value)) { + return { + valid: false, + error: { + blockId, + blockType, + field: fieldName, + value, + error: `Invalid model-fallback-list value for field "${fieldName}" - expected an array of { model, apiKey? } objects`, + }, + } + } + if (value.length > MAX_FALLBACK_MODELS) { + return { + valid: false, + error: { + blockId, + blockType, + field: fieldName, + value, + error: `"${fieldName}" allows at most ${MAX_FALLBACK_MODELS} fallback models`, + }, + } + } + const fallbackErrors = value + .map((item, index) => validateFallbackModelEntry(item, index)) + .filter((err): err is string => err !== null) + if (fallbackErrors.length > 0) { + return { + valid: false, + error: { + blockId, + blockType, + field: fieldName, + value, + error: `Invalid fallback ${fallbackErrors.length === 1 ? 'entry' : 'entries'} in "${fieldName}": ${fallbackErrors.join('; ')}`, + }, + } + } + return { + valid: true, + value: value.map((item: Record & { model: string }) => ({ + id: typeof item.id === 'string' && item.id ? item.id : generateShortId(), + model: item.model.trim(), + ...(isWholeEnvVarReference(item.apiKey) ? { apiKey: item.apiKey.trim() } : {}), + ...normalizeTuningValues(item), + })), + } + } + case 'skill-input': { // Should be an array of skill reference objects ({ skillId, name? }) if (!Array.isArray(value)) { diff --git a/apps/sim/lib/workflows/search-replace/indexer.test.ts b/apps/sim/lib/workflows/search-replace/indexer.test.ts index e2581a05a3c..861b4fa4ec5 100644 --- a/apps/sim/lib/workflows/search-replace/indexer.test.ts +++ b/apps/sim/lib/workflows/search-replace/indexer.test.ts @@ -1384,6 +1384,11 @@ describe('indexWorkflowSearchMatches', () => { type: 'input-mapping', value: { childInput: 'mapped visible value' }, }, + fallbackModels: { + id: 'fallbackModels', + type: 'model-fallback-list', + value: [{ id: 'row-1', model: 'fallback-visible-model', apiKey: '{{HIDDEN_KEY_REF}}' }], + }, }, } const blockConfigs = { @@ -1396,6 +1401,7 @@ describe('indexWorkflowSearchMatches', () => { { id: 'skills', title: 'Skills', type: 'skill-input' }, { id: 'runAt', title: 'Run At', type: 'time-input' }, { id: 'mapping', title: 'Input Mapping', type: 'input-mapping' }, + { id: 'fallbackModels', title: 'Fallback models', type: 'model-fallback-list' }, ], }, } @@ -1430,7 +1436,28 @@ describe('indexWorkflowSearchMatches', () => { mode: 'text', blockConfigs, }).filter((match) => match.blockId === 'structured-1') + const fallbackMatches = indexWorkflowSearchMatches({ + workflow, + query: 'fallback-visible', + mode: 'text', + blockConfigs, + }).filter((match) => match.blockId === 'structured-1') + expect(fallbackMatches).toEqual([ + expect.objectContaining({ + subBlockId: 'fallbackModels', + valuePath: [0, 'model'], + searchText: 'fallback-visible-model', + }), + ]) + /** A row key is a `{{VAR}}` reference; text search must never offer to rewrite it. */ + const keyMatches = indexWorkflowSearchMatches({ + workflow, + query: 'HIDDEN_KEY_REF', + mode: 'text', + blockConfigs, + }).filter((match) => match.blockId === 'structured-1') + expect(keyMatches).toEqual([]) expect(containsMatches).toEqual([ expect.objectContaining({ subBlockId: 'filters', diff --git a/apps/sim/lib/workflows/search-replace/indexer.ts b/apps/sim/lib/workflows/search-replace/indexer.ts index 128972236fd..c2f6880c816 100644 --- a/apps/sim/lib/workflows/search-replace/indexer.ts +++ b/apps/sim/lib/workflows/search-replace/indexer.ts @@ -212,6 +212,10 @@ function isSearchableLeafPath( if (mode === 'text' && subBlockType === 'messages-input' && lastSegment === 'role') { return false } + /** A fallback row's key is a `{{VAR}}` reference; rewriting it would turn it into a raw value. */ + if (mode === 'text' && subBlockType === 'model-fallback-list' && lastSegment === 'apiKey') { + return false + } if (mode === 'text' && subBlockType === 'tool-input') { if (TOOL_INPUT_TEXT_EXCLUDED_LEAF_KEYS.has(lastSegment)) return false if (lastSegment.endsWith('Id')) return false diff --git a/apps/sim/lib/workflows/subblocks/display.test.ts b/apps/sim/lib/workflows/subblocks/display.test.ts index 863765cd467..50cfbfa0095 100644 --- a/apps/sim/lib/workflows/subblocks/display.test.ts +++ b/apps/sim/lib/workflows/subblocks/display.test.ts @@ -14,6 +14,7 @@ vi.mock('@/blocks', () => ({ import { getDisplayValue, resolveDropdownLabel, + resolveFallbackModelsLabel, resolveFilterFieldLabel, resolveFolderPathLabel, resolveSandboxLabel, @@ -187,6 +188,26 @@ describe('resolveSkillsLabel', () => { }) }) +describe('resolveFallbackModelsLabel', () => { + const fallbackList = { id: 'fallbackModels', type: 'model-fallback-list' } as SubBlockConfig + + it('lists the models in order and never the row keys', () => { + expect( + resolveFallbackModelsLabel(fallbackList, [ + { id: 'a', model: 'gpt-5' }, + { id: 'b', model: 'openrouter/x', apiKey: '{{OPENROUTER_API_KEY}}' }, + { id: 'c', model: 'gemini-3.6-flash' }, + ]) + ).toBe('gpt-5, openrouter/x +1') + }) + + it('returns null for other subblocks and for an empty or model-less list', () => { + expect(resolveFallbackModelsLabel(skillInput, [{ model: 'gpt-5' }])).toBeNull() + expect(resolveFallbackModelsLabel(fallbackList, [])).toBeNull() + expect(resolveFallbackModelsLabel(fallbackList, [{ id: 'a', model: '' }])).toBeNull() + }) +}) + describe('resolveSandboxLabel', () => { const sandboxes = [{ id: '443f4934-26ab-44ab-8000-000000000000', name: 'Test' }] diff --git a/apps/sim/lib/workflows/subblocks/display.ts b/apps/sim/lib/workflows/subblocks/display.ts index b371fda035e..3d47d307a09 100644 --- a/apps/sim/lib/workflows/subblocks/display.ts +++ b/apps/sim/lib/workflows/subblocks/display.ts @@ -582,6 +582,29 @@ export function resolveSkillsLabel( return summarizeNames(names) } +/** + * Resolves a fallback-model list to its model ids, e.g. "gpt-5.6, gemini-3.6-flash +1". + * Returns null for other subblocks and for an empty list so callers fall through. + * Row keys are never shown. + */ +export function resolveFallbackModelsLabel( + subBlock: SubBlockConfig | undefined, + rawValue: unknown +): string | null { + if (subBlock?.type !== 'model-fallback-list') return null + if (!Array.isArray(rawValue) || rawValue.length === 0) return null + + const models = rawValue + .map((row: unknown) => { + if (!row || typeof row !== 'object') return null + const model = (row as { model?: unknown }).model + return typeof model === 'string' && model.trim() ? model.trim() : null + }) + .filter((model): model is string => !!model) + + return summarizeNames(models) +} + /** * Resolves the Function block's stored sandbox id to the sandbox name. * diff --git a/apps/sim/tools/params.ts b/apps/sim/tools/params.ts index 7666075e19b..f1cce808250 100644 --- a/apps/sim/tools/params.ts +++ b/apps/sim/tools/params.ts @@ -659,6 +659,7 @@ const EXCLUDED_SUBBLOCK_TYPES = new Set([ 'mcp-dynamic-args', 'variables-input', 'messages-input', + 'model-fallback-list', 'router-input', 'text', ]) diff --git a/packages/workflow-types/src/blocks.ts b/packages/workflow-types/src/blocks.ts index e40cf7441d8..10e08a588b0 100644 --- a/packages/workflow-types/src/blocks.ts +++ b/packages/workflow-types/src/blocks.ts @@ -54,6 +54,7 @@ export type SubBlockType = | 'text' | 'router-input' | 'table-selector' + | 'model-fallback-list' | 'column-selector' | 'modal' diff --git a/scripts/generate-docs.ts b/scripts/generate-docs.ts index 92853f28f38..9849117afbe 100755 --- a/scripts/generate-docs.ts +++ b/scripts/generate-docs.ts @@ -4966,6 +4966,7 @@ const SUBBLOCK_TYPE_TO_SEMANTIC: Record = { 'oauth-input': 'string', code: 'string', 'file-upload': 'string', + 'model-fallback-list': 'json', text: 'string', }