fix(agent): close the adversarial review's fallback findings

A routed sim-auto primary that fails now shows in the trace as the auto
identity rather than the pool model, which is the name applyAutoModelLabel
exists to hide. Per-row tuning resolves against the model the builder
configured, the one the editor showed the fields for, so a row's value
applies under sim-auto whatever pool model was routed. A fallback on
another provider no longer receives the primary's Azure, Vertex, or
Bedrock fields, which only that family reads.

Author: mzxchandra

apps/sim/executor/handlers/agent/agent-handler.test.ts

@@ -757,6 +757,32 @@ describe('AgentBlockHandler', () => {
       )
     })
 
+    it('leaves provider-family credentials off a fallback on another provider', async () => {
+      mockExecuteProviderRequest
+        .mockRejectedValueOnce(new Error('one'))
+        .mockRejectedValueOnce(new Error('two'))
+        .mockResolvedValueOnce(providerResponse('gpt-4o-mini'))
+
+      await handler.execute(mockContext, mockBlock, {
+        ...baseInputs,
+        vertexCredential: 'vertex-secret',
+        bedrockSecretKey: 'bedrock-secret',
+        azureEndpoint: 'https://azure.example.com',
+        fallbackModels: [{ model: 'claude-sonnet-5' }, { model: 'gpt-4o-mini' }],
+      })
+
+      const [, crossProvider] = mockExecuteProviderRequest.mock.calls[1]
+      const [, sameProvider] = mockExecuteProviderRequest.mock.calls[2]
+      expect(crossProvider.vertexCredential).toBeUndefined()
+      expect(crossProvider.bedrockSecretKey).toBeUndefined()
+      expect(crossProvider.azureEndpoint).toBeUndefined()
+      expect(JSON.stringify(crossProvider)).not.toMatch(
+        /vertex-secret|bedrock-secret|azure\.example/
+      )
+      expect(sameProvider.bedrockSecretKey).toBe('bedrock-secret')
+      expect(sameProvider.azureEndpoint).toBe('https://azure.example.com')
+    })
+
     it('ignores a row key the block did not store as a reference', async () => {
       mockExecuteProviderRequest
         .mockRejectedValueOnce(new Error('one'))
@@ -1112,17 +1138,22 @@ describe('AgentBlockHandler', () => {
     it('keeps the fallback name when a routed sim-auto primary fails and a fallback answers', async () => {
       mockExecuteProviderRequest
         .mockRejectedValueOnce(new Error('pool model down'))
-        .mockResolvedValueOnce(providerResponse('gpt-4o-mini'))
+        .mockResolvedValueOnce(providerResponse('gpt-5.4-mini'))
+      const blockLog = openLog()
 
-      const result = (await handler.execute(mockContext, mockBlock, {
+      const result = (await handler.execute({ ...mockContext, blockLogs: [blockLog] }, mockBlock, {
         model: SIM_AUTO_MODEL_ID,
         systemPrompt: 'Be brief.',
         userPrompt: 'Hello!',
-        fallbackModels: [{ model: 'gpt-4o-mini' }],
+        fallbackModels: [{ model: 'gpt-5.4-mini', reasoningEffort: 'low' }],
       })) as { model: string }
 
-      expect(result.model).toBe('gpt-4o-mini')
+      expect(result.model).toBe('gpt-5.4-mini')
       expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(2)
+      /** The trace names the auto identity, never the pool model that was routed. */
+      expect(blockLog.modelFallbacks).toEqual([SIM_AUTO_MODEL_ID])
+      /** The row's tuning was set against the auto id in the editor, so it applies whatever was routed. */
+      expect(mockExecuteProviderRequest.mock.calls[1][1].reasoningEffort).toBe('low')
       /** The auto identity preamble belongs to the pool model, not a named fallback. */
       const systemText = (request: { messages?: Array<{ role: string; content: string }> }) =>
         (request.messages ?? [])

apps/sim/executor/handlers/agent/agent-handler.ts

@@ -170,14 +170,42 @@ function stripAutoPreamble(messages: Message[] | undefined): Message[] | undefin
   })
 }
 
+/**
+ * Block fields that only a provider family reads. A fallback on another provider
+ * never needs them, so they are left off its request rather than handed to a
+ * provider that has no use for a Bedrock secret or a Vertex credential.
+ */
+const PROVIDER_FAMILY_CREDENTIAL_FIELDS = [
+  'azureEndpoint',
+  'azureApiVersion',
+  'vertexProject',
+  'vertexLocation',
+  'vertexCredential',
+  'bedrockAccessKeyId',
+  'bedrockSecretKey',
+  'bedrockRegion',
+] as const satisfies ReadonlyArray<keyof AgentInputs>
+
 /** One model in the order the block tries them; the primary carries the block's own key. */
 interface ModelCandidate extends FallbackModelCandidate {
   isPrimary: boolean
+  /**
+   * What the trace calls this model when it fails. A routed sim-auto primary
+   * shows as the auto identity, since naming the pool model is the leak that
+   * `applyAutoModelLabel` exists to close.
+   */
+  traceName?: string
 }
 
 interface ExecuteAcrossModelsConfig {
   candidates: ModelCandidate[]
   primaryModel: string
+  /**
+   * The model the builder configured, which is what the editor showed the
+   * per-row tuning fields against. Under sim-auto that is the auto id, not the
+   * pool model routed for this run, so a row's value applies whatever was routed.
+   */
+  configuredModel: string
   primaryProviderId: string
   messages: Message[] | undefined
   /** Provider id to hydrated messages; seeded with the primary, filled per fallback provider. */
@@ -523,7 +551,12 @@ export class AgentBlockHandler implements BlockHandler {
         })
       }
       const candidates: ModelCandidate[] = [
-        { model, apiKey: modelInputs.apiKey, isPrimary: true },
+        {
+          model,
+          apiKey: modelInputs.apiKey,
+          isPrimary: true,
+          ...(autoRouting ? { traceName: SIM_AUTO_MODEL_ID } : {}),
+        },
         ...fallbackCandidates.map((candidate) => ({ ...candidate, isPrimary: false })),
       ]
       const {
@@ -533,6 +566,7 @@ export class AgentBlockHandler implements BlockHandler {
       } = await this.executeAcrossModels(ctx, block, {
         candidates,
         primaryModel: model,
+        configuredModel: autoRouting ? SIM_AUTO_MODEL_ID : model,
         primaryProviderId: providerId,
         messages: messagesWithInputFiles,
         hydratedByProvider,
@@ -2534,7 +2568,7 @@ export class AgentBlockHandler implements BlockHandler {
       if (!candidate.isPrimary) {
         const { adjustments, ...tuning } = resolveFallbackTuning(
           candidate,
-          config.primaryModel,
+          config.configuredModel,
           config.modelInputs
         )
         /**
@@ -2552,13 +2586,12 @@ export class AgentBlockHandler implements BlockHandler {
           })
           rowKey = undefined
         }
+        const sameProvider = candidateProviderId === config.primaryProviderId
         inputs = {
-          ...config.modelInputs,
-          apiKey:
-            rowKey ??
-            (candidateProviderId === config.primaryProviderId
-              ? config.modelInputs.apiKey
-              : undefined),
+          ...(sameProvider
+            ? config.modelInputs
+            : omit(config.modelInputs, [...PROVIDER_FAMILY_CREDENTIAL_FIELDS])),
+          apiKey: rowKey ?? (sameProvider ? config.modelInputs.apiKey : undefined),
           previousInteractionId: undefined,
           ...(config.fallbackSystemPrompt !== undefined
             ? { systemPrompt: config.fallbackSystemPrompt }
@@ -2607,7 +2640,7 @@ export class AgentBlockHandler implements BlockHandler {
         return { result, servedModel: candidate.model, resultRegistry }
       } catch (error) {
         lastError = error
-        failedModels.push(candidate.model)
+        failedModels.push(candidate.traceName ?? candidate.model)
         if (!hasNext || ctx.abortSignal?.aborted || !isRetryableBlockError(error)) {
           this.recordModelFallbacks(ctx, block, failedModels.slice(0, -1))
           throw error