improvement(media): label fal auth failures with the refused key and use the shared env-flags mock

getFalApiKey now returns the env var its key came from, so a refused key in the
numbered pool is named on the metric. The metrics module reads isHosted on each
call instead of copying it at import, and the test sets it through the central
env-flags mock under fake timers.

Author: waleedlatif1

apps/sim/lib/media/falai-audio.ts

@@ -141,7 +141,7 @@ function buildInput(
 
 export async function generateFalAudio(params: GenerateFalAudioParams): Promise<GeneratedAudio> {
   const type: AudioType = params.type || 'speech'
-  const apiKey = getFalApiKey()
+  const { apiKey, envVarName } = getFalApiKey()
 
   // Voice cloning: a reference sample routes to a zero-shot clone model (F5-TTS),
   // which conditions on the sample (ref_audio_url) and speaks the prompt in that voice.
@@ -156,7 +156,7 @@ export async function generateFalAudio(params: GenerateFalAudioParams): Promise<
       model,
       input,
       apiKey,
-      'generate_audio',
+      { tool: 'generate_audio', envVarName },
       params.signal
     )
     const cost = await recordFalCost(apiKey, model, requestId, params.onCost)
@@ -182,7 +182,7 @@ export async function generateFalAudio(params: GenerateFalAudioParams): Promise<
     model,
     input,
     apiKey,
-    'generate_audio',
+    { tool: 'generate_audio', envVarName },
     params.signal
   )
   const cost = await recordFalCost(apiKey, model, requestId, params.onCost)

apps/sim/lib/media/falai-hosted-key-metrics.test.ts

@@ -1,6 +1,7 @@
 import { jsonResponse } from '@sim/testing/helpers/http'
+import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock'
 import { inputValidationMock } from '@sim/testing/mocks/input-validation.mock'
-import { afterEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const { sentMetricData } = vi.hoisted(() => ({
   sentMetricData: [] as Array<{
@@ -10,10 +11,6 @@ const { sentMetricData } = vi.hoisted(() => ({
 }))
 
 vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock)
-vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({
-  ...(await importOriginal<typeof import('@/lib/core/config/env-flags')>()),
-  isHosted: true,
-}))
 vi.mock('@aws-sdk/client-cloudwatch', () => ({
   StandardUnit: { Count: 'Count', None: 'None', Milliseconds: 'Milliseconds' },
   PutMetricDataCommand: class {
@@ -36,41 +33,58 @@ function stubFalSubmit(status: number, body: unknown) {
   )
 }
 
-async function failedHostedKeyReasons(): Promise<Array<Record<string, string | undefined>>> {
+async function failedHostedKeyDimensions(): Promise<Array<Record<string, string | undefined>>> {
   await flushHostedKeyMetrics()
   return sentMetricData
     .filter((datum) => datum.MetricName === 'Failed')
     .map((datum) => Object.fromEntries((datum.Dimensions ?? []).map((d) => [d.Name, d.Value])))
 }
 
 describe('a fal submit the hosted account refuses', () => {
-  afterEach(() => {
-    vi.unstubAllGlobals()
+  beforeEach(() => {
+    vi.useFakeTimers()
+    setEnvFlags({ isHosted: true })
     sentMetricData.length = 0
   })
 
+  afterEach(() => {
+    vi.useRealTimers()
+    resetEnvFlagsMock()
+  })
+
   it.each([
     [403, { detail: 'User is locked. Reason: Exhausted balance.' }],
     [401, { detail: 'Invalid API key' }],
-  ])('records a hosted-key auth failure for a %i', async (status, body) => {
+  ])('records a hosted-key auth failure against the refused key for a %i', async (status, body) => {
     stubFalSubmit(status, body)
 
-    await expect(runFalQueue('fal-ai/model', {}, 'key', 'generate_audio')).rejects.toThrow(
-      `Fal.ai API error: ${status}`
-    )
+    await expect(
+      runFalQueue('fal-ai/model', {}, 'key', {
+        tool: 'generate_audio',
+        envVarName: 'FALAI_API_KEY_2',
+      })
+    ).rejects.toThrow(`Fal.ai API error: ${status}`)
 
-    expect(await failedHostedKeyReasons()).toEqual([
-      expect.objectContaining({ Provider: 'falai', Tool: 'generate_audio', Reason: 'auth' }),
+    expect(await failedHostedKeyDimensions()).toEqual([
+      expect.objectContaining({
+        Provider: 'falai',
+        Tool: 'generate_audio',
+        Key: 'FALAI_API_KEY_2',
+        Reason: 'auth',
+      }),
     ])
   })
 
   it('records nothing when fal rejects the request input', async () => {
     stubFalSubmit(422, { detail: [{ msg: 'field required' }] })
 
-    await expect(runFalQueue('fal-ai/model', {}, 'key', 'generate_video')).rejects.toThrow(
-      'Fal.ai API error: 422'
-    )
+    await expect(
+      runFalQueue('fal-ai/model', {}, 'key', {
+        tool: 'generate_video',
+        envVarName: 'FALAI_API_KEY_1',
+      })
+    ).rejects.toThrow('Fal.ai API error: 422')
 
-    expect(await failedHostedKeyReasons()).toEqual([])
+    expect(await failedHostedKeyDimensions()).toEqual([])
   })
 })

apps/sim/lib/media/falai-video.ts

@@ -148,7 +148,7 @@ export async function generateFalVideo(params: GenerateFalVideoParams): Promise<
     )
   }
 
-  const apiKey = getFalApiKey()
+  const { apiKey, envVarName } = getFalApiKey()
 
   let endpoint = config.endpoint
   const input: Record<string, unknown> = { prompt: params.prompt }
@@ -181,7 +181,7 @@ export async function generateFalVideo(params: GenerateFalVideoParams): Promise<
     endpoint,
     input,
     apiKey,
-    'generate_video',
+    { tool: 'generate_video', envVarName },
     params.signal
   )
   const cost = await recordFalCost(apiKey, endpoint, requestId, params.onCost)

apps/sim/lib/media/falai.test.ts

@@ -13,6 +13,7 @@ import { downloadFalMedia, type FalCostRecorder, runFalQueue } from '@/lib/media
 import { generateFalAudio } from '@/lib/media/falai-audio'
 import { generateFalVideo } from '@/lib/media/falai-video'
 
+const FAL_KEY_USE = { tool: 'generate_video', envVarName: 'FALAI_API_KEY' } as const
 const CANCEL_URL = 'https://queue.fal.run/fal-ai/model/requests/req-1/cancel'
 const STATUS_URL = 'https://queue.fal.run/fal-ai/model/requests/req-1/status'
 
@@ -71,7 +72,7 @@ describe('runFalQueue on a request that never finishes', () => {
       'fal-ai/model',
       { prompt: 'p' },
       'key',
-      'generate_video',
+      FAL_KEY_USE,
       caller.signal
     )
     const outcome = generation.then(
@@ -97,7 +98,7 @@ describe('runFalQueue on a request that never finishes', () => {
       'fal-ai/model',
       { prompt: 'p' },
       'key',
-      'generate_video',
+      FAL_KEY_USE,
       caller.signal
     )
     const outcome = generation.then(
@@ -116,7 +117,7 @@ describe('runFalQueue on a request that never finishes', () => {
 
   it('fails and cancels after its own ceiling even when nothing aborts it', async () => {
     const fal = stubStuckFalQueue()
-    const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', 'generate_video')
+    const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', FAL_KEY_USE)
     const outcome = generation.then(
       () => 'resolved',
       (error: unknown) => String(error)
@@ -135,7 +136,7 @@ describe('runFalQueue on a request that never finishes', () => {
       'fal-ai/model',
       { prompt: 'p' },
       'key',
-      'generate_video',
+      FAL_KEY_USE,
       caller.signal
     ).catch(() => undefined)
     await vi.advanceTimersByTimeAsync(3_500)
@@ -153,7 +154,7 @@ describe('runFalQueue on a request that never finishes', () => {
     const fal = stubStuckFalQueue()
 
     await expect(
-      runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', 'generate_video', AbortSignal.abort())
+      runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', FAL_KEY_USE, AbortSignal.abort())
     ).rejects.toThrow()
     expect(fal.calls).toEqual([])
   })
@@ -180,7 +181,7 @@ describe('runFalQueue on a request that never finishes', () => {
       'fal-ai/model',
       { prompt: 'p' },
       'key',
-      'generate_video',
+      FAL_KEY_USE,
       caller.signal
     ).catch(() => 'rejected')
 
@@ -199,7 +200,7 @@ describe('runFalQueue on a request that never finishes', () => {
       'fal-ai/model',
       { prompt: 'p' },
       'key',
-      'generate_video',
+      FAL_KEY_USE,
       caller.signal
     ).catch(() => undefined)
     await vi.advanceTimersByTimeAsync(0)

apps/sim/lib/media/falai.ts

@@ -34,8 +34,11 @@ const FAL_CANCEL_TIMEOUT_MS = 5000
  */
 const FAL_SUBMIT_TIMEOUT_MS = 60_000
 
-/** The Mothership tools that generate media on Sim's hosted Fal.ai keys. */
-type FalMediaTool = 'generate_audio' | 'generate_video'
+/** Who spent a hosted Fal.ai key: the Mothership tool and the env var the key came from. */
+interface FalKeyUse {
+  tool: 'generate_audio' | 'generate_video'
+  envVarName: string
+}
 
 /**
  * Statuses fal answers when the hosted account itself refuses the call (a revoked key, or an
@@ -48,15 +51,16 @@ const FAL_ACCOUNT_REFUSAL_STATUSES = new Set([401, 402, 403])
  * (FALAI_API_KEY_COUNT + FALAI_API_KEY_1..N), round-robined by minute,
  * mirroring getRotatingApiKey. Falls back to a single FALAI_API_KEY for dev.
  */
-export function getFalApiKey(): string {
+export function getFalApiKey(): { apiKey: string; envVarName: string } {
   const count = Number.parseInt(process.env.FALAI_API_KEY_COUNT || '0', 10)
-  const keys: string[] = []
+  const keys: Array<{ apiKey: string; envVarName: string }> = []
   for (let i = 1; i <= count; i++) {
-    const key = process.env[`FALAI_API_KEY_${i}`]
-    if (key) keys.push(key)
+    const envVarName = `FALAI_API_KEY_${i}`
+    const apiKey = process.env[envVarName]
+    if (apiKey) keys.push({ apiKey, envVarName })
   }
   if (keys.length === 0 && process.env.FALAI_API_KEY) {
-    keys.push(process.env.FALAI_API_KEY)
+    keys.push({ apiKey: process.env.FALAI_API_KEY, envVarName: 'FALAI_API_KEY' })
   }
   if (keys.length === 0) {
     throw new Error(
@@ -111,7 +115,7 @@ export async function runFalQueue(
   endpoint: string,
   input: Record<string, unknown>,
   apiKey: string,
-  tool: FalMediaTool,
+  keyUse: FalKeyUse,
   signal?: AbortSignal
 ): Promise<FalQueueResult> {
   const ceiling = new AbortController()
@@ -121,7 +125,7 @@ export async function runFalQueue(
   )
   const queueSignal = signal ? AbortSignal.any([signal, ceiling.signal]) : ceiling.signal
   try {
-    return await submitAndPollFalQueue(endpoint, input, apiKey, tool, queueSignal)
+    return await submitAndPollFalQueue(endpoint, input, apiKey, keyUse, queueSignal)
   } finally {
     clearTimeout(ceilingTimer)
   }
@@ -131,7 +135,7 @@ async function submitAndPollFalQueue(
   endpoint: string,
   input: Record<string, unknown>,
   apiKey: string,
-  tool: FalMediaTool,
+  keyUse: FalKeyUse,
   queueSignal: AbortSignal
 ): Promise<FalQueueResult> {
   // Only a submit already under way outlives its caller; a stopped call never starts one.
@@ -150,8 +154,8 @@ async function submitAndPollFalQueue(
     if (FAL_ACCOUNT_REFUSAL_STATUSES.has(createResponse.status)) {
       hostedKeyMetrics.recordFailed({
         provider: 'falai',
-        tool,
-        key: 'FALAI_API_KEY',
+        tool: keyUse.tool,
+        key: keyUse.envVarName,
         reason: 'auth',
       })
     }

apps/sim/lib/monitoring/metrics.ts

@@ -46,11 +46,6 @@ type QueueReason = 'actor_requests' | 'dimension' | 'queue_position'
 /** `metering` marks a provider call that succeeded but could not be priced. */
 type FailureReason = 'rate_limited' | 'auth' | 'other' | 'metering'
 
-// Hosted keys exist only on Sim Cloud, whose app and workers can always reach
-// CloudWatch (task role or workload identity). Elsewhere recorders are no-ops,
-// so they stay always-safe to call.
-const ENABLED = isHosted
-
 const ENVIRONMENT =
   process.env.OTEL_DEPLOYMENT_ENVIRONMENT ||
   process.env.DEPLOYMENT_ENVIRONMENT ||
@@ -98,13 +93,16 @@ function buildDimensions(labels: Record<string, string | undefined>) {
   return dimensions
 }
 
+// Hosted keys exist only on Sim Cloud, whose app and workers can always reach
+// CloudWatch (task role or workload identity). Elsewhere recorders are no-ops,
+// so they stay always-safe to call.
 function enqueue(
   MetricName: string,
   Value: number,
   Unit: StandardUnit,
   labels: Record<string, string | undefined>
 ): void {
-  if (!ENABLED) return
+  if (!isHosted) return
   buffer.push({
     MetricName,
     Value,
@@ -129,7 +127,7 @@ export async function flushHostedKeyMetrics(): Promise<void> {
     logger.warn('Dropped hosted-key metric points (buffer cap reached)', { dropped })
     dropped = 0
   }
-  if (!ENABLED || buffer.length === 0) return
+  if (!isHosted || buffer.length === 0) return
   const pending = buffer
   buffer = []
   for (let i = 0; i < pending.length; i += MAX_BATCH) {

scripts/check-unused-exports.baseline.json

@@ -4702,7 +4702,6 @@
   "apps/sim/lib/messaging/email/mailer.ts#sendBatchEmails",
   "apps/sim/lib/messaging/email/types.ts#EmailAttachment",
   "apps/sim/lib/messaging/email/utils.ts#extractEmailFromAddress",
-  "apps/sim/lib/monitoring/metrics.ts#flushHostedKeyMetrics",
   "apps/sim/lib/mothership/agent-cli/types.ts#AgentCliClient",
   "apps/sim/lib/mothership/application/execute-knowledge-use-case.ts#requireCopilotKnowledgeWorkspaceId",
   "apps/sim/lib/mothership/application/execute-knowledge-use-case.ts#resolveCopilotKnowledgePrincipal",