Merge remote-tracking branch 'origin/staging' into feat/agent-fallback-models

Author: mzxchandra

.agents/skills/ship/SKILL.md

@@ -112,12 +112,19 @@ The repo is public. **Everything you publish — title, description, commit mess
 
 Describe the bug by its mechanism, not by how you found it. "Expired OAuth credentials fail to refresh in the worker" — not "the Sheets canary failed at 16:31Z for workspace abc-123". Aggregate counts are fine once detached from the tenant ("1,379 PDFs failed"); the same number attributed to a named customer is not. Replace real examples with placeholders (`<real sheet name>`) rather than cutting them — the illustration is usually the useful part.
 
-**Scrub before publishing, not after** — a leak is public the instant it posts, and editing later does not unsend the notification email. This applies to every PR you open, including ones created directly with `gh pr create` rather than through this skill. Grep the title, body, and `git log origin/staging..HEAD` before publishing:
+**Measurements are not the problem; absolute production scale is.** Keep the numbers that justify a change — durations, ratios, before/after timings, test and audit counts. They are the evidence a reviewer needs, and stripping them makes the rationale unfalsifiable. What does not belong is anything that sizes production or a tenant: table and index byte sizes, row/chunk/document totals, dead-tuple counts, buffer and heap-fetch counts, worker or instance counts. "Visiting four times as many tuples took 5.1s and 9.7s on consecutive runs" is fine; "on a 132k-chunk index" or "reclaims ~19 GB" is not. The same rule applies to code comments and migration comments, which are published exactly like a PR body — this is the most commonly missed case, because they do not feel like publishing.
+
+**Scrub before publishing, not after** — a leak is public the instant it posts, and editing later does not unsend the notification email. This applies to every PR you open, including ones created directly with `gh pr create` rather than through this skill. Grep the title, body, `git log origin/staging..HEAD`, AND the diff itself before publishing:
 
 ```bash
+# identities, IDs, infrastructure
 grep -niE 'customer-or-company-name|@[a-z0-9.-]+\.(com|io|ai)|[0-9a-f]{8}-[0-9a-f]{4}-|\.sharepoint\.com|arn:aws|https?://[a-z0-9.-]*\.internal'
+# absolute production scale — byte sizes, k/M-scale entity counts, 7-figure totals
+grep -niE '[0-9][0-9.,]* ?(TB|GB)\b|[0-9]+(\.[0-9]+)?[kKmM][- ](row|chunk|document|vector|tuple|doc)|[0-9]{1,3}(,[0-9]{3}){2,}'
 ```
 
+The second pattern deliberately allows ordinary engineering numbers (`5.1s`, `46 audits`, `2,921 tests`) and flags only production sizing.
+
 ## PR Description Format
 
 Use this exact template in the user's voice (concise, bullet points):

.github/workflows/test-build.yml

@@ -94,11 +94,14 @@ jobs:
         working-directory: packages/db
         run: bun run db:migrate
 
-      - name: Verify retired-column contract migration in PostgreSQL
+      - name: Verify schema contract migrations in PostgreSQL
         working-directory: packages/db
         env:
-          RETIRED_COLUMNS_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
-        run: bunx vitest run scripts/retired-columns.postgres.test.ts
+          MIGRATION_CONTRACT_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
+        run: >-
+          bunx vitest run
+          scripts/retired-columns.postgres.test.ts
+          scripts/connector-sync-schedule-precision.postgres.test.ts
 
       - name: Verify OAuth lifecycle and SCIM membership guards in PostgreSQL
         working-directory: apps/sim

apps/sim/background/workspace-file-search-index.test.ts

@@ -35,7 +35,7 @@ describe('workspace file search index task', () => {
   it('uses isolated medium workers with a hard global concurrency and duration cap', () => {
     expect(workspaceFileSearchIndexTask).toMatchObject({
       id: 'workspace-file-search-index',
-      machine: 'medium-1x',
+      machine: 'medium-2x',
       maxDuration: FILE_SEARCH_INDEX_MAX_DURATION_SECONDS,
       retry: { maxAttempts: 3 },
       queue: {

apps/sim/background/workspace-file-search-index.ts

@@ -15,7 +15,7 @@ import {
  */
 export const workspaceFileSearchIndexTask = task({
   id: 'workspace-file-search-index',
-  machine: 'medium-1x',
+  machine: 'medium-2x',
   maxDuration: FILE_SEARCH_INDEX_MAX_DURATION_SECONDS,
   retry: { maxAttempts: 3 },
   queue: {

apps/sim/connectors/google-workspace/company-crawl.test.ts

@@ -502,21 +502,24 @@ describe('Google Workspace per-user central crawl', () => {
     expect((await list(context(), undefined, CONFIG, 'google_calendar')).documents).toHaveLength(1)
   })
 
-  it('isolates explicit Calendar list access failures without claiming a disabled service', async () => {
-    listUserDocuments.mockRejectedValueOnce(
-      new GoogleApiError('calendar.events.list', 403, ['forbidden'])
-    )
-    const first = await list(context(), undefined, CONFIG, 'google_calendar')
-    expect(first.listingFailures?.samples[0]).toEqual({
-      scope: 'alice@corp.com',
-      operation: 'calendar.events.list',
-      status: 403,
-      reasons: ['forbidden'],
-    })
-    const second = await list(context(), first.nextCursor, CONFIG, 'google_calendar')
-    expect(second.documents[0].acl).toEqual(['u:bob@corp.com'])
-    expect(second.reconciliationSafe).toBe(false)
-  })
+  it.each(['forbidden', 'notACalendarUser'])(
+    'isolates explicit Calendar list access failures (%s) without claiming a disabled service',
+    async (reason) => {
+      listUserDocuments.mockRejectedValueOnce(
+        new GoogleApiError('calendar.events.list', 403, [reason])
+      )
+      const first = await list(context(), undefined, CONFIG, 'google_calendar')
+      expect(first.listingFailures?.samples[0]).toEqual({
+        scope: 'alice@corp.com',
+        operation: 'calendar.events.list',
+        status: 403,
+        reasons: [reason],
+      })
+      const second = await list(context(), first.nextCursor, CONFIG, 'google_calendar')
+      expect(second.documents[0].acl).toEqual(['u:bob@corp.com'])
+      expect(second.reconciliationSafe).toBe(false)
+    }
+  )
 
   it.each([{ error: { code: 403 } }, { error: { code: 403, errors: [], details: [] } }])(
     'propagates a Calendar 403 without reason codes: %j',
@@ -542,6 +545,9 @@ describe('Google Workspace per-user central crawl', () => {
     [403, ['domainPolicy']],
     [403, ['unrecognized-provider-code']],
     [403, ['forbidden', 'unrecognized-provider-code']],
+    [403, ['notACalendarUser', 'unrecognized-provider-code']],
+    [403, ['notACalendarUser', 'insufficientPermissions']],
+    [403, ['notACalendarUser', 'rateLimitExceeded']],
     [401, ['authError']],
     [429, []],
     [500, ['backendError']],
@@ -571,6 +577,15 @@ describe('Google Workspace per-user central crawl', () => {
     await expect(list(context(), undefined, CONFIG, 'google_calendar')).rejects.toBe(error)
   })
 
+  it.each([
+    new GoogleApiError('calendar.events.list', 403, ['notACalendarUser'], false),
+    new GoogleApiError('calendar.calendarList.list', 403, ['notACalendarUser']),
+    new GoogleApiError('calendar.events.list', 401, ['notACalendarUser']),
+  ])('does not isolate Calendar unavailability outside a complete list 403: %s', async (error) => {
+    listUserDocuments.mockRejectedValueOnce(error)
+    await expect(list(context(), undefined, CONFIG, 'google_calendar')).rejects.toBe(error)
+  })
+
   it('does not suppress delegation failures that resemble provider list failures', async () => {
     const ctx = context()
     const error = new GoogleApiError('gmail.threads.list', 400, ['failedPrecondition'])

apps/sim/connectors/google-workspace/company-crawl.ts

@@ -177,7 +177,7 @@ function userListingFailure(
       : error.diagnostic.operation === 'calendar.events.list' &&
         error.status === 403 &&
         reasons.length > 0 &&
-        reasons.every((reason) => reason === 'forbidden')
+        reasons.every((reason) => reason === 'forbidden' || reason === 'notACalendarUser')
   return isolated
     ? { operation: error.diagnostic.operation, status: error.status, reasons: [...reasons] }
     : null

apps/sim/lib/knowledge/application/search-source-overview.test.ts

@@ -24,6 +24,7 @@ vi.mock('@/lib/knowledge/read-access', () => ({
 }))
 
 import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview'
+import { MAX_SEARCH_SOURCE_PROVIDER_TYPES } from '@/lib/knowledge/constants'
 
 const principal = { kind: 'session', userId: 'reader', sessionId: 'session' } as const
 const input = { organizationId: 'org-1', workspaceId: null }
@@ -36,6 +37,14 @@ const searchableProbeCount = () =>
   dbChainMockFns.limit.mock.calls.filter(([rows]) => rows === 1).length -
   AUTHORIZATION_SINGLE_ROW_READS
 
+/** The configured-provider list is read once, before the batches, under the same bound. */
+const CONFIGURED_PROVIDER_READS = 1
+
+/** Every other provider-bounded read in this use case is the indexing probe. */
+const indexingProbeCount = () =>
+  dbChainMockFns.limit.mock.calls.filter(([rows]) => rows === MAX_SEARCH_SOURCE_PROVIDER_TYPES)
+    .length - CONFIGURED_PROVIDER_READS
+
 function yieldBatches(count: number) {
   mocks.batches.mockImplementation(async function* () {
     for (let index = 0; index < count; index += 1) yield sql`batch-${sql.raw(String(index))}`
@@ -73,4 +82,31 @@ describe('readSearchSourceOverview', () => {
     expect(result.hasSearchableDocuments).toBe(false)
     expect(searchableProbeCount()).toBe(3)
   })
+
+  it('stops probing for indexing once every configured provider type is known', async () => {
+    yieldBatches(3)
+    queueTableRows(member, [{ role: 'owner' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+
+    const result = await readSearchSourceOverview.execute({ principal, input })
+
+    expect(result.providers).toEqual([{ connectorType: 'gmail', isSyncing: true }])
+    expect(indexingProbeCount()).toBe(1)
+  })
+
+  it('keeps probing every batch while a configured provider type is still unaccounted for', async () => {
+    yieldBatches(3)
+    queueTableRows(member, [{ role: 'owner' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }, { connectorType: 'notion' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+
+    const result = await readSearchSourceOverview.execute({ principal, input })
+
+    expect(result.providers).toEqual([
+      { connectorType: 'gmail', isSyncing: true },
+      { connectorType: 'notion', isSyncing: false },
+    ])
+    expect(indexingProbeCount()).toBe(3)
+  })
 })

apps/sim/lib/knowledge/application/search-source-overview.ts

@@ -117,16 +117,30 @@ export const readSearchSourceOverview = instrumentSourceOverviewUseCase(
         /** One searchable document is the whole answer, so later batches skip the probe entirely. */
         const probesSearchable: boolean = probesSources && !hasSearchableDocuments
         if (probesSearchable) searchableProbes += 1
+        /**
+         * A provider type is only read back as membership of `indexingTypes`, so once every
+         * configured type is in the set no later batch can change the answer.
+         */
+        const probesIndexing: boolean =
+          probesSources && providers.some(({ connectorType }) => !indexingTypes.has(connectorType))
         /** Annotated so the searchable probe's guard does not infer through its own result. */
         const [indexing, searchable]: [{ connectorType: string }[], { id: string }[]] =
           await Promise.all([
-            probesSources
+            probesIndexing
               ? measureSearchStage('source_overview.indexing', () =>
                   configuredProvidersQuery()
                     .where(
                       and(
                         configured,
                         syncingEnabled,
+                        /**
+                         * The probe narrows the configured set the provider list came from, so a
+                         * type already found stays found; excluding it only drops repeated work.
+                         * An empty set adds no predicate rather than a no-op one.
+                         */
+                        indexingTypes.size > 0
+                          ? notInArray(knowledgeConnector.connectorType, [...indexingTypes])
+                          : undefined,
                         or(
                           inArray(knowledgeConnector.status, ['pending', 'syncing']),
                           and(

apps/sim/lib/knowledge/connectors/google-company-scheduler.test.ts

@@ -15,15 +15,18 @@ import type {
 } from '@/lib/knowledge/connectors/partition-work'
 import { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors'
 import { GoogleApiError } from '@/connectors/google-workspace/api-errors'
+import { listGoogleWorkspaceDocuments } from '@/connectors/google-workspace/company-crawl'
 import { googleCompanyUserContextSchema } from '@/connectors/google-workspace/company-work'
 import type { GoogleWorkspaceUser } from '@/connectors/google-workspace/users'
 import { ConnectorSourceError } from '@/connectors/source-error'
 import type { ConnectorConfig, ExternalDocument, ExternalListingFailures } from '@/connectors/types'
+import { memberDocumentId } from '@/connectors/utils'
 
-const mocks = vi.hoisted(() => ({ directory: vi.fn() }))
+const mocks = vi.hoisted(() => ({ directory: vi.fn(), getUser: vi.fn() }))
 vi.mock('@/connectors/google-workspace/users', async (original) => ({
   ...(await original<typeof import('@/connectors/google-workspace/users')>()),
   listGoogleWorkspaceUsers: mocks.directory,
+  getGoogleWorkspaceUser: mocks.getUser,
 }))
 
 const document: ExternalDocument = {
@@ -203,6 +206,7 @@ function fixture(provider = 'google_calendar', syncIntervalMinutes = 60) {
 
 beforeEach(() => {
   mocks.directory.mockReset()
+  mocks.getUser.mockReset().mockImplementation(async (_token: string, id: string) => user(id))
 })
 
 describe('durable Google company user scheduling', () => {
@@ -269,6 +273,58 @@ describe('durable Google company user scheduling', () => {
     }
   )
 
+  it('continues past unavailable Calendar users without the unresolved-error pause and retries them later', async () => {
+    mocks.directory.mockResolvedValue({ users: ['a', 'b', 'c', 'z'].map(user) })
+    const f = fixture()
+    const listUserDocuments = vi.fn<ConnectorConfig['listDocuments']>(
+      async (_token, _config, _cursor, ctx) => ({
+        documents: [{ ...document, externalId: memberDocumentId('event', ctx) }],
+        hasMore: false,
+      })
+    )
+    for (let i = 0; i < 3; i++) {
+      listUserDocuments.mockRejectedValueOnce(
+        new GoogleApiError('calendar.events.list', 403, ['notACalendarUser'])
+      )
+    }
+    const syncContext = {
+      mirrorsSourceAcls: true,
+      getDelegatedAccessToken: vi.fn(async () => 'user-token'),
+    }
+    f.list.mockImplementation(async (accessToken, sourceConfig, cursor) =>
+      listGoogleWorkspaceDocuments({
+        provider: 'google_calendar',
+        accessToken,
+        sourceConfig,
+        cursor,
+        syncContext,
+        listUserDocuments,
+      })
+    )
+
+    await f.step(5)
+
+    expect(f.rows.get('z:content')?.complete).toBe(true)
+    expect(f.saved()).toMatchObject({ complete: false, unsafe: true, resumeAt: null })
+    for (const id of ['a', 'b', 'c']) {
+      expect(f.rows.get(`${id}:content`)).toMatchObject({
+        complete: false,
+        attempts: 1,
+        retryAt: new Date('2026-09-17T01:00:00Z'),
+        failure: { status: 403, reasons: ['notACalendarUser'] },
+      })
+    }
+
+    f.advance(60 * 60 * 1000)
+    f.restart()
+    await f.step(4)
+
+    for (const id of ['a', 'b', 'c']) {
+      expect(f.rows.get(`${id}:content`)).toMatchObject({ complete: true, attempts: 0 })
+      expect(f.rows.get(`${id}:content`)?.failure).toBeUndefined()
+    }
+  })
+
   it('bounds a run of unresolved user errors rather than marking the tenant complete', async () => {
     mocks.directory.mockResolvedValue({ users: ['a', 'b', 'c', 'd'].map(user) })
     const f = fixture()

apps/sim/lib/knowledge/connectors/member-queue.ts

@@ -161,6 +161,7 @@ async function describeUnacceptedMemberSync(
       memberSyncStatus: knowledgeConnector.memberSyncStatus,
       nextMemberSyncAt: knowledgeConnector.nextMemberSyncAt,
       syncLockToken: knowledgeConnector.syncLockToken,
+      memberSyncLockToken: knowledgeConnector.memberSyncLockToken,
       archivedAt: knowledgeConnector.archivedAt,
       deletedAt: knowledgeConnector.deletedAt,
     })
@@ -181,7 +182,14 @@ async function describeUnacceptedMemberSync(
   ) {
     return 'The member sync schedule changed after this run was scheduled'
   }
-  return 'A member sync is already queued or running for this connector'
+  if (row.memberSyncLockToken) {
+    return 'A member sync is already queued or running for this connector'
+  }
+  /**
+   * No checked condition explains the refusal. Naming a cause here instead once hid a connector
+   * that was refused on every attempt for days, because the reason read as ordinary contention.
+   */
+  return 'The connector refused the claim while no lock or status explains it'
 }
 
 /**