From d48d985431417931bca0104ff95938c53a22c1c3 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Tue, 15 Sep 2026 11:33:13 -0700 Subject: [PATCH 01/15] fix(knowledge): remove connections with durable background cleanup (#7854) --- .../[connectorId]/source-detail.test.tsx | 14 +- .../sources/[connectorId]/source-detail.tsx | 9 +- .../connectors-section.test.tsx | 21 +- .../use-connector-actions.ts | 27 +-- .../queries/kb/connectors-cache.test.tsx | 40 ++++ apps/sim/hooks/queries/kb/connectors.ts | 24 +- .../storage-accounting.integration.ts | 142 +++++++++++- .../lib/knowledge/access/predicate.test.ts | 8 +- apps/sim/lib/knowledge/access/predicate.ts | 5 +- apps/sim/lib/knowledge/access/types.ts | 5 +- apps/sim/lib/knowledge/connectors/deletion.md | 13 ++ .../lib/knowledge/connectors/deletion.test.ts | 189 ++++++++++++++++ apps/sim/lib/knowledge/connectors/deletion.ts | 205 ++++++++++++++++++ .../documents/connector-lifecycle.ts | 13 ++ .../documents/processing-outbox-handler.ts | 5 + apps/sim/lib/knowledge/documents/service.ts | 15 +- .../orchestration/connectors.test.ts | 91 +++++++- .../lib/knowledge/orchestration/connectors.ts | 202 +++++++++-------- apps/sim/lib/knowledge/tags/service.test.ts | 46 +++- apps/sim/lib/knowledge/tags/service.ts | 29 ++- 20 files changed, 943 insertions(+), 160 deletions(-) create mode 100644 apps/sim/lib/knowledge/connectors/deletion.md create mode 100644 apps/sim/lib/knowledge/connectors/deletion.test.ts create mode 100644 apps/sim/lib/knowledge/connectors/deletion.ts create mode 100644 apps/sim/lib/knowledge/documents/connector-lifecycle.ts diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx index 0dbbe4341de..d277c6a025d 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx @@ -13,6 +13,7 @@ const mocks = vi.hoisted(() => ({ detail: vi.fn(), integrations: vi.fn(), push: vi.fn(), + replace: vi.fn(), documents: vi.fn(), actions: vi.fn(), recovery: vi.fn(), @@ -23,7 +24,7 @@ const mocks = vi.hoisted(() => ({ save: vi.fn(), })) vi.mock('next/navigation', () => ({ - useRouter: () => ({ push: mocks.push }), + useRouter: () => ({ push: mocks.push, replace: mocks.replace }), usePathname: () => '/o/org-one/settings/integrations/sources/source-one', })) vi.mock('@/app/o/[organizationId]/providers/organization-provider', () => ({ @@ -185,6 +186,17 @@ describe('organization source detail navigation', () => { expect(button, `Missing ${text}`).toBeTruthy() await act(async () => button!.click()) } + it.each(['documents', 'settings', 'history'])( + 'replaces the removed connection with Sources from the %s view', + async (view) => { + await render(`?view=${view}`) + const options: ConnectorActionsOptions = mocks.actions.mock.lastCall![0] + act(() => options.onRemoved?.()) + expect(mocks.replace).toHaveBeenCalledWith('/o/org-one/settings/integrations') + expect(mocks.push).not.toHaveBeenCalled() + } + ) + it('opens documents by default and uses the exact canonical search index', async () => { await render() expect(mocks.detail).toHaveBeenLastCalledWith('index-one', 'source-one') diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.tsx index 939dfd52563..df92230fa62 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.tsx @@ -197,6 +197,8 @@ function SourceDetailContent({ const description = [title === meta?.name ? undefined : meta?.name, status].filter(Boolean).join(' · ') || undefined const onBack = () => router.push(backHref) + const onRemoved = () => + router.replace(organizationRoutes(organization.id).settingsSection('integrations')) const onViewChange = (value: string) => { const next = sourceViewParam.parser.parse(value) if (next) void setView(next) @@ -254,6 +256,7 @@ function SourceDetailContent({ queryError={integrationFeedback} backText={backText} onBack={onBack} + onRemoved={onRemoved} onViewChange={onViewChange} /> ) @@ -264,7 +267,7 @@ function SourceDetailContent({ title={title} description={description} docsLink={meta?.searchDocsUrl} - onRemoved={onBack} + onRemoved={onRemoved} > {integrationFeedback} @@ -367,6 +370,7 @@ interface SourceSettingsEditorProps { queryError?: ReactNode backText: string onBack: () => void + onRemoved: () => void onViewChange: (view: string) => void } @@ -400,6 +404,7 @@ function SourceSettingsForm({ queryError, backText, onBack, + onRemoved, onViewChange, onSaved, onDiscard, @@ -420,7 +425,7 @@ function SourceSettingsForm({ description={description} docsLink={form.docsUrl} lifecycleDisabled={form.dirty || form.saving} - onRemoved={onBack} + onRemoved={onRemoved} actions={saveDiscardActions({ dirty: form.dirty, saving: form.saving, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.test.tsx index b8559b55027..c8574915cbe 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.test.tsx @@ -36,6 +36,7 @@ const { isFetching: false, }, lifecycle: { + removeOptions: { onSuccess: undefined as (() => void) | undefined }, sync: { mutate: vi.fn(), reset: vi.fn(), error: null as Error | null, isPending: false }, update: { mutate: vi.fn(), reset: vi.fn(), error: null as Error | null, isPending: false }, remove: { mutate: vi.fn(), reset: vi.fn(), error: null as Error | null, isPending: false }, @@ -235,7 +236,10 @@ vi.mock('@/hooks/queries/kb/connectors', () => ({ isPlaceholderData: lifecycle.detail.isPlaceholderData, refetch: lifecycle.detail.refetch, })), - useDeleteConnector: () => lifecycle.remove, + useDeleteConnector: (options: { onSuccess: () => void }) => { + lifecycle.removeOptions = options + return lifecycle.remove + }, useTriggerSync: () => lifecycle.sync, useUpdateConnector: () => lifecycle.update, })) @@ -943,15 +947,12 @@ describe('shared connector lifecycle actions', () => { expect(dialog.textContent).not.toContain('remain unless') } act(() => findButton(dialog, 'Remove').click()) - expect(lifecycle.remove.mutate).toHaveBeenCalledWith( - { - knowledgeBaseId: 'knowledge-1', - connectorId: 'connector-1', - deleteDocuments: accessMode !== 'workspace', - }, - expect.any(Object) - ) - act(() => lifecycle.remove.mutate.mock.calls[0][1].onSuccess()) + expect(lifecycle.remove.mutate).toHaveBeenCalledWith({ + knowledgeBaseId: 'knowledge-1', + connectorId: 'connector-1', + deleteDocuments: accessMode !== 'workspace', + }) + act(() => lifecycle.removeOptions.onSuccess?.()) expect(onRemoved).toHaveBeenCalledOnce() expect(container.querySelector('[role="dialog"]')).toBeNull() } diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/use-connector-actions.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/use-connector-actions.ts index 9fbbcda5690..c413b5733b4 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/use-connector-actions.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/use-connector-actions.ts @@ -31,9 +31,15 @@ export function useConnectorActions({ }: ConnectorActionsOptions) { const sync = useTriggerSync() const update = useUpdateConnector() - const remove = useDeleteConnector() const [confirmRemove, setConfirmRemove] = useState(false) const [deleteDocuments, setDeleteDocuments] = useState(false) + const remove = useDeleteConnector({ + onSuccess: () => { + setConfirmRemove(false) + setDeleteDocuments(false) + onRemoved?.() + }, + }) const requiresDocumentDeletion = connector.accessMode !== 'workspace' const state = getConnectorSyncState(connector) const actionsDisabled = disabled || sync.isPending || update.isPending || remove.isPending @@ -117,20 +123,11 @@ export function useConnectorActions({ error: remove.error, onConfirm: () => { if (!canEdit || actionsDisabled) return - remove.mutate( - { - knowledgeBaseId, - connectorId: connector.id, - deleteDocuments: requiresDocumentDeletion || deleteDocuments, - }, - { - onSuccess: () => { - setConfirmRemove(false) - setDeleteDocuments(false) - onRemoved?.() - }, - } - ) + remove.mutate({ + knowledgeBaseId, + connectorId: connector.id, + deleteDocuments: requiresDocumentDeletion || deleteDocuments, + }) }, }, } diff --git a/apps/sim/hooks/queries/kb/connectors-cache.test.tsx b/apps/sim/hooks/queries/kb/connectors-cache.test.tsx index 89ee868601c..72af149fd60 100644 --- a/apps/sim/hooks/queries/kb/connectors-cache.test.tsx +++ b/apps/sim/hooks/queries/kb/connectors-cache.test.tsx @@ -388,6 +388,46 @@ describe('connector Search result cache reconciliation', () => { }) describe('Search source list reconciliation', () => { + it('runs removal navigation before refetches and retains it after the caller unmounts', async () => { + const client = createQueryClient() + const request = Promise.withResolvers() + mocks.requestJson.mockReturnValueOnce(request.promise) + const invalidated = vi.spyOn(client, 'invalidateQueries') + const onSuccess = vi.fn(() => expect(invalidated).not.toHaveBeenCalled()) + const mutation = renderMutation(client, () => useDeleteConnector({ onSuccess })) + let done!: Promise + await act(async () => { + done = mutation().mutateAsync({ + knowledgeBaseId: KNOWLEDGE_BASE_ID, + connectorId: CONNECTOR_ID, + deleteDocuments: true, + }) + }) + act(() => mountedRoots.pop()!.unmount()) + request.resolve({ success: true }) + await act(async () => { + await done + }) + expect(onSuccess).toHaveBeenCalledOnce() + expect(invalidated).toHaveBeenCalledWith({ + queryKey: connectorKeys.detail(KNOWLEDGE_BASE_ID, CONNECTOR_ID), + refetchType: 'none', + }) + }) + + it('does not navigate when removal fails', async () => { + const client = createQueryClient() + const onSuccess = vi.fn() + mocks.requestJson.mockRejectedValueOnce(new Error('Removal failed')) + const mutation = renderMutation(client, () => useDeleteConnector({ onSuccess })) + await act(async () => { + await expect( + mutation().mutateAsync({ knowledgeBaseId: KNOWLEDGE_BASE_ID, connectorId: CONNECTOR_ID }) + ).rejects.toThrow('Removal failed') + }) + expect(onSuccess).not.toHaveBeenCalled() + }) + it('refreshes summaries after editing source configuration or pausing sync', async () => { const queryClient = createQueryClient() const mutation = renderMutation(queryClient, useUpdateConnector) diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 66d5fee637c..fa507b4d24f 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -657,19 +657,37 @@ async function deleteConnector({ }) } -export function useDeleteConnector() { +interface UseDeleteConnectorOptions { + onSuccess?: () => void +} + +export function useDeleteConnector(options?: UseDeleteConnectorOptions) { const queryClient = useQueryClient() return useMutation({ mutationFn: deleteConnector, + /** Run before invalidation can unmount the source page on a 404 response. */ + onSuccess: () => options?.onSuccess?.(), /** * Removing a connector can take its documents with it, so the document * lists and the base's own totals move — but nothing below them does. * Invalidating `knowledgeKeys.detail` as a prefix would also refetch every * cached document detail, chunk page, and chunk search in the base. */ - onSettled: (_data, _error, { knowledgeBaseId, deleteDocuments }) => { - queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + onSettled: (_data, error, { knowledgeBaseId, connectorId, deleteDocuments }) => { + if (error) { + queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + } else { + queryClient.invalidateQueries({ queryKey: connectorKeys.lists(knowledgeBaseId) }) + /** Retire stale detail pages without fetching the just-deleted resource during navigation. */ + void queryClient.cancelQueries({ + queryKey: connectorKeys.detail(knowledgeBaseId, connectorId), + }) + queryClient.invalidateQueries({ + queryKey: connectorKeys.detail(knowledgeBaseId, connectorId), + refetchType: 'none', + }) + } queryClient.invalidateQueries({ queryKey: searchSourceKeys.lists() }) queryClient.invalidateQueries({ queryKey: searchIntegrationKeys.lists() }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentLists(knowledgeBaseId) }) diff --git a/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts b/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts index 8103bfb464a..1d678bd8dc8 100644 --- a/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts @@ -5,20 +5,33 @@ import { promisify } from 'node:util' import { db } from '@sim/db' import { document, + embedding, knowledgeBase, knowledgeConnector, organization, + outboxEvent, user, workspace, } from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { and, eq, inArray, isNull, sql } from 'drizzle-orm' import { afterAll, describe, expect, it, vi } from 'vitest' +import { processOutboxEventById } from '@/lib/core/outbox/service' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { createSingleDocument, hardDeleteDocuments } from '@/lib/knowledge/documents/service' +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' +import { KNOWLEDGE_CONNECTOR_CLEANUP_EVENT } from '@/lib/knowledge/connectors/deletion' +import { createContentSyncLease, SyncLockLostException } from '@/lib/knowledge/connectors/sync-lock' +import { persistSkippedDocuments } from '@/lib/knowledge/connectors/sync-persistence' +import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' +import { + createSingleDocument, + getKnowledgeDocument, + hardDeleteDocuments, +} from '@/lib/knowledge/documents/service' import { performDeleteKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' type Fixture = ReturnType @@ -93,6 +106,14 @@ function disconnect(ids: Fixture, deleteDocuments = false) { afterAll(async () => { for (const ids of fixtures) { + await db + .delete(outboxEvent) + .where( + and( + eq(outboxEvent.eventType, KNOWLEDGE_CONNECTOR_CLEANUP_EVENT), + sql`${outboxEvent.payload}->>'knowledgeBaseId' = ${ids.knowledgeBaseId}` + ) + ) await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) await db.delete(organization).where(eq(organization.id, ids.organizationId)) @@ -176,19 +197,100 @@ describe('knowledge document storage ledgers', () => { expect(await ledger(ids)).toEqual({ workspaceBytes: 0, payerBytes: 0 }) }) - it('deletes a paginated source including archived documents without debiting manual storage', async () => { + it('hides a source immediately and cleans bounded batches without debiting manual storage', async () => { const ids = await seed() await manualDocument(ids, 31) const rows = Array.from({ length: 501 }, (_, index) => sourceDocument(ids, index + 1, index % 2 ? { archivedAt: new Date() } : {}) ) await db.insert(document).values(rows) + await db.insert(embedding).values( + Array.from({ length: 1_001 }, (_, chunkIndex) => ({ + id: generateId(), + knowledgeBaseId: ids.knowledgeBaseId, + documentId: rows[0].id, + chunkIndex, + chunkHash: `hash-${chunkIndex}`, + content: 'test chunk', + contentLength: 10, + tokenCount: 2, + startOffset: 0, + endOffset: 10, + embedding384: Array(384).fill(0.1), + })) + ) expect(await disconnect(ids, true)).toEqual({ success: true, documentsKept: 0, documentsDeleted: 501, }) + const [tombstone] = await db + .select() + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, ids.connectorId)) + expect(tombstone.deletedAt).toBeInstanceOf(Date) + expect(tombstone.status).toBe('disabled') + expect(tombstone.syncLockToken).toBeNull() + const [retained] = await db + .select({ count: sql`COUNT(*)::integer` }) + .from(document) + .where(eq(document.connectorId, ids.connectorId)) + expect(retained.count).toBe(501) + for (const access of [SYSTEM_ACCESS_SCOPE, WORKSPACE_ACCESS_SCOPE]) { + expect(await getKnowledgeDocument(ids.knowledgeBaseId, rows[0].id, access)).toBeNull() + } + const [job] = await db + .select() + .from(outboxEvent) + .where( + and( + eq(outboxEvent.eventType, KNOWLEDGE_CONNECTOR_CLEANUP_EVENT), + sql`${outboxEvent.payload}->>'connectorId' = ${ids.connectorId}` + ) + ) + .limit(1) + expect(job).toBeDefined() + await expect( + persistSkippedDocuments( + ids.knowledgeBaseId, + ids.connectorId, + 'confluence', + [ + { + type: 'skip', + extDoc: { + externalId: generateId(), + title: 'Late source write', + content: '', + mimeType: 'text/plain', + contentHash: 'late-source', + skippedReason: 'Too large', + }, + }, + ], + undefined, + 'workspace', + createContentSyncLease(ids.connectorId, ids.lockId) + ) + ).rejects.toBeInstanceOf(SyncLockLostException) + const handlers = knowledgeDocumentProcessingOutboxHandlers + let status = await processOutboxEventById(job.id, handlers) + expect(status).toBe('pending') + for (let attempt = 0; status === 'pending' && attempt < 5; attempt++) { + await db + .update(outboxEvent) + .set({ availableAt: new Date() }) + .where(eq(outboxEvent.id, job.id)) + status = await processOutboxEventById(job.id, handlers) + } + expect(status).toBe('completed') + expect( + await db + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, ids.connectorId)) + ).toHaveLength(0) const [remaining] = await db .select({ count: sql`COUNT(*)::integer` }) .from(document) @@ -197,6 +299,42 @@ describe('knowledge document storage ledgers', () => { expect(await ledger(ids)).toEqual({ workspaceBytes: 31, payerBytes: 31 }) }) + it('rolls back both the source tombstone and cleanup intent on a failed commit', async () => { + const ids = await seed() + const row = sourceDocument(ids, 10) + await db.insert(document).values(row) + const transaction = db.transaction.bind(db) + const failure = vi.spyOn(db, 'transaction').mockImplementationOnce((callback, config) => + transaction(async (tx) => { + await callback(tx) + throw new Error('Removal transaction failed') + }, config) + ) + try { + expect(await disconnect(ids, true)).toMatchObject({ success: false }) + } finally { + failure.mockRestore() + } + const [source] = await db + .select({ deletedAt: knowledgeConnector.deletedAt }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, ids.connectorId)) + expect(source.deletedAt).toBeNull() + expect( + await getKnowledgeDocument(ids.knowledgeBaseId, row.id, WORKSPACE_ACCESS_SCOPE) + ).not.toBeNull() + const events = await db + .select({ id: outboxEvent.id }) + .from(outboxEvent) + .where( + and( + eq(outboxEvent.eventType, KNOWLEDGE_CONNECTOR_CLEANUP_EVENT), + sql`${outboxEvent.payload}->>'connectorId' = ${ids.connectorId}` + ) + ) + expect(events).toHaveLength(0) + }) + it('keeps the source and every document attached when detachment exceeds the quota', async () => { const ids = await seed() const rows = [sourceDocument(ids, 800_000_000), sourceDocument(ids, 800_000_000)] diff --git a/apps/sim/lib/knowledge/access/predicate.test.ts b/apps/sim/lib/knowledge/access/predicate.test.ts index 2ff74b38a51..11582cc360e 100644 --- a/apps/sim/lib/knowledge/access/predicate.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.test.ts @@ -77,7 +77,11 @@ describe('knowledgeAccessCondition', () => { ) }) - it('exempts only the branded system scope', () => { - expect(render(knowledgeAccessCondition(SYSTEM_ACCESS_SCOPE)).sql).toBe('true') + it('exempts system jobs from ACL checks while refusing removed sources', () => { + const { sql } = render(knowledgeAccessCondition(SYSTEM_ACCESS_SCOPE)) + expect(sql).toContain('"document"."connector_id" IS NULL OR EXISTS') + expect(sql).toContain('"knowledge_connector"."deleted_at" IS NULL') + expect(sql).toContain('"knowledge_connector"."archived_at" IS NULL') + expect(sql).not.toContain('"document"."acl"') }) }) diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts index 21a653271fb..bbce2808f25 100644 --- a/apps/sim/lib/knowledge/access/predicate.ts +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -16,6 +16,7 @@ import { type SQL, sql } from 'drizzle-orm' import { EXTERNAL_GROUP_STALE_AFTER_MS } from '@/lib/knowledge/access/external-groups' import { SOURCE_ACL_MAX_AGE_MS } from '@/lib/knowledge/access/freshness' import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' +import { documentConnectorIsActive } from '@/lib/knowledge/documents/connector-lifecycle' import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' @@ -202,7 +203,7 @@ function storedKnowledgeAccessCondition( scope: KnowledgeAccessScope | SystemAccessScope, liveSourceAccess: SQL ): SQL { - if (scope.kind === 'system') return sql`true` + if (scope.kind === 'system') return documentConnectorIsActive() if (scope.tokens.length === 0) return sql`false` const tokens = textArrayLiteral(scope.tokens) const cutoff = sql`statement_timestamp() - (${SOURCE_ACL_MAX_AGE_MS} * interval '1 millisecond')` @@ -217,6 +218,8 @@ function storedKnowledgeAccessCondition( OR EXISTS ( SELECT 1 FROM ${knowledgeConnector} WHERE ${knowledgeConnector.id} = ${document.connectorId} + AND ${knowledgeConnector.deletedAt} IS NULL + AND ${knowledgeConnector.archivedAt} IS NULL AND ${knowledgeConnector.accessRewritePending} = false AND ${searchIntegrationAccessCondition()} AND ${liveSourceAccess} diff --git a/apps/sim/lib/knowledge/access/types.ts b/apps/sim/lib/knowledge/access/types.ts index d4db3abd081..1ad91607e41 100644 --- a/apps/sim/lib/knowledge/access/types.ts +++ b/apps/sim/lib/knowledge/access/types.ts @@ -110,8 +110,9 @@ export const MAX_KNOWLEDGE_ACCESS_CANDIDATES = 400 declare const systemAccessScopeBrand: unique symbol /** - * The one exemption from access filtering: a background job acting on rows it - * owns (document processing, connector sync). It is a branded type so it cannot + * The exemption from ACL filtering for a background job acting on rows it + * owns (document processing, connector sync). Removed sources remain inaccessible. + * It is a branded type so it cannot * be assembled from a literal, and this module is its only source, so every * caller is one grep away. Never construct it on a request path. */ diff --git a/apps/sim/lib/knowledge/connectors/deletion.md b/apps/sim/lib/knowledge/connectors/deletion.md new file mode 100644 index 00000000000..826ea5dbecf --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/deletion.md @@ -0,0 +1,13 @@ +# Connection removal + +All existing UI, API, and Copilot callers still enter `knowledge.connectors.delete` through its authorized application use case. Roles, scope checks, response shapes, audit attribution, and the workspace-only option to retain documents are unchanged. + +When documents are removed, a transaction locks the canonical knowledge base and connector, counts the attached documents, marks the connector deleted, invalidates both sync leases, disables scheduling, and inserts one `knowledge.connector.cleanup` outbox event. Failure rolls back both the deletion and the event. Returned deletion counts describe documents logically removed from Sim; physical deletion follows asynchronously. The keep-documents path still performs its storage quota check and detachment atomically. + +The shared document access predicate excludes deleted connectors, including public and workspace access and internal indexing reads. Ingestion also checks connector liveness before claiming or committing document processing. Existing source-write leases reject late sync writes. Documents keep their connector reference until physical deletion, so they cannot become standalone readable or billable documents during cleanup. Restoring a knowledge base does not restore a directly deleted connector. + +The existing outbox worker runs cleanup, with 48 failure attempts and bounded continuations that do not consume that retry budget. Each transaction removes at most 1,000 chunks, 250 documents, or 1,000 sync-history/member rows. A run does at most four batches and yields after its time budget. Transactions use lock and statement timeouts. The worker verifies the connector's deletion timestamp, locks documents against late indexing commits, and commits storage cleanup intents before deleting those documents. It resolves storage ownership from the currently locked knowledge base. Already committed batches survive worker restarts. Failures remain retryable; exhausted events remain as dead letters while the source stays inaccessible. + +After document and connector cleanup, credential grant revocation and unused-tag cleanup are retried as needed. Tag cleanup checks for existence rather than counting the entire remaining corpus. No provider credentials or document contents enter the connector cleanup payload. + +The mutation's success handler navigates before cache invalidation and survives the source component unmounting. Source detail replaces its history entry with the Sources page from Documents, Settings, or Sync history; failed removal stays on the current page with the error. diff --git a/apps/sim/lib/knowledge/connectors/deletion.test.ts b/apps/sim/lib/knowledge/connectors/deletion.test.ts new file mode 100644 index 00000000000..d885a707a0b --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/deletion.test.ts @@ -0,0 +1,189 @@ +/** @vitest-environment node */ +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeBase, + knowledgeConnector, + knowledgeConnectorMember, + knowledgeConnectorMemberSyncLog, + knowledgeConnectorSyncLog, +} from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { OutboxEventContext } from '@/lib/core/outbox/service' + +const mocks = vi.hoisted(() => ({ storage: vi.fn(), tags: vi.fn(), revoke: vi.fn() })) +vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ + enqueueKnowledgeStorageCleanup: mocks.storage, +})) +vi.mock('@/lib/knowledge/tags/service', () => ({ cleanupUnusedTagDefinitions: mocks.tags })) +vi.mock('@/lib/knowledge/connectors/member-access', () => ({ + revokeKnowledgeConnectorCredentialAccess: mocks.revoke, +})) + +import { + cleanupKnowledgeConnector, + enqueueConnectorDeletion, + KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, +} from '@/lib/knowledge/connectors/deletion' + +const payload = { + version: 1 as const, + knowledgeBaseId: 'kb-1', + connectorId: 'connector-1', + deletedAt: '2026-09-15T12:00:00.000Z', + credentialAccess: { workspaceId: 'ws-1', credentialGroupId: 'group-1', actorUserId: 'user-1' }, +} +const owner = { workspaceId: 'ws-1', organizationId: null, userId: 'user-1' } + +function context(): OutboxEventContext { + return { + eventId: 'event-1', + eventType: KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, + attempts: 0, + maxAttempts: 48, + signal: new AbortController().signal, + checkpointPayload: vi.fn(), + } +} + +function queueBatch(docs: { id: string; fileUrl: string }[], chunks: { id: string }[] = []) { + queueTableRows(knowledgeBase, [owner]) + queueTableRows(knowledgeConnector, [{ deletedAt: new Date(payload.deletedAt) }]) + queueTableRows(document, docs) + if (docs.length) queueTableRows(embedding, chunks) +} + +describe('durable connector cleanup', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.storage.mockResolvedValue([]) + mocks.tags.mockResolvedValue(0) + mocks.revoke.mockResolvedValue(undefined) + }) + afterEach(resetDbChainMock) + + it('enqueues a bounded immutable identity with a retry budget', async () => { + await enqueueConnectorDeletion(db, payload) + expect(dbChainMockFns.values).toHaveBeenCalledWith( + expect.objectContaining({ + eventType: KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, + payload, + maxAttempts: 48, + }) + ) + }) + + it('preserves storage cleanup intent before removing documents and then the connector', async () => { + const docs = [{ id: 'doc-1', fileUrl: '/file.txt' }] + queueBatch(docs) + queueBatch([]) + await cleanupKnowledgeConnector(payload, context()) + expect(mocks.storage).toHaveBeenCalledWith( + expect.anything(), + [{ ...docs[0], ...owner }], + 'event-1' + ) + expect(mocks.storage.mock.invocationCallOrder[0]).toBeLessThan( + dbChainMockFns.delete.mock.invocationCallOrder[0] + ) + expect(dbChainMockFns.delete.mock.calls.map(([table]) => table)).toEqual([ + document, + knowledgeConnector, + ]) + expect(mocks.revoke).toHaveBeenCalledWith( + { + workspaceId: 'ws-1', + credentialGroupId: 'group-1', + connectorId: 'connector-1', + }, + 'user-1' + ) + expect(mocks.tags).toHaveBeenCalledOnce() + }) + + it('yields after four bounded chunk batches without spending the failure retry budget', async () => { + const docs = Array.from({ length: 250 }, (_, index) => ({ id: `doc-${index}`, fileUrl: '' })) + const chunks = Array.from({ length: 1000 }, (_, index) => ({ id: `chunk-${index}` })) + for (let batch = 0; batch < 4; batch++) queueBatch(docs, chunks) + expect(await cleanupKnowledgeConnector(payload, context())).toMatchObject({ + outcome: 'deferred', + consumeAttempt: false, + }) + expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(4) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(250) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(1000) + expect(dbChainMockFns.delete.mock.calls.map(([table]) => table)).toEqual( + Array(4).fill(embedding) + ) + expect(mocks.storage).not.toHaveBeenCalled() + expect(mocks.revoke).not.toHaveBeenCalled() + }) + + it('keeps the documents when storage cleanup intent cannot be persisted', async () => { + queueBatch([{ id: 'doc-1', fileUrl: '/file.txt' }]) + mocks.storage.mockRejectedValueOnce(new Error('Outbox unavailable')) + await expect(cleanupKnowledgeConnector(payload, context())).rejects.toThrow( + 'Outbox unavailable' + ) + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + expect(mocks.tags).not.toHaveBeenCalled() + }) + + it.each([knowledgeConnectorSyncLog, knowledgeConnectorMemberSyncLog, knowledgeConnectorMember])( + 'drains related rows before deleting their connector', + async (table) => { + const rows = Array.from({ length: 1000 }, (_, index) => ({ id: `row-${index}` })) + for (let batch = 0; batch < 4; batch++) { + queueBatch([]) + queueTableRows(table, rows) + } + expect(await cleanupKnowledgeConnector(payload, context())).toMatchObject({ + outcome: 'deferred', + consumeAttempt: false, + }) + expect(dbChainMockFns.delete.mock.calls.map(([target]) => target)).toEqual( + Array(4).fill(table) + ) + expect(mocks.revoke).not.toHaveBeenCalled() + expect(mocks.tags).not.toHaveBeenCalled() + } + ) + + it.each([null, new Date('2026-09-14T12:00:00.000Z')])( + 'leaves a connector with a different deletion generation untouched: %s', + async (deletedAt) => { + queueTableRows(knowledgeBase, [owner]) + queueTableRows(knowledgeConnector, [{ deletedAt }]) + await cleanupKnowledgeConnector(payload, context()) + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + expect(mocks.storage).not.toHaveBeenCalled() + expect(mocks.revoke).not.toHaveBeenCalled() + } + ) + + it('retries final effects after the connector deletion already committed', async () => { + queueBatch([]) + mocks.tags.mockRejectedValueOnce(new Error('Temporary tag failure')) + await expect(cleanupKnowledgeConnector(payload, context())).rejects.toThrow( + 'Temporary tag failure' + ) + resetDbChainMock() + queueTableRows(knowledgeBase, [owner]) + queueTableRows(knowledgeConnector, []) + await cleanupKnowledgeConnector(payload, context()) + expect(mocks.tags).toHaveBeenCalledTimes(2) + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + }) + + it('does no work after cancellation', async () => { + const controller = new AbortController() + controller.abort() + await expect( + cleanupKnowledgeConnector(payload, { ...context(), signal: controller.signal }) + ).rejects.toThrow() + expect(dbChainMockFns.transaction).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/deletion.ts b/apps/sim/lib/knowledge/connectors/deletion.ts new file mode 100644 index 00000000000..b95d27c6fc7 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/deletion.ts @@ -0,0 +1,205 @@ +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeBase, + knowledgeConnector, + knowledgeConnectorMember, + knowledgeConnectorMemberSyncLog, + knowledgeConnectorSyncLog, +} from '@sim/db/schema' +import { and, eq, inArray, sql } from 'drizzle-orm' +import { z } from 'zod' +import { + continueOutboxHandler, + enqueueOutboxEvent, + type OutboxHandler, +} from '@/lib/core/outbox/service' +import type { DbOrTx } from '@/lib/db/types' +import { revokeKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' +import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup' +import { cleanupUnusedTagDefinitions } from '@/lib/knowledge/tags/service' + +export const KNOWLEDGE_CONNECTOR_CLEANUP_EVENT = 'knowledge.connector.cleanup' +const DOCUMENT_BATCH_SIZE = 250 +const EMBEDDING_BATCH_SIZE = 1_000 +const RELATED_ROW_BATCH_SIZE = 1_000 +const MAX_BATCHES_PER_RUN = 4 +const RUN_BUDGET_MS = 30_000 + +const deletionPayloadSchema = z + .object({ + version: z.literal(1), + knowledgeBaseId: z.string().min(1).max(256), + connectorId: z.string().min(1).max(256), + deletedAt: z.iso.datetime(), + credentialAccess: z + .object({ + workspaceId: z.string().min(1).max(256), + credentialGroupId: z.string().min(1).max(256), + actorUserId: z.string().min(1).max(256), + }) + .optional(), + }) + .strict() + +type ConnectorDeletionPayload = z.infer + +/** The tombstone and its durable cleanup intent must commit together. */ +export async function enqueueConnectorDeletion( + tx: DbOrTx, + payload: Omit +): Promise { + await enqueueOutboxEvent( + tx, + KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, + deletionPayloadSchema.parse({ version: 1, ...payload }), + { maxAttempts: 48 } + ) +} + +/** + * Each transaction releases at most 250 documents or 1,000 chunks. Locks prevent a late + * indexing commit from inserting chunks between the final chunk scan and document deletion. + * The connector stays attached until every document is gone, preserving storage accounting + * and preventing removed content from becoming a standalone workspace document. + */ +export const cleanupKnowledgeConnector: OutboxHandler = async (rawPayload, context) => { + const payload = deletionPayloadSchema.parse(rawPayload) + const deadline = Math.min( + Date.now() + RUN_BUDGET_MS, + context.deadlineAt ?? Number.POSITIVE_INFINITY + ) + for (let batch = 0; batch < MAX_BATCHES_PER_RUN; batch++) { + context.signal.throwIfAborted() + const outcome = await db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL lock_timeout = '5s'`) + await tx.execute(sql`SET LOCAL statement_timeout = '10s'`) + const [owner] = await tx + .select({ + workspaceId: knowledgeBase.workspaceId, + organizationId: knowledgeBase.organizationId, + userId: knowledgeBase.userId, + }) + .from(knowledgeBase) + .where(eq(knowledgeBase.id, payload.knowledgeBaseId)) + .for('share') + .limit(1) + if (!owner) return 'complete' + const [connector] = await tx + .select({ deletedAt: knowledgeConnector.deletedAt }) + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, payload.connectorId), + eq(knowledgeConnector.knowledgeBaseId, payload.knowledgeBaseId) + ) + ) + .for('update') + .limit(1) + if (!connector) return 'complete' + if (connector.deletedAt?.toISOString() !== payload.deletedAt) return 'obsolete' + + /** Drain the indexed connector bucket; sorting the whole remaining corpus on every batch is unnecessary. */ + const docs = await tx + .select({ id: document.id, fileUrl: document.fileUrl }) + .from(document) + .where( + and( + eq(document.connectorId, payload.connectorId), + eq(document.knowledgeBaseId, payload.knowledgeBaseId) + ) + ) + .limit(DOCUMENT_BATCH_SIZE) + .for('update') + context.signal.throwIfAborted() + if (docs.length === 0) { + for (const table of [ + knowledgeConnectorSyncLog, + knowledgeConnectorMemberSyncLog, + knowledgeConnectorMember, + ]) { + const rows = await tx + .select({ id: table.id }) + .from(table) + .where(eq(table.connectorId, payload.connectorId)) + .limit(RELATED_ROW_BATCH_SIZE) + if (rows.length === 0) continue + await tx.delete(table).where( + inArray( + table.id, + rows.map(({ id }) => id) + ) + ) + context.signal.throwIfAborted() + return 'progress' + } + await tx + .delete(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, payload.connectorId), + eq(knowledgeConnector.knowledgeBaseId, payload.knowledgeBaseId), + eq(knowledgeConnector.deletedAt, new Date(payload.deletedAt)) + ) + ) + return 'complete' + } + const documentIds = docs.map(({ id }) => id) + const chunks = await tx + .select({ id: embedding.id }) + .from(embedding) + .where(inArray(embedding.documentId, documentIds)) + .limit(EMBEDDING_BATCH_SIZE) + if (chunks.length > 0) { + await tx.delete(embedding).where( + inArray( + embedding.id, + chunks.map(({ id }) => id) + ) + ) + } else { + await enqueueKnowledgeStorageCleanup( + tx, + docs.map((doc) => ({ ...doc, ...owner })), + context.eventId + ) + await tx.delete(document).where(inArray(document.id, documentIds)) + } + context.signal.throwIfAborted() + return 'progress' + }) + if (outcome === 'obsolete') return + if (outcome === 'complete') { + context.signal.throwIfAborted() + if (payload.credentialAccess) { + await revokeKnowledgeConnectorCredentialAccess( + { + workspaceId: payload.credentialAccess.workspaceId, + credentialGroupId: payload.credentialAccess.credentialGroupId, + connectorId: payload.connectorId, + }, + payload.credentialAccess.actorUserId + ) + } + context.signal.throwIfAborted() + await db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL lock_timeout = '5s'`) + await tx.execute(sql`SET LOCAL statement_timeout = '10s'`) + await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where(eq(knowledgeBase.id, payload.knowledgeBaseId)) + .for('no key update') + await cleanupUnusedTagDefinitions(payload.knowledgeBaseId, context.eventId, { + executor: tx, + signal: context.signal, + }) + context.signal.throwIfAborted() + }) + return + } + if (Date.now() >= deadline) break + } + return continueOutboxHandler('Connector cleanup committed a bounded batch', 1_000) +} diff --git a/apps/sim/lib/knowledge/documents/connector-lifecycle.ts b/apps/sim/lib/knowledge/documents/connector-lifecycle.ts new file mode 100644 index 00000000000..747b8ff9ee7 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/connector-lifecycle.ts @@ -0,0 +1,13 @@ +import { document, knowledgeConnector } from '@sim/db/schema' +import { sql } from 'drizzle-orm' + +/** A removed source hides its documents immediately, before permanent cleanup catches up. */ +export function documentConnectorIsActive() { + return sql`(${document.connectorId} IS NULL OR EXISTS ( + SELECT 1 FROM ${knowledgeConnector} + WHERE ${knowledgeConnector.id} = ${document.connectorId} + AND ${knowledgeConnector.knowledgeBaseId} = ${document.knowledgeBaseId} + AND ${knowledgeConnector.deletedAt} IS NULL + AND ${knowledgeConnector.archivedAt} IS NULL + ))` +} diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts index e1ee59af466..eef0ceb2a25 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts @@ -7,6 +7,10 @@ import { } from '@/lib/core/outbox/service' import { isBYOKEmbeddingCredentialRejection, isEmbeddingQuotaExhaustion } from '@/lib/embeddings' import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' +import { + cleanupKnowledgeConnector, + KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, +} from '@/lib/knowledge/connectors/deletion' import { getOcrRequestRejection, isPermanentDocumentProcessingError, @@ -228,6 +232,7 @@ const KNOWLEDGE_HANDLER_TIMEOUT_MS = Math.min( ) export const knowledgeDocumentProcessingOutboxHandlers = { + [KNOWLEDGE_CONNECTOR_CLEANUP_EVENT]: cleanupKnowledgeConnector, [KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanupKnowledgeStorage, [OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT]: cleanupOcrCheckpoint, [EMBEDDING_CHECKPOINT_CLEANUP_EVENT]: cleanupEmbeddingCheckpoint, diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 188a5c0308d..4b5d9cf2761 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -81,6 +81,7 @@ import { SYSTEM_ACCESS_SCOPE, } from '@/lib/knowledge/access/types' import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' +import { documentConnectorIsActive } from '@/lib/knowledge/documents/connector-lifecycle' import { assertDocumentChunkCountWithinLimit, getOcrRequestRejection, @@ -1506,6 +1507,7 @@ export async function processDocumentAsync( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + documentConnectorIsActive(), isNull(knowledgeBase.deletedAt) ) ) @@ -1533,7 +1535,8 @@ export async function processDocumentAsync( ...queueGenerationConditions(attemptContext), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + documentConnectorIsActive() ) ) return @@ -1601,7 +1604,8 @@ export async function processDocumentAsync( : queueGenerationConditions(attemptContext)), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + documentConnectorIsActive() ) ) .returning({ id: document.id }) @@ -1861,6 +1865,7 @@ export async function processDocumentAsync( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + documentConnectorIsActive(), isNull(knowledgeBase.deletedAt) ) ) @@ -1930,7 +1935,8 @@ export async function processDocumentAsync( ...queueGenerationConditions(attemptContext), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + documentConnectorIsActive() ) ) signal.throwIfAborted() @@ -2124,7 +2130,8 @@ export async function processDocumentAsync( ...queueGenerationConditions(attemptContext), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + documentConnectorIsActive() ) ) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index 0f4783471ac..2a08a2bc246 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -25,6 +25,7 @@ const { mockResolveStorageBillingContext, mockIncrementStorage, mockNotifyStorage, + mockEnqueueConnectorDeletion, } = vi.hoisted(() => ({ mockCaptureServerEvent: vi.fn(), mockDispatchSync: vi.fn(), @@ -38,6 +39,7 @@ const { mockResolveStorageBillingContext: vi.fn(), mockIncrementStorage: vi.fn(), mockNotifyStorage: vi.fn(), + mockEnqueueConnectorDeletion: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -63,6 +65,9 @@ vi.mock('@/lib/billing/storage', () => ({ vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ enqueueKnowledgeStorageCleanup: vi.fn().mockResolvedValue(undefined), })) +vi.mock('@/lib/knowledge/connectors/deletion', () => ({ + enqueueConnectorDeletion: mockEnqueueConnectorDeletion, +})) vi.mock('@/lib/knowledge/connectors/queue', () => ({ dispatchSync: mockDispatchSync })) vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ dispatchMemberSync: mockDispatchMemberSync, @@ -296,11 +301,11 @@ const STORAGE_CONTEXT = { customStorageLimitGB: null, } -function queueConnectorDeletionOwnerAndLock(accessMode = 'workspace') { +function queueConnectorDeletionOwnerAndLock(accessMode = 'workspace', credentialGroupId?: string) { const owner = { id: 'kb-1', workspaceId: 'ws-1', organizationId: null, userId: 'user-1' } queueTableRows(schemaMock.knowledgeBase, [owner]) queueTableRows(schemaMock.knowledgeBase, [owner]) - queueTableRows(schemaMock.knowledgeConnector, [{ accessMode }]) + queueTableRows(schemaMock.knowledgeConnector, [{ accessMode, credentialGroupId }]) } describe('performDeleteKnowledgeConnector', () => { @@ -340,11 +345,11 @@ describe('performDeleteKnowledgeConnector', () => { ) }) - it('reports the documents it deleted when asked to delete them', async () => { + it('hides the connector and queues cleanup without deleting documents in the request', async () => { dbChainMockFns.limit.mockResolvedValueOnce([ { id: 'conn-1', connectorType: 'notion', accessMode: 'workspace' }, ]) - queueTableRows(document, [{ id: 'doc-1', fileUrl: '/a.txt' }]) + queueTableRows(document, [{ count: 501 }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-1' }]) const outcome = await performDeleteKnowledgeConnector({ @@ -354,9 +359,67 @@ describe('performDeleteKnowledgeConnector', () => { deleteDocuments: true, }) - expect(outcome).toMatchObject({ success: true, documentsDeleted: 1, documentsKept: 0 }) + expect(outcome).toMatchObject({ success: true, documentsDeleted: 501, documentsKept: 0 }) + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + expect(dbChainMockFns.update).not.toHaveBeenCalledWith(document) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ + deletedAt: expect.any(Date), + status: 'disabled', + memberSyncStatus: 'disabled', + syncLockToken: null, + memberSyncLockToken: null, + }) + ) + expect(mockEnqueueConnectorDeletion).toHaveBeenCalledWith(expect.anything(), { + knowledgeBaseId: KB.id, + connectorId: 'conn-1', + deletedAt: expect.any(String), + }) }) + it('fails the transaction without auditing success when cleanup cannot be queued', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', accessMode: 'workspace' }, + ]) + queueTableRows(document, [{ count: 2 }]) + mockEnqueueConnectorDeletion.mockRejectedValueOnce(new Error('Queue unavailable')) + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + deleteDocuments: true, + }) + expect(outcome).toMatchObject({ success: false }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) + + it.each(['55P03', '57014', '40P01'])( + 'returns a retryable message for transaction contention %s', + async (code) => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', accessMode: 'workspace' }, + ]) + dbChainMockFns.transaction.mockRejectedValueOnce( + Object.assign(new Error('Database detail'), { code }) + ) + expect( + await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + deleteDocuments: true, + }) + ).toMatchObject({ + success: false, + errorCode: 'conflict', + error: 'Connection is busy. Try removing it again in a moment.', + }) + expect(mockRecordAudit).not.toHaveBeenCalled() + } + ) + it('reports a missing connector as not found', async () => { dbChainMockFns.limit.mockResolvedValueOnce([]) @@ -1256,9 +1319,9 @@ describe('members-mode connectors', () => { expect(dbChainMockFns.delete).not.toHaveBeenCalled() }) - it('revokes the credential grant once the connector and its documents are gone', async () => { + it('defers credential grant cleanup with the connector deletion', async () => { queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) - queueConnectorDeletionOwnerAndLock('members') + queueConnectorDeletionOwnerAndLock('members', 'group-1') queueTableRows(schemaMock.document, []) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) @@ -1270,9 +1333,17 @@ describe('members-mode connectors', () => { }) expect(outcome).toMatchObject({ success: true }) - expect(mockRevoke).toHaveBeenCalledWith( - { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, - 'user-1' + expect(mockRevoke).not.toHaveBeenCalled() + expect(mockEnqueueConnectorDeletion).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + connectorId: 'c-1', + credentialAccess: { + workspaceId: 'ws-1', + credentialGroupId: 'group-1', + actorUserId: ACTOR.userId, + }, + }) ) }) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index e7985025438..f3ee6da4858 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -3,15 +3,15 @@ import { db } from '@sim/db' import { credentialGroup, document, - embedding, knowledgeBase, knowledgeBaseTagDefinitions, knowledgeConnector, knowledgeConnectorMember, } from '@sim/db/schema' import { createLogger } from '@sim/logger' +import { getPostgresErrorCode } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, asc, eq, gt, inArray, isNull, sql } from 'drizzle-orm' +import { and, eq, isNull, sql } from 'drizzle-orm' import { encryptApiKey } from '@/lib/api-key/crypto' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { @@ -44,6 +44,7 @@ import { type ConnectorAccessToken, syncContextForToken, } from '@/lib/knowledge/connectors/access-token' +import { enqueueConnectorDeletion } from '@/lib/knowledge/connectors/deletion' import { findListingCapViolation, grantKnowledgeConnectorCredentialAccess, @@ -52,7 +53,6 @@ import { } from '@/lib/knowledge/connectors/member-access' import type { PreparedConnectorPermissions } from '@/lib/knowledge/connectors/permission-config' import { allocateTagSlots } from '@/lib/knowledge/constants' -import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup' import { auditActorFields, classifyKnowledgeFailure, @@ -60,7 +60,7 @@ import { type KnowledgeOperationContext, type KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' -import { cleanupUnusedTagDefinitions, createTagDefinition } from '@/lib/knowledge/tags/service' +import { createTagDefinition } from '@/lib/knowledge/tags/service' import { captureServerEvent } from '@/lib/posthog/server' import { searchSourceIdentity } from '@/lib/sim-search/source-identity' import { getConnectorApiKeyConfig } from '@/connectors/auth' @@ -1077,8 +1077,8 @@ export interface PerformDeleteKnowledgeConnectorParams extends KnowledgeOperatio knowledgeBase: ConnectorKnowledgeBase connectorId: string /** - * Also hard-delete the documents the connector produced. Defaults to keeping - * them, which turns them into ordinary standalone knowledge base entries. + * Immediately hide the connector's documents and durably queue permanent cleanup. + * Defaults to keeping them as ordinary standalone knowledge base entries. */ deleteDocuments?: boolean /** False only when an authorized application use case projects the semantic audit. */ @@ -1094,8 +1094,8 @@ export type PerformDeleteKnowledgeConnectorResult = KnowledgeOrchestrationResult }> /** - * Hard-deletes a connector, either removing the documents it produced or - * releasing them as standalone entries. + * Removes a connector, either tombstoning it for bounded background cleanup or + * releasing its documents as standalone entries before deleting the connector. * * Returns the counts so callers state what happened rather than assert it. The * copilot tool used to reach this through an internal HTTP self-call that sent @@ -1153,6 +1153,8 @@ export async function performDeleteKnowledgeConnector( : undefined docCount = await db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL lock_timeout = '5s'`) + await tx.execute(sql`SET LOCAL statement_timeout = '10s'`) /** Match source writes and document deletion: parent KB, connector, then storage ledgers. */ const [lockedOwner] = await tx .select({ @@ -1162,7 +1164,7 @@ export async function performDeleteKnowledgeConnector( }) .from(knowledgeBase) .where(and(eq(knowledgeBase.id, kb.id), isNull(knowledgeBase.deletedAt))) - .for('update') + .for(deleteDocuments ? 'share' : 'update') .limit(1) if ( !lockedOwner || @@ -1173,7 +1175,10 @@ export async function performDeleteKnowledgeConnector( throw new OrchestrationError('conflict', 'Knowledge base ownership changed; retry deletion') } const [lockedConnector] = await tx - .select({ accessMode: knowledgeConnector.accessMode }) + .select({ + accessMode: knowledgeConnector.accessMode, + credentialGroupId: knowledgeConnector.credentialGroupId, + }) .from(knowledgeConnector) .where( and( @@ -1193,93 +1198,99 @@ export async function performDeleteKnowledgeConnector( ) } - let count = 0 if (deleteDocuments) { - let afterId: string | undefined - for (;;) { - /** Archived rows also lose their connector FK and must not escape deletion or cleanup. */ - const docs = await tx - .select({ id: document.id, fileUrl: document.fileUrl }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - eq(document.knowledgeBaseId, kb.id), - afterId ? gt(document.id, afterId) : undefined - ) - ) - .orderBy(asc(document.id)) - .limit(250) - if (docs.length === 0) break - const documentIds = docs.map((doc) => doc.id) - await tx.delete(embedding).where(inArray(embedding.documentId, documentIds)) - await tx.delete(document).where(inArray(document.id, documentIds)) - await enqueueKnowledgeStorageCleanup( - tx, - docs.map((doc) => ({ - ...doc, - workspaceId: owner.workspaceId, - organizationId: owner.organizationId, - userId: owner.userId, - })), - requestId - ) - count += docs.length - afterId = docs.at(-1)?.id - } - } else { - /** Legacy skipped rows used remote size despite retaining no artifact. */ - await tx - .update(document) - .set({ fileSize: 0 }) - .where( - and( - eq(document.connectorId, connectorId), - eq(document.knowledgeBaseId, kb.id), - isNull(document.storageKey), - eq(document.fileUrl, '') - ) - ) - /** - * Connector bytes are unmetered until detachment. Count retained archived files too; - * live tombstones are resurrected below, while archived tombstones remain nonbillable. - */ const [totals] = await tx - .select({ - count: sql`COUNT(*)::integer`, - bytes: sql`COALESCE(SUM(${document.fileSize}::bigint) FILTER ( - WHERE ${document.archivedAt} IS NULL OR ${document.deletedAt} IS NULL - ), 0)::text`, - }) + .select({ count: sql`COUNT(*)::integer` }) .from(document) .where(and(eq(document.connectorId, connectorId), eq(document.knowledgeBaseId, kb.id))) - count = totals?.count ?? 0 - const retainedBytes = Number(totals?.bytes ?? 0) - if (!Number.isSafeInteger(retainedBytes) || retainedBytes < 0) { - throw new Error('Invalid retained connector storage size') - } - if (retainedBytes > 0) { - if (storageContext) { - const updatedUsage = await incrementStorageUsageForBillingContextInTx( - tx, - storageContext, - retainedBytes - ) - if (updatedUsage !== undefined) - storageNotification = { context: storageContext, updatedUsage } - } - } + const deletedAt = new Date() await tx - .update(document) - .set({ deletedAt: null }) + .update(knowledgeConnector) + .set({ + deletedAt, + updatedAt: deletedAt, + status: 'disabled', + memberSyncStatus: 'disabled', + syncLockToken: null, + syncLockLeaseAt: null, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + nextSyncAt: null, + nextMemberSyncAt: null, + }) .where( and( - eq(document.connectorId, connectorId), - eq(document.knowledgeBaseId, kb.id), - isNull(document.archivedAt) + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, kb.id) ) ) + await enqueueConnectorDeletion(tx, { + knowledgeBaseId: kb.id, + connectorId, + deletedAt: deletedAt.toISOString(), + ...(lockedConnector.credentialGroupId && owner.workspaceId + ? { + credentialAccess: { + workspaceId: owner.workspaceId, + credentialGroupId: lockedConnector.credentialGroupId, + actorUserId: params.userId, + }, + } + : {}), + }) + return totals?.count ?? 0 + } + /** Legacy skipped rows used remote size despite retaining no artifact. */ + await tx + .update(document) + .set({ fileSize: 0 }) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.knowledgeBaseId, kb.id), + isNull(document.storageKey), + eq(document.fileUrl, '') + ) + ) + /** + * Connector bytes are unmetered until detachment. Count retained archived files too; + * live tombstones are resurrected below, while archived tombstones remain nonbillable. + */ + const [totals] = await tx + .select({ + count: sql`COUNT(*)::integer`, + bytes: sql`COALESCE(SUM(${document.fileSize}::bigint) FILTER ( + WHERE ${document.archivedAt} IS NULL OR ${document.deletedAt} IS NULL + ), 0)::text`, + }) + .from(document) + .where(and(eq(document.connectorId, connectorId), eq(document.knowledgeBaseId, kb.id))) + const count = totals?.count ?? 0 + const retainedBytes = Number(totals?.bytes ?? 0) + if (!Number.isSafeInteger(retainedBytes) || retainedBytes < 0) { + throw new Error('Invalid retained connector storage size') } + if (retainedBytes > 0) { + if (storageContext) { + const updatedUsage = await incrementStorageUsageForBillingContextInTx( + tx, + storageContext, + retainedBytes + ) + if (updatedUsage !== undefined) + storageNotification = { context: storageContext, updatedUsage } + } + } + await tx + .update(document) + .set({ deletedAt: null }) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.knowledgeBaseId, kb.id), + isNull(document.archivedAt) + ) + ) const deletedConnectors = await tx .delete(knowledgeConnector) @@ -1298,6 +1309,13 @@ export async function performDeleteKnowledgeConnector( return count }) } catch (error) { + if (['55P03', '57014', '40P01'].includes(getPostgresErrorCode(error) ?? '')) { + logger.warn(`[${requestId}] Connector removal could not acquire or finish its transaction`, { + connectorId, + error, + }) + return fail('Connection is busy. Try removing it again in a moment.', 'conflict') + } return classifyKnowledgeFailure(error, requestId, `Delete connector ${connectorId}`) } @@ -1308,15 +1326,7 @@ export async function performDeleteKnowledgeConnector( ) } - if (deleteDocuments) { - await Promise.all([ - cleanupUnusedTagDefinitions(kb.id, requestId).catch((error) => { - logger.warn(`[${requestId}] Failed to cleanup tag definitions`, error) - }), - ]) - } - - if (existing.credentialGroupId && kb.workspaceId) { + if (!deleteDocuments && existing.credentialGroupId && kb.workspaceId) { await revokeKnowledgeConnectorCredentialAccess( { workspaceId: kb.workspaceId, diff --git a/apps/sim/lib/knowledge/tags/service.test.ts b/apps/sim/lib/knowledge/tags/service.test.ts index 92e8c31f515..43f36ddcef4 100644 --- a/apps/sim/lib/knowledge/tags/service.test.ts +++ b/apps/sim/lib/knowledge/tags/service.test.ts @@ -2,7 +2,8 @@ * @vitest-environment node */ -import { knowledgeBaseTagDefinitions } from '@sim/db/schema' +import { db } from '@sim/db' +import { document, embedding, knowledgeBaseTagDefinitions } from '@sim/db/schema' import { dbChainMockFns, hasMockCondition, queueTableRows, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -12,6 +13,7 @@ vi.mock('@sim/utils/id', () => ({ })) import { + cleanupUnusedTagDefinitions, createOrUpdateTagDefinitionsBulk, createTagDefinition, getDocumentTagDefinitions, @@ -34,6 +36,48 @@ function existingDefinition(overrides: Record) { } } +describe('cleanupUnusedTagDefinitions', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('keeps tags used by either documents or chunks and removes only unused definitions', async () => { + queueTableRows(knowledgeBaseTagDefinitions, [ + existingDefinition({ id: 'document-tag', tagSlot: 'tag1' }), + existingDefinition({ id: 'chunk-tag', tagSlot: 'tag2' }), + existingDefinition({ id: 'unused-tag', tagSlot: 'tag3' }), + ]) + queueTableRows(document, [{ id: 'doc-1' }]) + queueTableRows(document, []) + queueTableRows(embedding, [{ id: 'chunk-1' }]) + queueTableRows(document, []) + queueTableRows(embedding, []) + + expect(await cleanupUnusedTagDefinitions('kb-1', 'request-1')).toBe(1) + expect(dbChainMockFns.delete).toHaveBeenCalledOnce() + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls.at(-1)?.[0], + (node) => node.type === 'eq' && node.right === 'unused-tag' + ) + ).toBe(true) + }) + + it('stops cleanup before deleting tags when its worker is cancelled', async () => { + queueTableRows(knowledgeBaseTagDefinitions, [existingDefinition({})]) + const controller = new AbortController() + controller.abort() + await expect( + cleanupUnusedTagDefinitions('kb-1', 'request-1', { + executor: db, + signal: controller.signal, + }) + ).rejects.toThrow() + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + }) +}) + describe('getDocumentTagDefinitionsByKnowledgeBaseIds', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 7a599fea365..b414c969f1c 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -563,17 +563,20 @@ export async function getTagDefinitionById( */ export async function cleanupUnusedTagDefinitions( knowledgeBaseId: string, - requestId: string + requestId: string, + options?: { executor: DbOrTx; signal: AbortSignal } ): Promise { - const definitions = await getDocumentTagDefinitions(knowledgeBaseId) + const executor = options?.executor ?? db + const definitions = await getDocumentTagDefinitions(knowledgeBaseId, executor) let cleanedUp = 0 for (const def of definitions) { + options?.signal.throwIfAborted() const tagSlot = def.tagSlot validateTagSlot(tagSlot) - const docCountResult = await db - .select({ count: sql`count(*)` }) + const [taggedDocument] = await executor + .select({ id: document.id }) .from(document) .where( and( @@ -583,9 +586,12 @@ export async function cleanupUnusedTagDefinitions( sql`${sql.raw(tagSlot)} IS NOT NULL` ) ) + .limit(1) + if (taggedDocument) continue - const chunkCountResult = await db - .select({ count: sql`count(*)` }) + options?.signal.throwIfAborted() + const [taggedChunk] = await executor + .select({ id: embedding.id }) .from(embedding) .innerJoin(document, eq(embedding.documentId, document.id)) .where( @@ -596,12 +602,13 @@ export async function cleanupUnusedTagDefinitions( sql`${sql.raw(`embedding.${tagSlot}`)} IS NOT NULL` ) ) + .limit(1) - const docCount = Number(docCountResult[0]?.count || 0) - const chunkCount = Number(chunkCountResult[0]?.count || 0) - - if (docCount === 0 && chunkCount === 0) { - await db.delete(knowledgeBaseTagDefinitions).where(eq(knowledgeBaseTagDefinitions.id, def.id)) + if (!taggedChunk) { + options?.signal.throwIfAborted() + await executor + .delete(knowledgeBaseTagDefinitions) + .where(eq(knowledgeBaseTagDefinitions.id, def.id)) cleanedUp++ logger.info( From bb2023972de97038b3d756e030c875f3d1fadc86 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 15 Sep 2026 12:12:33 -0700 Subject: [PATCH 02/15] fix(search): preserve filters and report incomplete results (#7855) --- .../o/[organizationId]/search/search.test.tsx | 6 +- .../knowledge-search-results.test.tsx | 18 +- .../knowledge-search-results.tsx | 223 ++++++------ .../search-transitions.test.tsx | 342 ++++++++++++++++++ apps/sim/hooks/queries/kb/knowledge.test.ts | 19 +- apps/sim/hooks/queries/kb/knowledge.ts | 18 +- .../sim/hooks/queries/utils/knowledge-keys.ts | 10 +- .../server/knowledge/workspace-search.test.ts | 17 +- .../search-latency.integration.ts | 9 +- .../lib/knowledge/search/diagnostics.test.ts | 33 ++ apps/sim/lib/knowledge/search/diagnostics.ts | 4 +- 11 files changed, 572 insertions(+), 127 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx diff --git a/apps/sim/app/o/[organizationId]/search/search.test.tsx b/apps/sim/app/o/[organizationId]/search/search.test.tsx index 9b9a6f78714..f6c9d62e804 100644 --- a/apps/sim/app/o/[organizationId]/search/search.test.tsx +++ b/apps/sim/app/o/[organizationId]/search/search.test.tsx @@ -17,6 +17,9 @@ const mocks = vi.hoisted(() => ({ })) vi.mock('@/hooks/use-speech-to-text', () => ({ useSpeechToText: mocks.speech })) +vi.mock('@/lib/auth/auth-client', () => ({ + useSession: () => ({ data: { user: { id: 'reader' } } }), +})) vi.mock('next/navigation', () => ({ useRouter: () => ({ push: mocks.push }), usePathname: () => '/o/organization-a/search', @@ -32,9 +35,6 @@ vi.mock('@/hooks/queries/kb/connectors', () => ({ useSearchIndex: () => ({ data: { knowledgeBaseId: 'index-a' }, isPending: false }), useSearchSourceOverview: () => ({ data: { providers: [], hasSearchableDocuments: true } }), })) -vi.mock('@/app/workspace/[workspaceId]/home/components/search-sources', () => ({ - isIndexing: () => false, -})) vi.mock( '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags', () => ({ diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index 6b9617c5d38..2264db73f56 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -10,6 +10,9 @@ const mocks = vi.hoisted(() => ({ search: vi.fn(), retry: vi.fn(), })) +vi.mock('@/lib/auth/auth-client', () => ({ + useSession: () => ({ data: { user: { id: 'reader' } } }), +})) vi.mock('@/hooks/queries/kb/connectors', () => ({ useSearchIndex: mocks.index, useSearchSourceOverview: mocks.overview, @@ -69,7 +72,8 @@ describe('source indexing context in search results', () => { await render() expect(mocks.overview).toHaveBeenCalledWith({ kind: 'workspace', workspaceId: 'workspace' }) expect(container.textContent).toContain('Google Drive') - expect(container.textContent).not.toContain('Slack') + expect(container.textContent).toContain('Slack') + expect(container.textContent).toContain('Still indexing Google Drive;') }) it('does not invent indexing progress while the overview is unavailable', async () => { mocks.overview.mockReturnValue({ data: undefined }) @@ -81,7 +85,7 @@ describe('source indexing context in search results', () => { describe('incomplete search coverage', () => { it.each([false, true])( - 'shows matches without timeout copy or retry controls (hasResults=%s)', + 'distinguishes incomplete retrieval and permits retry (hasResults=%s)', async (hasResults) => { mocks.search.mockReturnValue({ data: { @@ -113,17 +117,17 @@ describe('incomplete search coverage', () => { await render() expect(container.textContent).not.toContain('Search couldn’t run') expect(container.textContent).not.toContain('No documents') - expect(container.textContent).not.toContain('Some results may be missing.') - expect(container.textContent).not.toContain('Search is incomplete.') expect(container.textContent).toContain( - hasResults ? '1 document' : 'Search found no results.' + hasResults ? '1 document · some results may be missing.' : 'Search didn’t finish.' ) + expect(container.textContent).not.toContain('Search found no results.') if (hasResults) expect(container.textContent).toContain('Release plan') const retry = [...container.querySelectorAll('button')].find( (button) => button.textContent === 'Try again' ) - expect(retry).toBeUndefined() - expect(mocks.retry).not.toHaveBeenCalled() + expect(retry).toBeDefined() + await act(async () => retry?.click()) + expect(mocks.retry).toHaveBeenCalledOnce() } ) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 3c92726ee09..b4bff7f98dd 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -1,14 +1,15 @@ 'use client' -import { useMemo } from 'react' -import { Chip, ChipLink } from '@sim/emcn' +import { useState } from 'react' +import { Chip, ChipLink, cn } from '@sim/emcn' import { useQueryStates } from 'nuqs' import { ActivityStatus } from '@/components/ui/activity-status' import type { WorkspaceKnowledgeSearchResult, WorkspaceSearchFilters, } from '@/lib/api/contracts/knowledge' -import type { ResourceScope } from '@/lib/core/resource-scope' +import { useSession } from '@/lib/auth/auth-client' +import { type ResourceScope, resourceScopeKey } from '@/lib/core/resource-scope' import { getBaseUrl } from '@/lib/core/utils/urls' import { matchSnippet } from '@/lib/knowledge/search/snippet' import { connectorDisplayName } from '@/lib/sim-search/connectors' @@ -25,8 +26,6 @@ import { import { useSearchIndex, useSearchSourceOverview } from '@/hooks/queries/kb/connectors' import { useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' -/** Filters appear only once a list is long and mixed enough for them to help. */ -const FILTERS_MIN_RESULTS = 10 const DAY_MS = 24 * 60 * 60 * 1000 /** Every result without a connector is an upload; the filter names them so. */ const UPLOAD_SOURCE = 'upload' @@ -82,6 +81,7 @@ function handleResultsKeyDown(event: React.KeyboardEvent) { const links = [...event.currentTarget.querySelectorAll('a[data-source-link]')] if (links.length === 0) return const index = links.findIndex((link) => link === document.activeElement) + if (index < 0) return const next = event.key === 'ArrowDown' ? Math.min(index + 1, links.length - 1) : Math.max(index - 1, 0) if (next === index) return @@ -98,15 +98,7 @@ type KnowledgeSearchResultsProps = ( onSummarize: (prompt: string, filters: WorkspaceSearchFilters) => void } -/** - * Search results include documents the signed-in person may read that - * match their query in the canonical Enterprise Search index, as rows - * that open the source. A header says how many and that the search ran as - * them; while a connected source is still indexing it says so, and the list - * grows as documents land. Filters by source and recency appear only once the - * list is long and mixed enough to need them, and live in the URL beside the - * query so a filtered search is a shareable link. - */ +/** A new query or access scope starts a fresh search and rolling-date anchor. */ export function KnowledgeSearchResults({ workspaceId, scope: suppliedScope, @@ -114,6 +106,26 @@ export function KnowledgeSearchResults({ onSummarize, }: KnowledgeSearchResultsProps) { const scope: ResourceScope = suppliedScope ?? { kind: 'workspace', workspaceId: workspaceId! } + const { data: session } = useSession() + const trimmed = query.trim() + return ( + + ) +} + +interface SearchResultsProps { + scope: ResourceScope + query: string + onSummarize: KnowledgeSearchResultsProps['onSummarize'] +} + +function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { + const [searchedAt] = useState(Date.now) const { data: index, isPending: basesPending, @@ -121,21 +133,19 @@ export function KnowledgeSearchResults({ isFetching: basesFetching, refetch: refetchIndex, } = useSearchIndex(scope) - const knowledgeBaseIds = index?.knowledgeBaseId ? [index.knowledgeBaseId] : [] const [filters, setFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) - const searchFilters = useMemo(() => { - const window = UPDATED_WINDOWS.find((entry) => entry.id === filters.updated) - return { - ...(filters.source ? { source: filters.source } : {}), - ...(window?.days - ? { modifiedAfter: new Date(Date.now() - window.days * DAY_MS).toISOString() } - : {}), - } - }, [filters.source, filters.updated]) + const window = UPDATED_WINDOWS.find((entry) => entry.id === filters.updated) + const searchFilters: WorkspaceSearchFilters = { + ...(filters.source ? { source: filters.source } : {}), + ...(window?.days + ? { modifiedAfter: new Date(searchedAt - window.days * DAY_MS).toISOString() } + : {}), + } const { data: search, isPending, isFetching, + isPlaceholderData, isError: searchFailed, refetch: refetchSearch, } = useWorkspaceKnowledgeSearch(scope, query, searchFilters) @@ -143,35 +153,22 @@ export function KnowledgeSearchResults({ const indexing = (overview?.providers ?? []) .filter((provider) => provider.isSyncing) .map((provider) => connectorDisplayName(provider.connectorType)) - const documents = useMemo(() => groupResultsByDocument(search?.results ?? []), [search?.results]) + const documents = groupResultsByDocument(search?.results ?? []) const sourceTypes = [ ...new Set([ ...(filters.source ? [filters.source] : []), - ...documents.map((result) => result.connectorType ?? UPLOAD_SOURCE), + ...(overview?.providers.map((provider) => provider.connectorType) ?? []), + UPLOAD_SOURCE, ]), - ] - const filtersActive = filters.source !== null || filters.updated !== 'any' - /** The controls appear once the list is long and mixed, and stay while a filter from the link is active. */ - const showFilters = - filtersActive || (documents.length >= FILTERS_MIN_RESULTS && sourceTypes.length > 1) + ].sort((left, right) => connectorDisplayName(left).localeCompare(connectorDisplayName(right))) + const failed = basesFailed || searchFailed + const pending = basesPending || isPending + const fetching = basesFetching || isFetching + const noSources = !basesPending && !basesFailed && !index?.knowledgeBaseId + const partial = search?.retrieval.status === 'partial' + const documentCount = documents.length === 1 ? '1 document' : `${documents.length} documents` - /** A failed search offers a retry; server diagnostics carry the cause. */ - if (basesFailed || searchFailed) { - const retrying = basesFetching || isFetching - return ( -
-

Search couldn’t run.

- void (basesFailed ? refetchIndex() : refetchSearch())} - > - {retrying ? 'Retrying…' : 'Try again'} - -
- ) - } - if (!basesPending && knowledgeBaseIds.length === 0) { + if (noSources) { return (

No sources are set up yet.

@@ -187,14 +184,6 @@ export function KnowledgeSearchResults({
) } - if (isPending || (isFetching && !search)) { - return ( -
- -
- ) - } - const indexingNote = indexing.length > 0 ? `Still indexing ${indexing.join(', ')}; results grow as documents land.` @@ -203,54 +192,81 @@ export function KnowledgeSearchResults({ return (
- - {documents.length === 0 ? ( - 'Search found no results.' +
+ {fetching || (pending && !failed) ? ( + ) : ( - <> - - {documents.length === 1 ? '1 document' : `${documents.length} documents`} - - {' · searched as you'} - +

+ {failed + ? 'Search couldn’t run.' + : partial + ? documents.length === 0 + ? 'Search didn’t finish.' + : `${documentCount} · some results may be missing.` + : documents.length === 0 + ? 'Search found no results.' + : `${documentCount} · searched as you`} +

+ )} + {indexingNote && !failed && ( +

{indexingNote}

)} - {indexingNote && {indexingNote}} - +
+ {(failed || partial) && ( + void (basesFailed ? refetchIndex() : refetchSearch())} + > + {fetching ? 'Retrying…' : 'Try again'} + + )}
- {showFilters && ( -
+
+ setFilters({ source: null })} + > + All sources + + {sourceTypes.map((type) => ( setFilters({ source: null })} + active={filters.source === type} + aria-pressed={filters.source === type} + onClick={() => setFilters({ source: filters.source === type ? null : type })} > - All sources + {type === UPLOAD_SOURCE ? 'Uploads' : connectorDisplayName(type)} - {sourceTypes.map((type) => ( - setFilters({ source: filters.source === type ? null : type })} - > - {type === UPLOAD_SOURCE ? 'Uploads' : connectorDisplayName(type)} - - ))} - - {UPDATED_WINDOWS.map((window) => ( - setFilters({ updated: window.id })} - > - {window.label} - - ))} -
- )} - {documents.length > 0 && ( -
+ ))} + + {UPDATED_WINDOWS.map((window) => ( + setFilters({ updated: window.id })} + > + {window.label} + + ))} +
+ {!failed && !basesPending && documents.length > 0 && ( +
{documents.map((result) => { const source = toSource(result, query, scope) return ( @@ -258,11 +274,14 @@ export function KnowledgeSearchResults({ key={result.documentId} source={source} query={query} - onSummarize={(cited) => - onSummarize(`Summarize "${cited.title ?? cited.url}"`, { - ...searchFilters, - documentIds: [result.documentId], - }) + onSummarize={ + isPlaceholderData + ? undefined + : (cited) => + onSummarize(`Summarize "${cited.title ?? cited.url}"`, { + ...searchFilters, + documentIds: [result.documentId], + }) } /> ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx new file mode 100644 index 00000000000..f6f22427549 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx @@ -0,0 +1,342 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { NuqsTestingAdapter } from 'nuqs/adapters/testing' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + request: vi.fn(), + userId: 'reader', + summarize: vi.fn(), + urlUpdate: vi.fn(), +})) +vi.mock('@/lib/auth/auth-client', () => ({ + useSession: () => ({ data: { user: { id: mocks.userId } } }), +})) +vi.mock('@/lib/api/client/request', () => ({ requestJson: mocks.request })) +vi.mock('@/hooks/queries/kb/connectors', () => ({ + useSearchIndex: () => ({ data: { knowledgeBaseId: 'index' }, isPending: false }), + useSearchSourceOverview: () => ({ + data: { + providers: [ + { connectorType: 'slack', isSyncing: false }, + { connectorType: 'gmail', isSyncing: false }, + ], + }, + }), +})) +vi.mock( + '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card', + () => ({ + SourceCard: ({ + source, + onSummarize, + }: { + source: { title: string; url: string } + onSummarize?: (source: { title: string; url: string }) => void + }) => ( +
+ + {source.title} + + + {onSummarize && ( + + )} +
+ ), + }) +) + +import type { + WorkspaceKnowledgeSearchBody, + WorkspaceKnowledgeSearchData, +} from '@/lib/api/contracts/knowledge' +import type { ResourceScope } from '@/lib/core/resource-scope' +import { KnowledgeSearchResults } from '@/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results' +import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' + +interface PendingSearch { + body: WorkspaceKnowledgeSearchBody + signal: AbortSignal + resolve: (data: { data: WorkspaceKnowledgeSearchData }) => void + reject: (error: Error) => void +} + +let root: Root +let container: HTMLDivElement +let client: QueryClient +let requests: PendingSearch[] + +beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-15T12:00:00Z')) + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + mocks.userId = 'reader' + requests = [] + mocks.request.mockImplementation( + (_contract, input) => + new Promise((resolve, reject) => { + requests.push({ ...input, resolve, reject }) + }) + ) + client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + container = document.createElement('div') + document.body.append(container) + root = createRoot(container) +}) + +afterEach(() => { + act(() => root.unmount()) + client.clear() + container.remove() + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +async function render({ + scope = { kind: 'organization', organizationId: 'organization' }, + query = 'launch', + params = '', +}: { + scope?: ResourceScope + query?: string + params?: string +} = {}) { + await act(async () => { + root.render( + + + + + + ) + }) +} + +function button(label: string) { + const result = [...container.querySelectorAll('button')].find( + (item) => item.textContent === label + ) + if (!result) throw new Error(`Missing button: ${label}`) + return result +} + +async function click(label: string) { + await act(async () => { + button(label).focus() + button(label).click() + await vi.advanceTimersByTimeAsync(1) + }) +} + +async function complete( + index: number, + { title = 'Release plan', partial = false, empty = false } = {} +) { + await act(async () => { + requests[index].resolve({ + data: { + query: requests[index].body.query, + results: empty + ? [] + : [ + { + documentId: title, + knowledgeBaseId: 'index', + knowledgeBaseName: 'Search index', + documentName: title, + sourceUrl: 'https://example.com/release', + connectorType: requests[index].body.filters?.source ?? 'slack', + sourceModifiedAt: null, + author: null, + content: 'launch details', + chunkIndex: 0, + similarity: 0.9, + }, + ], + retrieval: { + status: partial ? 'partial' : 'complete', + timedOutLegs: partial ? ['vector'] : [], + }, + }, + }) + await vi.advanceTimersByTimeAsync(1) + }) +} + +describe('search refinement with the real query cache and URL state', () => { + it('replaces filter URL state while preserving unrelated parameters', async () => { + await render({ params: '?q=launch&panel=details' }) + await click('Gmail') + await click('Past week') + expect(mocks.urlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ + queryString: '?q=launch&panel=details&source=gmail&updated=7d', + options: expect.objectContaining({ history: 'replace' }), + }) + ) + await click('All sources') + await click('Any time') + expect(mocks.urlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ queryString: '?q=launch&panel=details' }) + ) + }) + + it('keeps controls and focus while retaining only the preceding refinement results', async () => { + await render() + expect(button('Gmail')).toBeDefined() + expect(container.textContent).toContain('Searching…') + await complete(0) + const gmail = button('Gmail') + await click('Gmail') + expect(button('Gmail')).toBe(gmail) + expect(document.activeElement).toBe(gmail) + expect(gmail.getAttribute('aria-pressed')).toBe('true') + expect(container.textContent).toContain('Updating results…') + expect(container.textContent).toContain('Release plan') + expect(container.textContent).not.toContain('Summarize') + expect(requests[1].body.filters).toMatchObject({ source: 'gmail' }) + await complete(1, { title: 'Email plan' }) + expect(document.activeElement).toBe(gmail) + expect(container.textContent).not.toContain('Release plan') + await click('Summarize') + expect(mocks.summarize).toHaveBeenCalledWith(expect.any(String), { + source: 'gmail', + documentIds: ['Email plan'], + }) + }) + + it('cancels an abandoned refinement and reuses a fresh cached result', async () => { + await render() + await complete(0) + await click('Gmail') + await click('All sources') + expect(requests).toHaveLength(2) + expect(requests[1].signal.aborted).toBe(true) + expect(container.textContent).toContain('Release plan') + expect(container.textContent).not.toContain('Updating results…') + await complete(1, { title: 'Abandoned result' }) + expect(container.textContent).not.toContain('Abandoned result') + }) + + it('keeps one rolling cutoff across source and date refinements, then resets for a new query', async () => { + await render({ params: '?updated=7d' }) + const cutoff = requests[0].body.filters?.modifiedAfter + expect(cutoff).toBe('2026-01-08T12:00:00.000Z') + await complete(0) + vi.setSystemTime(new Date('2026-01-15T12:00:20Z')) + await click('Gmail') + expect(requests[1].body.filters?.modifiedAfter).toBe(cutoff) + await click('Past month') + expect(requests[2].body.filters?.modifiedAfter).toBe('2025-12-16T12:00:00.000Z') + await click('Past week') + expect(requests[3].body.filters?.modifiedAfter).toBe(cutoff) + vi.setSystemTime(new Date('2026-01-15T12:00:30Z')) + await render({ query: 'another question', params: '?updated=7d' }) + expect(requests.at(-1)?.body.filters?.modifiedAfter).toBe('2026-01-08T12:00:30.000Z') + }) + + it.each(['query', 'organization', 'workspace', 'reader'])( + 'clears prior results when the %s changes', + async (change) => { + await render() + await complete(0) + if (change === 'reader') mocks.userId = 'another-reader' + await render({ + query: change === 'query' ? 'another question' : 'launch', + scope: + change === 'organization' + ? { kind: 'organization', organizationId: 'another-org' } + : change === 'workspace' + ? { kind: 'workspace', workspaceId: 'another-workspace' } + : { kind: 'organization', organizationId: 'organization' }, + }) + expect(container.textContent).not.toContain('Release plan') + expect(container.textContent).toContain('Searching…') + expect(requests).toHaveLength(2) + } + ) + + it('does not restore cleared access data as a placeholder', async () => { + await render() + await complete(0) + await act(async () => { + void client.resetQueries({ queryKey: knowledgeKeys.searches() }) + await vi.advanceTimersByTimeAsync(1) + }) + expect(container.textContent).not.toContain('Release plan') + await click('Gmail') + expect(container.textContent).not.toContain('Release plan') + }) + + it('clears displayed placeholder data when access is reset during a refinement', async () => { + await render() + await complete(0) + await click('Gmail') + expect(container.textContent).toContain('Release plan') + await act(async () => { + void client.resetQueries({ queryKey: knowledgeKeys.searches() }) + await vi.advanceTimersByTimeAsync(1) + }) + expect(container.textContent).not.toContain('Release plan') + }) + + it('does not retain invalidated results during a refinement', async () => { + await render() + await complete(0) + await act(async () => { + await client.invalidateQueries({ queryKey: knowledgeKeys.searches(), refetchType: 'none' }) + }) + await click('Gmail') + expect(container.textContent).not.toContain('Release plan') + }) + + it.each([true, false])( + 'keeps filters and useful matches for partial results, then retries (empty=%s)', + async (empty) => { + await render() + await complete(0, { partial: true, empty }) + expect(container.textContent).toContain( + empty ? 'Search didn’t finish.' : 'some results may be missing.' + ) + expect(container.textContent).not.toContain('Search found no results.') + const gmail = button('Gmail') + await click('Try again') + expect(button('Retrying…').disabled).toBe(true) + expect(button('Gmail')).toBe(gmail) + await complete(1, { empty: true }) + expect(container.textContent).toContain('Search found no results.') + expect(container.textContent).not.toContain('Try again') + } + ) + + it('preserves filter focus and permits recovery after a failed refinement', async () => { + await render() + await complete(0) + const gmail = button('Gmail') + await click('Gmail') + await act(async () => { + requests[1].reject(new Error('Search failed')) + await vi.advanceTimersByTimeAsync(1) + }) + expect(document.activeElement).toBe(gmail) + expect(container.textContent).toContain('Search couldn’t run.') + expect(container.textContent).not.toContain('Release plan') + await click('All sources') + expect(container.textContent).toContain('Release plan') + }) + + it('does not redirect arrow keys from row actions to the first result', async () => { + await render() + await complete(0) + const copy = button('Copy link') + copy.focus() + act(() => copy.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown', bubbles: true }))) + expect(document.activeElement).toBe(copy) + }) +}) diff --git a/apps/sim/hooks/queries/kb/knowledge.test.ts b/apps/sim/hooks/queries/kb/knowledge.test.ts index 71729ec1bde..07e2c6ca6e5 100644 --- a/apps/sim/hooks/queries/kb/knowledge.test.ts +++ b/apps/sim/hooks/queries/kb/knowledge.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ useMutation: vi.fn(), useQuery: vi.fn(), invalidateQueries: vi.fn(), + getQueryData: vi.fn(), })) vi.mock('@tanstack/react-query', () => ({ @@ -16,7 +17,14 @@ vi.mock('@tanstack/react-query', () => ({ useInfiniteQuery: vi.fn(), useMutation: mocks.useMutation, useQuery: mocks.useQuery, - useQueryClient: vi.fn(() => ({ invalidateQueries: mocks.invalidateQueries })), + useQueryClient: vi.fn(() => ({ + invalidateQueries: mocks.invalidateQueries, + getQueryData: mocks.getQueryData, + })), +})) + +vi.mock('@/lib/auth/auth-client', () => ({ + useSession: () => ({ data: { user: { id: 'reader' } } }), })) vi.mock('@sim/emcn', () => ({ @@ -202,13 +210,18 @@ describe('knowledge query placeholder scope', () => { ).toBeUndefined() }) - it('does not reuse results from a different query, workspace, or filter', () => { + it('partitions search cache entries by filter and reader', () => { const query = captureQuery(() => useWorkspaceKnowledgeSearch('workspace-1', 'new query', { source: 'slack' }) ) - expect(query.placeholderData).toBeUndefined() + expect(query.queryKey).toEqual( + knowledgeKeys.search('workspace-1', 'new query', { source: 'slack' }, 'reader') + ) expect(knowledgeKeys.search('workspace-1', 'query', { source: 'slack' })).not.toEqual( knowledgeKeys.search('workspace-1', 'query', { source: 'gitlab' }) ) + expect(knowledgeKeys.search('workspace-1', 'query', {}, 'reader')).not.toEqual( + knowledgeKeys.search('workspace-1', 'query', {}, 'another-reader') + ) }) }) diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index bb0a023bcda..5af98a7da79 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -55,6 +55,7 @@ import { type WorkspaceKnowledgeSearchData, } from '@/lib/api/contracts/knowledge' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge/search' +import { useSession } from '@/lib/auth/auth-client' import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import { type ResourceScope, @@ -1208,6 +1209,9 @@ export function useWorkspaceKnowledgeSearch( query: string, filters?: WorkspaceSearchFilters ) { + const { data: session } = useSession() + const queryClient = useQueryClient() + const userId = session?.user?.id const trimmed = query.trim() const scope = typeof owner === 'string' @@ -1218,7 +1222,7 @@ export function useWorkspaceKnowledgeSearch( const scopeKey = scope?.kind === 'workspace' ? scope.workspaceId : scope ? resourceScopeKey(scope) : undefined return useQuery({ - queryKey: knowledgeKeys.search(scopeKey, trimmed, filters), + queryKey: knowledgeKeys.search(scopeKey, trimmed, filters, userId), queryFn: ({ signal }) => searchWorkspaceKnowledge( { @@ -1228,8 +1232,18 @@ export function useWorkspaceKnowledgeSearch( }, signal ), - enabled: Boolean(scope) && trimmed.length > 0, + enabled: Boolean(scope && userId) && trimmed.length > 0, staleTime: WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME, retry: false, + placeholderData: (previous, previousQuery) => + userId && + previousQuery?.state.status === 'success' && + !previousQuery.state.isInvalidated && + knowledgeKeys + .searchQuery(scopeKey, trimmed, userId) + .every((part, index) => previousQuery.queryKey[index] === part) && + queryClient.getQueryData(previousQuery.queryKey) === previous + ? previous + : undefined, }) } diff --git a/apps/sim/hooks/queries/utils/knowledge-keys.ts b/apps/sim/hooks/queries/utils/knowledge-keys.ts index 493d6758bbf..8ad0ae229a6 100644 --- a/apps/sim/hooks/queries/utils/knowledge-keys.ts +++ b/apps/sim/hooks/queries/utils/knowledge-keys.ts @@ -32,8 +32,14 @@ export const knowledgeKeys = { detail: (knowledgeBaseId?: string) => [...knowledgeKeys.details(), knowledgeBaseId ?? ''] as const, searches: () => [...knowledgeKeys.all, 'search'] as const, - search: (workspaceId: string | undefined, query: string, filters?: WorkspaceSearchFilters) => - [...knowledgeKeys.searches(), workspaceId ?? '', query, filters ?? {}] as const, + searchQuery: (scopeKey: string | undefined, query: string, userId?: string) => + [...knowledgeKeys.searches(), scopeKey ?? '', userId ?? '', query] as const, + search: ( + scopeKey: string | undefined, + query: string, + filters?: WorkspaceSearchFilters, + userId?: string + ) => [...knowledgeKeys.searchQuery(scopeKey, query, userId), filters ?? {}] as const, tagDefinitions: (knowledgeBaseId: string) => [...knowledgeKeys.detail(knowledgeBaseId), 'tagDefinitions'] as const, tagUsage: (knowledgeBaseId: string) => diff --git a/apps/sim/lib/copilot/tools/server/knowledge/workspace-search.test.ts b/apps/sim/lib/copilot/tools/server/knowledge/workspace-search.test.ts index f14efc854e8..0ef7d3df4bb 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/workspace-search.test.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/workspace-search.test.ts @@ -41,6 +41,7 @@ import { searchWorkspaceServerTool, } from '@/lib/copilot/tools/server/knowledge/workspace-search' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { annotateSearchDiagnostics } from '@/lib/knowledge/search/diagnostics' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const context = { @@ -85,10 +86,16 @@ describe('Assistant retrieval tools', () => { }) }) it('returns empty incomplete retrieval as a recoverable search outcome and logs coverage', async () => { - mocks.search.mockResolvedValue({ - retrieval: { status: 'partial', timedOutLegs: ['vector', 'keyword'] }, - knowledgeBases: [{ id: 'index', name: 'Enterprise Search' }], - results: [], + mocks.search.mockImplementation(async () => { + annotateSearchDiagnostics({ + retrievalStatus: 'partial', + timedOutLegs: ['vector', 'keyword'], + }) + return { + retrieval: { status: 'partial', timedOutLegs: ['vector', 'keyword'] }, + knowledgeBases: [{ id: 'index', name: 'Enterprise Search' }], + results: [], + } }) const result = await searchWorkspaceServerTool.execute({ query: 'canaries' }, context) @@ -104,7 +111,7 @@ describe('Assistant retrieval tools', () => { expect(result).not.toHaveProperty('error') expect(mocks.info).toHaveBeenCalledWith( 'Knowledge search completed', - expect.objectContaining({ passageBytes: 0, originalPassageBytes: 0, outcome: 'success' }) + expect.objectContaining({ passageBytes: 0, originalPassageBytes: 0, outcome: 'partial' }) ) }) it('pins organization and private chat while reusing the canonical search index and citations', async () => { diff --git a/apps/sim/lib/knowledge/__integration__/search-latency.integration.ts b/apps/sim/lib/knowledge/__integration__/search-latency.integration.ts index 200952ca7cf..5d03eab792b 100644 --- a/apps/sim/lib/knowledge/__integration__/search-latency.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-latency.integration.ts @@ -171,7 +171,7 @@ function saveReport() { const diagnosticSchema = z .object({ surface: z.enum(['dashboard', 'copilot']), - outcome: z.literal('success'), + outcome: z.enum(['success', 'partial']), elapsedMs: z.number(), vectorBudgetMs: z.number().positive(), retrievalStatus: z.enum(['complete', 'partial']), @@ -259,6 +259,7 @@ async function searchDashboard(query = 'Orion deployment') { function expectCompleteVectorSearch(diagnostics: z.infer) { const budget = diagnostics.surface === 'dashboard' ? 3000 : 8000 expect(diagnostics).toMatchObject({ + outcome: 'success', vectorBudgetMs: budget, retrievalStatus: 'complete', timedOutLegs: [], @@ -635,7 +636,10 @@ describe.skipIf(!enabled)('Assistant search latency on a realistic indexed corpu const completed = diagnosticLog?.mock.calls.find( ([message]) => message === 'Knowledge search completed' ) - expect(diagnosticSchema.parse(completed?.[1]).vectorBudgetMs).toBe(8000) + expect(diagnosticSchema.parse(completed?.[1])).toMatchObject({ + vectorBudgetMs: 8000, + outcome: delayedLegs === 'vector' ? 'success' : 'partial', + }) if (delayedLegs === 'both') { expect(result).toMatchObject({ success: true, @@ -690,6 +694,7 @@ describe.skipIf(!enabled)('Assistant search latency on a realistic indexed corpu ([message]) => message === 'Knowledge search completed' ) const diagnostics = diagnosticSchema.parse(completed?.[1]) + expect(diagnostics.outcome).toBe('partial') expect(diagnostics.vectorBudgetMs).toBe(3000) expect(diagnostics.stages.vector.totalMs).toBeGreaterThan(2500) expect(diagnostics.stages.vector.totalMs).toBeLessThan(4000) diff --git a/apps/sim/lib/knowledge/search/diagnostics.test.ts b/apps/sim/lib/knowledge/search/diagnostics.test.ts index 21cf17c8e07..0d49ef060a2 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.test.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.test.ts @@ -123,4 +123,37 @@ describe('search pipeline diagnostics', () => { expect(logs.info).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) }) + + it.each([false, true])('counts partial completion separately (empty=%s)', async (empty) => { + const result = { results: empty ? [] : ['private result'] } + expect( + await withSearchDiagnostics({ surface: 'dashboard' }, async () => { + annotateSearchDiagnostics({ + retrievalStatus: 'partial', + timedOutLegs: ['vector'], + resultCount: result.results.length, + }) + return result + }) + ).toBe(result) + expect(logs.info).toHaveBeenCalledWith( + 'Knowledge search completed', + expect.objectContaining({ + outcome: 'partial', + retrievalStatus: 'partial', + timedOutLegs: ['vector'], + }) + ) + expect(JSON.stringify(logs.info.mock.calls)).not.toContain('private result') + }) + + it('keeps a later failure distinct from partial retrieval', async () => { + await expect( + withSearchDiagnostics({}, async () => { + annotateSearchDiagnostics({ retrievalStatus: 'partial' }) + throw new Error('metadata failed') + }) + ).rejects.toThrow('metadata failed') + expect(logs.info.mock.calls[0][1].outcome).toBe('error') + }) }) diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts index fe30814a125..9f5f3726575 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.ts @@ -180,7 +180,9 @@ export async function withSearchDiagnostics( outcome = result && typeof result === 'object' && 'success' in result && result.success === false ? 'error' - : 'success' + : trace.metadata.retrievalStatus === 'partial' + ? 'partial' + : 'success' return result } finally { clearInterval(timer) From 8fea7cf8b1240b96563bcbcc44aeb8f72eb3794a Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 15 Sep 2026 12:14:54 -0700 Subject: [PATCH 03/15] feat(cleanup): add per-type row limits to existing jobs (#7842) * feat(cleanup): add bounded manual retention runs * fix(cleanup): coordinate deletion with resource eligibility * fix(cleanup): persist storage retries and complete test fixtures * fix(cleanup): guard storage retries by file generation * improvement(cleanup): reuse existing jobs for row limits * fix(cleanup): fail manual jobs on owner lookup errors --- .../api/cron/cleanup-soft-deletes/route.ts | 20 +- apps/sim/app/api/logs/cleanup/route.test.ts | 72 +++++++ apps/sim/app/api/logs/cleanup/route.ts | 20 +- apps/sim/background/cleanup-logs.test.ts | 7 +- apps/sim/background/cleanup-logs.ts | 92 ++++++-- .../background/cleanup-soft-deletes.test.ts | 7 + apps/sim/background/cleanup-soft-deletes.ts | 198 +++++++++++------- apps/sim/lib/api/contracts/cleanup.test.ts | 22 ++ apps/sim/lib/api/contracts/cleanup.ts | 68 ++++++ .../lib/billing/cleanup-dispatcher.test.ts | 109 +++++++++- apps/sim/lib/billing/cleanup-dispatcher.ts | 84 ++++++-- apps/sim/lib/cleanup/batch-delete.test.ts | 75 ++++++- apps/sim/lib/cleanup/batch-delete.ts | 30 ++- apps/sim/lib/cleanup/bounded-cleanup.md | 18 ++ apps/sim/lib/cleanup/limits.ts | 41 ++++ apps/sim/lib/cleanup/queue.ts | 2 + .../payloads/large-value-metadata.ts | 31 ++- .../lib/logs/execution/snapshot/service.ts | 7 +- 18 files changed, 780 insertions(+), 123 deletions(-) create mode 100644 apps/sim/app/api/logs/cleanup/route.test.ts create mode 100644 apps/sim/lib/api/contracts/cleanup.test.ts create mode 100644 apps/sim/lib/api/contracts/cleanup.ts create mode 100644 apps/sim/lib/cleanup/bounded-cleanup.md create mode 100644 apps/sim/lib/cleanup/limits.ts create mode 100644 apps/sim/lib/cleanup/queue.ts diff --git a/apps/sim/app/api/cron/cleanup-soft-deletes/route.ts b/apps/sim/app/api/cron/cleanup-soft-deletes/route.ts index 1df6df035e3..520db937568 100644 --- a/apps/sim/app/api/cron/cleanup-soft-deletes/route.ts +++ b/apps/sim/app/api/cron/cleanup-soft-deletes/route.ts @@ -1,18 +1,36 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' +import { softDeletesCleanupContract } from '@/lib/api/contracts/cleanup' +import { parseRequest } from '@/lib/api/server/validation' import { verifyCronAuth } from '@/lib/auth/internal' -import { dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher' +import { dispatchBoundedCleanup, dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' export const dynamic = 'force-dynamic' const logger = createLogger('SoftDeleteCleanupAPI') +/** Cron-secret maintenance protocol is global; workspace principal authorization does not apply. */ export const GET = withRouteHandler(async (request: NextRequest) => { try { const authError = verifyCronAuth(request, 'soft-delete cleanup') if (authError) return authError + const parsed = await parseRequest( + softDeletesCleanupContract, + request, + {}, + { + rejectDuplicateQueryValues: true, + rejectBlankQueryValues: true, + } + ) + if (!parsed.success) return parsed.response + if (parsed.data.query) { + const result = await dispatchBoundedCleanup('cleanup-soft-deletes', parsed.data.query) + return NextResponse.json(result, { status: 202 }) + } + const result = await dispatchCleanupJobs('cleanup-soft-deletes') logger.info('Soft-delete cleanup jobs dispatched', result) diff --git a/apps/sim/app/api/logs/cleanup/route.test.ts b/apps/sim/app/api/logs/cleanup/route.test.ts new file mode 100644 index 00000000000..2841acf3a0f --- /dev/null +++ b/apps/sim/app/api/logs/cleanup/route.test.ts @@ -0,0 +1,72 @@ +import { createMockRequest } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { auth, bounded, scheduled } = vi.hoisted(() => ({ + auth: vi.fn(), + bounded: vi.fn(), + scheduled: vi.fn(), +})) +vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: auth })) +vi.mock('@/lib/billing/cleanup-dispatcher', () => ({ + dispatchBoundedCleanup: bounded, + dispatchCleanupJobs: scheduled, +})) + +import { GET as softDeletes } from '@/app/api/cron/cleanup-soft-deletes/route' +import { GET as logs } from '@/app/api/logs/cleanup/route' + +for (const [path, GET, type, limit] of [ + ['/api/logs/cleanup', logs, 'cleanup-logs', 'workflowLogs'], + ['/api/cron/cleanup-soft-deletes', softDeletes, 'cleanup-soft-deletes', 'workflows'], +] as const) { + describe(path, () => { + beforeEach(() => { + vi.clearAllMocks() + auth.mockReturnValue(null) + bounded.mockResolvedValue({ triggered: true, runId: 'run-one', limits: { [limit]: 2 } }) + scheduled.mockResolvedValue({ + jobIds: ['batch-one'], + jobCount: 1, + chunkCount: 2, + workspaceCount: 3, + }) + }) + const request = (query = '') => + createMockRequest('GET', undefined, {}, `http://localhost:3000${path}${query}`) + it('authenticates before parsing invalid limits', async () => { + auth.mockReturnValue(new Response(null, { status: 401 })) + expect((await GET(request('?unknown=1'))).status).toBe(401) + expect(bounded).not.toHaveBeenCalled() + expect(scheduled).not.toHaveBeenCalled() + }) + it('keeps no-parameter scheduled dispatch unchanged', async () => { + const response = await GET(request()) + expect(response.status).toBe(200) + expect(scheduled).toHaveBeenCalledWith(type) + expect(bounded).not.toHaveBeenCalled() + }) + it('accepts one bounded run', async () => { + const response = await GET(request(`?${limit}=2`)) + expect(response.status).toBe(202) + expect(bounded).toHaveBeenCalledWith(type, { [limit]: 2 }) + expect(await response.json()).toEqual({ + triggered: true, + runId: 'run-one', + limits: { [limit]: 2 }, + }) + expect(scheduled).not.toHaveBeenCalled() + }) + it.each(['?dryRun=true', '?unknown=1', '?batchSize=3', `?${limit}=2&${limit}=3`])( + 'rejects invalid query %s', + async (query) => { + expect((await GET(request(query))).status).toBe(400) + expect(bounded).not.toHaveBeenCalled() + expect(scheduled).not.toHaveBeenCalled() + } + ) + it('reports a dispatch failure', async () => { + bounded.mockRejectedValue(new Error('Trigger unavailable')) + expect((await GET(request(`?${limit}=2`))).status).toBe(500) + }) + }) +} diff --git a/apps/sim/app/api/logs/cleanup/route.ts b/apps/sim/app/api/logs/cleanup/route.ts index 7891a763bc6..7b8d4259cb4 100644 --- a/apps/sim/app/api/logs/cleanup/route.ts +++ b/apps/sim/app/api/logs/cleanup/route.ts @@ -1,18 +1,36 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' +import { logsCleanupContract } from '@/lib/api/contracts/cleanup' +import { parseRequest } from '@/lib/api/server/validation' import { verifyCronAuth } from '@/lib/auth/internal' -import { dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher' +import { dispatchBoundedCleanup, dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' export const dynamic = 'force-dynamic' const logger = createLogger('LogsCleanupAPI') +/** Cron-secret maintenance protocol is global; workspace principal authorization does not apply. */ export const GET = withRouteHandler(async (request: NextRequest) => { try { const authError = verifyCronAuth(request, 'logs cleanup') if (authError) return authError + const parsed = await parseRequest( + logsCleanupContract, + request, + {}, + { + rejectDuplicateQueryValues: true, + rejectBlankQueryValues: true, + } + ) + if (!parsed.success) return parsed.response + if (parsed.data.query) { + const result = await dispatchBoundedCleanup('cleanup-logs', parsed.data.query) + return NextResponse.json(result, { status: 202 }) + } + const result = await dispatchCleanupJobs('cleanup-logs') logger.info('Log cleanup jobs dispatched', result) diff --git a/apps/sim/background/cleanup-logs.test.ts b/apps/sim/background/cleanup-logs.test.ts index 2b1adb3cdc5..feb631202fa 100644 --- a/apps/sim/background/cleanup-logs.test.ts +++ b/apps/sim/background/cleanup-logs.test.ts @@ -45,9 +45,12 @@ const { mockTask: vi.fn((config: unknown) => config), })) -vi.mock('@trigger.dev/sdk', () => ({ task: mockTask })) +vi.mock('@trigger.dev/sdk', () => ({ task: mockTask, queue: vi.fn((config) => config) })) + +vi.mock('@/lib/billing/cleanup-dispatcher', () => ({ runCleanupWithLimits: vi.fn() })) vi.mock('@/lib/cleanup/batch-delete', () => ({ + consumeRowBudget: vi.fn(), batchDeleteByWorkspaceAndTimestamp: mockBatchDeleteByWorkspaceAndTimestamp, chunkedBatchDelete: mockChunkedBatchDelete, })) @@ -199,7 +202,7 @@ describe('cleanup logs worker', () => { it('caps Trigger.dev concurrency for log cleanup tasks', () => { expect(cleanupLogsTask).toMatchObject({ - queue: { concurrencyLimit: 2 }, + queue: { name: 'retention-cleanup', concurrencyLimit: 1 }, }) }) }) diff --git a/apps/sim/background/cleanup-logs.ts b/apps/sim/background/cleanup-logs.ts index 2d560ef74c0..e46127a8e6a 100644 --- a/apps/sim/background/cleanup-logs.ts +++ b/apps/sim/background/cleanup-logs.ts @@ -12,12 +12,16 @@ import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { task } from '@trigger.dev/sdk' import { and, asc, eq, inArray, isNull, lt, notInArray, or, sql } from 'drizzle-orm' -import type { CleanupJobPayload } from '@/lib/billing/cleanup-dispatcher' +import { type CleanupJobPayload, runCleanupWithLimits } from '@/lib/billing/cleanup-dispatcher' import { batchDeleteByWorkspaceAndTimestamp, chunkedBatchDelete, + consumeRowBudget, + type RowBudget, type TableCleanupResult, } from '@/lib/cleanup/batch-delete' +import type { CleanupBudgets, LimitedCleanupPayload } from '@/lib/cleanup/limits' +import { retentionCleanupQueue } from '@/lib/cleanup/queue' import { LIVE_PAUSED_REFERENCE_STATUSES, markLargeValuesDeleted, @@ -43,7 +47,6 @@ const WORKFLOW_LOG_CLEANUP_BATCH_SIZE = 500 const WORKFLOW_LOG_CLEANUP_MAX_BATCHES = 50 const WORKFLOW_LOG_CLEANUP_ROW_LIMIT = WORKFLOW_LOG_CLEANUP_BATCH_SIZE * WORKFLOW_LOG_CLEANUP_MAX_BATCHES -const LOG_CLEANUP_CONCURRENCY_LIMIT = 2 const LARGE_VALUE_CLEANUP_BATCH_SIZE = 500 const LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT = 5_000 const LARGE_VALUE_CLEANUP_GRACE_HOURS = 7 * 24 @@ -135,7 +138,8 @@ async function deleteLargeValueKeys(keys: string[]): Promise<{ deleted: number; async function cleanupLargeExecutionValues( workspaceIds: string[], retentionDate: Date, - label: string + label: string, + budget?: RowBudget ): Promise { const stats: LargeValueCleanupStats = { largeValuesTotal: 0, @@ -151,10 +155,11 @@ async function cleanupLargeExecutionValues( let attempted = 0 for (const chunkIds of workspaceChunks) { - while (attempted < LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) { + while (attempted < LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT && budget?.remaining !== 0) { const limit = Math.min( LARGE_VALUE_CLEANUP_BATCH_SIZE, - LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT - attempted + LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT - attempted, + budget?.remaining ?? LARGE_VALUE_CLEANUP_BATCH_SIZE ) const rows = await cleanupDb .select({ key: executionLargeValues.key }) @@ -176,19 +181,21 @@ async function cleanupLargeExecutionValues( if (rows.length === 0) break + consumeRowBudget(budget, rows.length) const keys = rows.map((row) => row.key) stats.largeValuesTotal += keys.length attempted += keys.length const result = await deleteLargeValueKeys(keys) stats.largeValuesDeleted += result.deleted stats.largeValuesDeleteFailed += result.failed + if (budget && result.failed) throw new Error('Large value cleanup failed') if (result.deleted === 0) { break } } - if (attempted >= LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) break + if (attempted >= LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT || budget?.remaining === 0) break } logger.info( @@ -201,7 +208,8 @@ async function cleanupLargeExecutionValues( async function cleanupLegacyLargeExecutionValues( workspaceIds: string[], retentionDate: Date, - label: string + label: string, + budget?: RowBudget ): Promise { const stats: LargeValueCleanupStats = { largeValuesTotal: 0, @@ -217,10 +225,11 @@ async function cleanupLegacyLargeExecutionValues( let attempted = 0 for (const chunkIds of workspaceChunks) { - while (attempted < LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) { + while (attempted < LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT && budget?.remaining !== 0) { const limit = Math.min( LARGE_VALUE_CLEANUP_BATCH_SIZE, - LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT - attempted + LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT - attempted, + budget?.remaining ?? LARGE_VALUE_CLEANUP_BATCH_SIZE ) const rows = await cleanupDb .select({ key: workspaceFiles.key }) @@ -329,19 +338,21 @@ async function cleanupLegacyLargeExecutionValues( if (rows.length === 0) break + consumeRowBudget(budget, rows.length) const keys = rows.map((row) => row.key) stats.largeValuesTotal += keys.length attempted += keys.length const result = await deleteLargeValueKeys(keys) stats.largeValuesDeleted += result.deleted stats.largeValuesDeleteFailed += result.failed + if (budget && result.failed) throw new Error('Large value cleanup failed') if (result.deleted === 0) { break } } - if (attempted >= LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) break + if (attempted >= LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT || budget?.remaining === 0) break } logger.info( @@ -351,7 +362,11 @@ async function cleanupLegacyLargeExecutionValues( return stats } -async function cleanupLargeValueMetadata(workspaceIds: string[], label: string): Promise { +async function cleanupLargeValueMetadata( + workspaceIds: string[], + label: string, + budgets?: CleanupBudgets +): Promise { try { const tombstonesDeletedBefore = new Date( Date.now() - LARGE_VALUE_TOMBSTONE_RETENTION_HOURS * 60 * 60 * 1000 @@ -359,12 +374,14 @@ async function cleanupLargeValueMetadata(workspaceIds: string[], label: string): const result = await pruneLargeValueMetadata({ workspaceIds, tombstonesDeletedBefore, + budgets, dbClient: cleanupDb, }) logger.info( `[${label}/execution_large_value_metadata] Pruned ${result.referencesDeleted} stale references, ${result.dependenciesDeleted} dependencies, ${result.tombstonesDeleted} tombstones` ) } catch (error) { + if (budgets) throw error logger.error(`[${label}/execution_large_value_metadata] Failed to prune metadata`, { error }) } } @@ -372,7 +389,8 @@ async function cleanupLargeValueMetadata(workspaceIds: string[], label: string): async function cleanupWorkflowExecutionLogs( workspaceIds: string[], retentionDate: Date, - label: string + label: string, + budget?: RowBudget ): Promise { const fileStats: FileDeleteStats = { filesTotal: 0, @@ -381,6 +399,7 @@ async function cleanupWorkflowExecutionLogs( } const dbStats = await chunkedBatchDelete({ + budget, tableDef: workflowExecutionLogs, workspaceIds, tableName: `${label}/workflow_execution_logs`, @@ -420,17 +439,27 @@ async function cleanupWorkflowExecutionLogs( return { ...dbStats, ...fileStats } } -async function cleanupFreePlanOrphanedSnapshots(retentionHours: number): Promise { +async function cleanupFreePlanOrphanedSnapshots( + retentionHours: number, + budget?: RowBudget +): Promise { try { const retentionDays = Math.floor(retentionHours / 24) - const snapshotsCleaned = await snapshotService.cleanupOrphanedSnapshots(retentionDays + 1) + const snapshotsCleaned = await snapshotService.cleanupOrphanedSnapshots( + retentionDays + 1, + budget + ) logger.info(`Cleaned up ${snapshotsCleaned} orphaned snapshots`) } catch (snapshotError) { + if (budget) throw snapshotError logger.error('Error cleaning up orphaned snapshots:', { snapshotError }) } } -export async function runCleanupLogs(payload: CleanupJobPayload): Promise { +export async function runCleanupLogs( + payload: CleanupJobPayload, + budgets?: CleanupBudgets +): Promise { const startTime = Date.now() const { workspaceIds, retentionHours, label, plan, runGlobalHousekeeping } = payload @@ -439,7 +468,7 @@ export async function runCleanupLogs(payload: CleanupJobPayload): Promise if (workspaceIds.length === 0) { logger.info(`[${label}] No workspaces to process`) if (runGlobalHousekeeping && plan === 'free') { - await cleanupFreePlanOrphanedSnapshots(retentionHours) + await cleanupFreePlanOrphanedSnapshots(retentionHours, budgets?.orphanSnapshots) } return } @@ -448,25 +477,38 @@ export async function runCleanupLogs(payload: CleanupJobPayload): Promise `[${label}] Cleaning ${workspaceIds.length} workspaces, cutoff: ${retentionDate.toISOString()}` ) - const workflowResults = await cleanupWorkflowExecutionLogs(workspaceIds, retentionDate, label) + const workflowResults = await cleanupWorkflowExecutionLogs( + workspaceIds, + retentionDate, + label, + budgets?.workflowLogs + ) logger.info( `[${label}] workflow_execution_logs files: ${workflowResults.filesDeleted}/${workflowResults.filesTotal} deleted, ${workflowResults.filesDeleteFailed} failed` ) - const largeValueResults = await cleanupLargeExecutionValues(workspaceIds, retentionDate, label) + if (budgets && workflowResults.filesDeleteFailed) throw new Error('Log file cleanup failed') + const largeValueResults = await cleanupLargeExecutionValues( + workspaceIds, + retentionDate, + label, + budgets?.largeValues + ) logger.info( `[${label}] execution_large_values: ${largeValueResults.largeValuesDeleted}/${largeValueResults.largeValuesTotal} deleted, ${largeValueResults.largeValuesDeleteFailed} failed` ) const legacyLargeValueResults = await cleanupLegacyLargeExecutionValues( workspaceIds, retentionDate, - label + label, + budgets?.legacyLargeValues ) logger.info( `[${label}] legacy_execution_large_values: ${legacyLargeValueResults.largeValuesDeleted}/${legacyLargeValueResults.largeValuesTotal} deleted, ${legacyLargeValueResults.largeValuesDeleteFailed} failed` ) - await cleanupLargeValueMetadata(workspaceIds, label) + await cleanupLargeValueMetadata(workspaceIds, label, budgets) await batchDeleteByWorkspaceAndTimestamp({ + budget: budgets?.jobLogs, tableDef: jobExecutionLogs, workspaceIdCol: jobExecutionLogs.workspaceId, timestampCol: jobExecutionLogs.startedAt, @@ -477,7 +519,7 @@ export async function runCleanupLogs(payload: CleanupJobPayload): Promise }) if (runGlobalHousekeeping && plan === 'free') { - await cleanupFreePlanOrphanedSnapshots(retentionHours) + await cleanupFreePlanOrphanedSnapshots(retentionHours, budgets?.orphanSnapshots) } const timeElapsed = (Date.now() - startTime) / 1000 @@ -487,6 +529,10 @@ export async function runCleanupLogs(payload: CleanupJobPayload): Promise export const cleanupLogsTask = task({ id: 'cleanup-logs', machine: 'large-1x', - queue: { concurrencyLimit: LOG_CLEANUP_CONCURRENCY_LIMIT }, - run: runCleanupLogs, + queue: retentionCleanupQueue, + retry: { maxAttempts: 1 }, + run: (payload: CleanupJobPayload | LimitedCleanupPayload) => + 'limits' in payload + ? runCleanupWithLimits('cleanup-logs', payload.limits, runCleanupLogs) + : runCleanupLogs(payload), }) diff --git a/apps/sim/background/cleanup-soft-deletes.test.ts b/apps/sim/background/cleanup-soft-deletes.test.ts index 2d6816d525a..cb9d262cdf4 100644 --- a/apps/sim/background/cleanup-soft-deletes.test.ts +++ b/apps/sim/background/cleanup-soft-deletes.test.ts @@ -48,7 +48,10 @@ const { mockSelectRowsByIdChunks: vi.fn(async () => [] as unknown[]), })) +vi.mock('@/lib/billing/cleanup-dispatcher', () => ({ runCleanupWithLimits: vi.fn() })) + vi.mock('@/lib/cleanup/batch-delete', () => ({ + consumeRowBudget: vi.fn(), batchDeleteByWorkspaceAndTimestamp: mockBatchDeleteByWorkspaceAndTimestamp, chunkedBatchDelete: mockChunkedBatchDelete, chunkedBatchDeleteByScope: mockScopedChunkedBatchDelete, @@ -57,6 +60,10 @@ vi.mock('@/lib/cleanup/batch-delete', () => ({ selectRowsByIdChunks: mockSelectRowsByIdChunks, })) +vi.mock('@/lib/cleanup/queue', () => ({ + retentionCleanupQueue: { name: 'retention-cleanup', concurrencyLimit: 1 }, +})) + vi.mock('@/lib/cleanup/chat-cleanup', () => ({ prepareChatCleanup: mockPrepareChatCleanup })) vi.mock('@/lib/billing/storage', () => ({ diff --git a/apps/sim/background/cleanup-soft-deletes.ts b/apps/sim/background/cleanup-soft-deletes.ts index 2f48f1d51d7..5497948fd26 100644 --- a/apps/sim/background/cleanup-soft-deletes.ts +++ b/apps/sim/background/cleanup-soft-deletes.ts @@ -16,7 +16,7 @@ import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { task } from '@trigger.dev/sdk' import { and, asc, eq, inArray, isNotNull, isNull, lt, sql } from 'drizzle-orm' -import type { CleanupJobPayload } from '@/lib/billing/cleanup-dispatcher' +import { type CleanupJobPayload, runCleanupWithLimits } from '@/lib/billing/cleanup-dispatcher' import { decrementStorageUsageForBillingContextInTx, resolveStorageBillingContext, @@ -26,10 +26,14 @@ import { batchDeleteByWorkspaceAndTimestamp, chunkedBatchDelete, chunkedBatchDeleteByScope, + consumeRowBudget, DEFAULT_DELETE_CHUNK_SIZE, + type RowBudget, selectRowsByIdChunks, } from '@/lib/cleanup/batch-delete' import { prepareChatCleanup } from '@/lib/cleanup/chat-cleanup' +import type { CleanupBudgets, LimitedCleanupPayload } from '@/lib/cleanup/limits' +import { retentionCleanupQueue } from '@/lib/cleanup/queue' import { type CleanupOwnerScope, cleanupOwnerCondition, @@ -93,47 +97,54 @@ interface WorkspaceFileStorageCleanupResult { */ async function selectExpiredWorkspaceFiles( scope: CleanupOwnerScope, - retentionDate: Date + retentionDate: Date, + budgets?: CleanupBudgets ): Promise { const [legacyRows, multiContextRows] = await Promise.all([ - selectRowsByIdChunks(scope.kind === 'workspace' ? scope.ids : [], (chunkIds, chunkLimit) => - cleanupDb - .select({ - id: workspaceFile.id, - key: workspaceFile.key, - workspaceId: workspaceFile.workspaceId, - }) - .from(workspaceFile) - .where( - and( - inArray(workspaceFile.workspaceId, chunkIds), - isNotNull(workspaceFile.deletedAt), - lt(workspaceFile.deletedAt, retentionDate) + selectRowsByIdChunks( + scope.kind === 'workspace' ? scope.ids : [], + (chunkIds, chunkLimit) => + cleanupDb + .select({ + id: workspaceFile.id, + key: workspaceFile.key, + workspaceId: workspaceFile.workspaceId, + }) + .from(workspaceFile) + .where( + and( + inArray(workspaceFile.workspaceId, chunkIds), + isNotNull(workspaceFile.deletedAt), + lt(workspaceFile.deletedAt, retentionDate) + ) ) - ) - .limit(chunkLimit) + .limit(chunkLimit), + { budget: budgets?.legacyFiles } ), - selectRowsByIdChunks(scope.ids, (chunkIds, chunkLimit) => - cleanupDb - .select({ - id: workspaceFiles.id, - key: workspaceFiles.key, - workspaceId: workspaceFiles.workspaceId, - context: workspaceFiles.context, - sizeBytes: workspaceFiles.sizeBytes, - }) - .from(workspaceFiles) - .where( - and( - cleanupOwnerCondition(workspaceFiles, scope, chunkIds), - scope.kind === 'organization' - ? eq(workspaceFiles.context, 'knowledge-base') - : undefined, - isNotNull(workspaceFiles.deletedAt), - lt(workspaceFiles.deletedAt, retentionDate) + selectRowsByIdChunks( + scope.ids, + (chunkIds, chunkLimit) => + cleanupDb + .select({ + id: workspaceFiles.id, + key: workspaceFiles.key, + workspaceId: workspaceFiles.workspaceId, + context: workspaceFiles.context, + sizeBytes: workspaceFiles.sizeBytes, + }) + .from(workspaceFiles) + .where( + and( + cleanupOwnerCondition(workspaceFiles, scope, chunkIds), + scope.kind === 'organization' + ? eq(workspaceFiles.context, 'knowledge-base') + : undefined, + isNotNull(workspaceFiles.deletedAt), + lt(workspaceFiles.deletedAt, retentionDate) + ) ) - ) - .limit(chunkLimit) + .limit(chunkLimit), + { budget: budgets?.files } ), ]) @@ -393,9 +404,11 @@ async function hardDeleteKnowledgeBaseDocuments( async function cleanupExpiredKnowledgeBases( scope: CleanupOwnerScope, retentionDate: Date, - label: string + label: string, + budget?: RowBudget ) { const options = { + budget, tableDef: knowledgeBase, tableName: `${label}/knowledgeBase`, batchSize: KB_RETENTION_BATCH_SIZE, @@ -687,25 +700,35 @@ const CLEANUP_TARGETS = [ ctx.retentionDate, ctx.label ), + budgetKey: 'folders', name: 'folder', }, { table: userTableDefinitions, softDeleteCol: userTableDefinitions.archivedAt, wsCol: userTableDefinitions.workspaceId, + budgetKey: 'userTables', name: 'userTableDefinitions', }, - { table: memory, softDeleteCol: memory.deletedAt, wsCol: memory.workspaceId, name: 'memory' }, + { + table: memory, + softDeleteCol: memory.deletedAt, + wsCol: memory.workspaceId, + budgetKey: 'memories', + name: 'memory', + }, { table: mcpServers, softDeleteCol: mcpServers.deletedAt, wsCol: mcpServers.workspaceId, + budgetKey: 'mcpServers', name: 'mcpServers', }, { table: workflowMcpServer, softDeleteCol: workflowMcpServer.deletedAt, wsCol: workflowMcpServer.workspaceId, + budgetKey: 'workflowMcpServers', name: 'workflowMcpServer', }, ] as const @@ -721,7 +744,8 @@ const CLEANUP_TARGETS = [ */ async function cleanupOrphanedKnowledgeBaseBindings( scope: CleanupOwnerScope, - label: string + label: string, + budget?: RowBudget ): Promise<{ total: number; deleted: number; failed: number }> { const stats = { total: 0, deleted: 0, failed: 0 } if (scope.ids.length === 0) return stats @@ -730,10 +754,11 @@ async function cleanupOrphanedKnowledgeBaseBindings( for (const chunkIds of chunkArray(scope.ids, KB_ORPHAN_BINDING_OWNER_CHUNK_SIZE)) { let attempted = 0 - while (attempted < KB_ORPHAN_BINDING_TOTAL_LIMIT) { + while (attempted < KB_ORPHAN_BINDING_TOTAL_LIMIT && budget?.remaining !== 0) { const limit = Math.min( KB_ORPHAN_BINDING_BATCH_SIZE, - KB_ORPHAN_BINDING_TOTAL_LIMIT - attempted + KB_ORPHAN_BINDING_TOTAL_LIMIT - attempted, + budget?.remaining ?? KB_ORPHAN_BINDING_BATCH_SIZE ) const rows = await cleanupDb .select({ key: workspaceFiles.key }) @@ -755,6 +780,7 @@ async function cleanupOrphanedKnowledgeBaseBindings( if (rows.length === 0) break + consumeRowBudget(budget, rows.length) const keys = rows.map((row) => row.key) stats.total += keys.length attempted += keys.length @@ -781,6 +807,7 @@ async function cleanupOrphanedKnowledgeBaseBindings( } } stats.deleted += deletedThisBatch + if (budget && stats.failed) throw new Error('Orphan binding cleanup failed') // No progress (every delete failed) — stop rather than reselect the same rows. if (deletedThisBatch === 0) break @@ -793,7 +820,10 @@ async function cleanupOrphanedKnowledgeBaseBindings( return stats } -export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise { +export async function runCleanupSoftDeletes( + payload: CleanupJobPayload, + budgets?: CleanupBudgets +): Promise { const startTime = Date.now() const { workspaceIds, retentionHours, label } = payload const scope = resolveCleanupOwnerScope(payload) @@ -813,32 +843,38 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise // could return different subsets above the LIMIT cap and orphan or // prematurely purge data. const [doomedWorkflows, fileScope, expiredSoftDeletedChats] = await Promise.all([ - selectRowsByIdChunks(workspaceIds, (chunkIds, chunkLimit) => - cleanupDb - .select({ id: workflow.id }) - .from(workflow) - .where( - and( - inArray(workflow.workspaceId, chunkIds), - isNotNull(workflow.archivedAt), - lt(workflow.archivedAt, retentionDate) + selectRowsByIdChunks( + workspaceIds, + (chunkIds, chunkLimit) => + cleanupDb + .select({ id: workflow.id }) + .from(workflow) + .where( + and( + inArray(workflow.workspaceId, chunkIds), + isNotNull(workflow.archivedAt), + lt(workflow.archivedAt, retentionDate) + ) ) - ) - .limit(chunkLimit) + .limit(chunkLimit), + { budget: budgets?.workflows } ), - selectExpiredWorkspaceFiles(scope, retentionDate), - selectRowsByIdChunks(scope.ids, (chunkIds, chunkLimit) => - cleanupDb - .select({ id: copilotChats.id }) - .from(copilotChats) - .where( - and( - cleanupOwnerCondition(copilotChats, scope, chunkIds), - isNotNull(copilotChats.deletedAt), - lt(copilotChats.deletedAt, retentionDate) + selectExpiredWorkspaceFiles(scope, retentionDate, budgets), + selectRowsByIdChunks( + scope.ids, + (chunkIds, chunkLimit) => + cleanupDb + .select({ id: copilotChats.id }) + .from(copilotChats) + .where( + and( + cleanupOwnerCondition(copilotChats, scope, chunkIds), + isNotNull(copilotChats.deletedAt), + lt(copilotChats.deletedAt, retentionDate) + ) ) - ) - .limit(chunkLimit) + .limit(chunkLimit), + { budget: budgets?.chats } ), ]) @@ -864,6 +900,7 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise } const fileCleanup = await cleanupWorkspaceFileStorage(fileScope) + if (budgets && fileCleanup.filesFailed) throw new Error('File storage cleanup failed') let totalDeleted = 0 @@ -886,6 +923,7 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise .returning({ id: workflow.id }) totalDeleted += deleted.length } catch (error) { + if (budgets) throw error logger.error(`[${label}/workflow] Archived workflow delete failed`, { error }) } } @@ -910,6 +948,7 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise .returning({ id: copilotChats.id }) totalDeleted += deleted.length } catch (error) { + if (budgets) throw error logger.error(`[${label}/copilotChats] Soft-deleted chat delete failed`, { error }) } } @@ -934,12 +973,23 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise label ) totalDeleted += unbilledFileResult.deleted + if ( + budgets && + (legacyFileResult.failed || billableFileResult.failed || unbilledFileResult.failed) + ) + throw new Error('File row cleanup failed') - const knowledgeBaseResult = await cleanupExpiredKnowledgeBases(scope, retentionDate, label) + const knowledgeBaseResult = await cleanupExpiredKnowledgeBases( + scope, + retentionDate, + label, + budgets?.knowledgeBases + ) totalDeleted += knowledgeBaseResult.deleted for (const target of scope.kind === 'workspace' ? CLEANUP_TARGETS : []) { const result = await batchDeleteByWorkspaceAndTimestamp({ + budget: budgets?.[target.budgetKey], tableDef: target.table, workspaceIdCol: target.wsCol, timestampCol: target.softDeleteCol, @@ -957,7 +1007,11 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise totalDeleted += result.deleted } - const orphanBindingStats = await cleanupOrphanedKnowledgeBaseBindings(scope, label) + const orphanBindingStats = await cleanupOrphanedKnowledgeBaseBindings( + scope, + label, + budgets?.orphanKnowledgeBaseBindings + ) logger.info( `[${label}] Complete: ${totalDeleted} rows deleted, ${fileCleanup.filesDeleted} files cleaned, ${orphanBindingStats.deleted} orphan KB bindings cleaned` @@ -975,6 +1029,10 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise export const cleanupSoftDeletesTask = task({ id: 'cleanup-soft-deletes', machine: 'large-1x', - queue: { concurrencyLimit: 5 }, - run: runCleanupSoftDeletes, + queue: retentionCleanupQueue, + retry: { maxAttempts: 1 }, + run: (payload: CleanupJobPayload | LimitedCleanupPayload) => + 'limits' in payload + ? runCleanupWithLimits('cleanup-soft-deletes', payload.limits, runCleanupSoftDeletes) + : runCleanupSoftDeletes(payload), }) diff --git a/apps/sim/lib/api/contracts/cleanup.test.ts b/apps/sim/lib/api/contracts/cleanup.test.ts new file mode 100644 index 00000000000..4a7ae0b9b6d --- /dev/null +++ b/apps/sim/lib/api/contracts/cleanup.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest' +import { logsCleanupQuerySchema, softDeletesCleanupQuerySchema } from '@/lib/api/contracts/cleanup' + +describe('cleanup query limits', () => { + it('preserves scheduled calls without parameters', () => { + expect(logsCleanupQuerySchema.parse({})).toBeUndefined() + }) + it('parses per-type counts', () => { + expect(logsCleanupQuerySchema.parse({ workflowLogs: '25', jobLogs: '0' })).toEqual({ + workflowLogs: 25, + jobLogs: 0, + }) + expect(softDeletesCleanupQuerySchema.parse({ files: '1' })).toEqual({ files: 1 }) + }) + it.each(['', '-1', '1.5', '5001', 'abc', '0'])('rejects invalid count %s', (value) => { + expect(logsCleanupQuerySchema.safeParse({ workflowLogs: value }).success).toBe(false) + }) + it('rejects unknown or wrong-endpoint types', () => { + expect(logsCleanupQuerySchema.safeParse({ files: '1' }).success).toBe(false) + expect(softDeletesCleanupQuerySchema.safeParse({ workflowLogs: '1' }).success).toBe(false) + }) +}) diff --git a/apps/sim/lib/api/contracts/cleanup.ts b/apps/sim/lib/api/contracts/cleanup.ts new file mode 100644 index 00000000000..2a7f9c81111 --- /dev/null +++ b/apps/sim/lib/api/contracts/cleanup.ts @@ -0,0 +1,68 @@ +import { z } from 'zod' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { + type CleanupLimits, + type CleanupType, + LOG_CLEANUP_TYPES, + SOFT_DELETE_CLEANUP_TYPES, +} from '@/lib/cleanup/limits' + +const limitSchema = z + .union([z.number(), z.string().regex(/^\d+$/).transform(Number)]) + .pipe(z.number().int().min(0).max(5000)) +function cleanupQuerySchema(types: readonly CleanupType[]) { + return z + .object(Object.fromEntries(types.map((type) => [type, limitSchema.optional()]))) + .strict() + .transform((query, ctx): CleanupLimits | undefined => { + if (Object.keys(query).length === 0) return undefined + if (!Object.values(query).some((limit) => limit !== undefined && limit > 0)) { + ctx.addIssue({ code: 'custom', message: 'At least one positive cleanup limit is required' }) + return z.NEVER + } + return query + }) +} +export const logsCleanupQuerySchema = cleanupQuerySchema(LOG_CLEANUP_TYPES) +export const softDeletesCleanupQuerySchema = cleanupQuerySchema(SOFT_DELETE_CLEANUP_TYPES) +const responseSchema = z.union([ + z.object({ + triggered: z.literal(true), + jobIds: z.array(z.string()), + jobCount: z.number(), + chunkCount: z.number(), + workspaceCount: z.number(), + }), + z.object({ + triggered: z.literal(true), + runId: z.string(), + limits: z.partialRecord( + z.enum([...LOG_CLEANUP_TYPES, ...SOFT_DELETE_CLEANUP_TYPES]), + z.number().int().min(0).max(5000) + ), + }), +]) +export const logsCleanupContract = defineRouteContract({ + method: 'GET', + path: '/api/logs/cleanup', + query: logsCleanupQuerySchema, + response: { mode: 'json', schema: responseSchema, status: [200, 202] }, +}) +export const softDeletesCleanupContract = defineRouteContract({ + method: 'GET', + path: '/api/cron/cleanup-soft-deletes', + query: softDeletesCleanupQuerySchema, + response: { mode: 'json', schema: responseSchema, status: [200, 202] }, +}) + +/** Apply the same bounds to direct task submissions as HTTP requests. */ +export function validateCleanupLimits( + jobType: 'cleanup-logs' | 'cleanup-soft-deletes', + limits: CleanupLimits +): CleanupLimits { + const parsed = ( + jobType === 'cleanup-logs' ? logsCleanupQuerySchema : softDeletesCleanupQuerySchema + ).parse(limits) + if (!parsed) throw new Error('Cleanup limits are required') + return parsed +} diff --git a/apps/sim/lib/billing/cleanup-dispatcher.test.ts b/apps/sim/lib/billing/cleanup-dispatcher.test.ts index fbf6a593652..f6df7c6f5a3 100644 --- a/apps/sim/lib/billing/cleanup-dispatcher.test.ts +++ b/apps/sim/lib/billing/cleanup-dispatcher.test.ts @@ -36,7 +36,14 @@ vi.mock('@/lib/workspaces/policy', () => ({ isOrganizationWorkspace: vi.fn(), })) -import { dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher' +import { tasks } from '@trigger.dev/sdk' +import { + dispatchBoundedCleanup, + dispatchCleanupJobs, + runCleanupWithLimits, +} from '@/lib/billing/cleanup-dispatcher' +import { getHighestPriorityPersonalSubscription } from '@/lib/billing/core/subscription' +import { isOrganizationWorkspace } from '@/lib/workspaces/policy' afterAll(resetEnvFlagsMock) @@ -222,3 +229,103 @@ describe('organization-owned Search retention dispatch', () => { expect(mockEnqueue).not.toHaveBeenCalled() }) }) + +describe('cleanup limits', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + setEnvFlags({ isBillingEnabled: false, isDataRetentionEnabled: true }) + mockIsTriggerAvailable.mockReturnValue(true) + vi.mocked(getHighestPriorityPersonalSubscription).mockReset() + mockGetOrganizationSubscription.mockReset() + vi.mocked(isOrganizationWorkspace).mockReset() + }) + + it('enqueues one job without querying owners or dispatching child jobs', async () => { + vi.mocked(tasks.trigger).mockResolvedValueOnce({ id: 'run-limited' } as never) + expect(await dispatchBoundedCleanup('cleanup-logs', { workflowLogs: 3 })).toEqual({ + triggered: true, + runId: 'run-limited', + limits: { workflowLogs: 3 }, + }) + expect(tasks.trigger).toHaveBeenCalledWith( + 'cleanup-logs', + { limits: { workflowLogs: 3 } }, + expect.objectContaining({ maxAttempts: 1 }) + ) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + expect(tasks.batchTrigger).not.toHaveBeenCalled() + }) + + it('shares budgets across owners and stops before the next page', async () => { + queueTableRows( + schemaMock.workspace, + ['a', 'b', 'c'].map((id) => ({ + id, + billedAccountUserId: 'user', + organizationId: null, + workspaceMode: 'personal', + organizationSettings: { logRetentionHours: 24 }, + })) + ) + const seen: number[] = [] + await runCleanupWithLimits('cleanup-logs', { workflowLogs: 2 }, async (_scope, budgets) => { + seen.push(budgets.workflowLogs.remaining) + expect(budgets.jobLogs.remaining).toBe(0) + budgets.workflowLogs.remaining-- + }) + expect(seen).toEqual([2, 1]) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + }) + + it('rejects invalid direct task input before querying', async () => { + await expect( + runCleanupWithLimits('cleanup-logs', { workflowLogs: -1 }, vi.fn()) + ).rejects.toThrow() + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it.each(['personal', 'organization-workspace', 'organization'] as const)( + 'fails a manual job when %s subscription lookup fails', + async (kind) => { + setEnvFlags({ isBillingEnabled: true }) + const error = new Error('subscription lookup unavailable') + vi.mocked(getHighestPriorityPersonalSubscription).mockRejectedValueOnce(error) + mockGetOrganizationSubscription.mockRejectedValueOnce(error) + vi.mocked(isOrganizationWorkspace).mockReturnValue(true) + queueTableRows( + schemaMock.workspace, + kind === 'organization' + ? [] + : [ + { + id: 'workspace', + billedAccountUserId: 'user', + organizationId: 'organization', + workspaceMode: kind === 'personal' ? 'personal' : 'organization', + organizationSettings: null, + }, + ] + ) + if (kind === 'organization') + queueTableRows(schemaMock.organization, [{ id: 'organization', settings: null }]) + const runScope = vi.fn() + await expect( + runCleanupWithLimits('cleanup-soft-deletes', { files: 1 }, runScope) + ).rejects.toBe(error) + expect(runScope).not.toHaveBeenCalled() + } + ) + + it('requires queued execution and respects the retention switch', async () => { + mockIsTriggerAvailable.mockReturnValue(false) + await expect(dispatchBoundedCleanup('cleanup-logs', { workflowLogs: 1 })).rejects.toThrow( + 'requires Trigger.dev' + ) + setEnvFlags({ isDataRetentionEnabled: false }) + await expect( + runCleanupWithLimits('cleanup-logs', { workflowLogs: 1 }, vi.fn()) + ).rejects.toThrow('retention is disabled') + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/billing/cleanup-dispatcher.ts b/apps/sim/lib/billing/cleanup-dispatcher.ts index d70bc439369..ea9551c3209 100644 --- a/apps/sim/lib/billing/cleanup-dispatcher.ts +++ b/apps/sim/lib/billing/cleanup-dispatcher.ts @@ -5,10 +5,12 @@ import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { tasks } from '@trigger.dev/sdk' import { and, asc, eq, gt, isNull } from 'drizzle-orm' +import { validateCleanupLimits } from '@/lib/api/contracts/cleanup' import { getOrganizationSubscription } from '@/lib/billing/core/billing' import { getHighestPriorityPersonalSubscription } from '@/lib/billing/core/subscription' import { getPlanType, type PlanCategory } from '@/lib/billing/plan-helpers' import { type RetentionHoursKey, resolveEffectiveRetentionHours } from '@/lib/billing/retention' +import { type CleanupBudgets, type CleanupLimits, createCleanupBudgets } from '@/lib/cleanup/limits' import { getJobQueue } from '@/lib/core/async-jobs' import { shouldExecuteInline } from '@/lib/core/async-jobs/config' import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' @@ -60,8 +62,8 @@ const DAY = 24 type PlanResolutionEntry = readonly [string, PlanCategory] -function getCleanupConcurrencyKey(jobType: CleanupJobType): string { - return `cleanup:${jobType}` +function getCleanupConcurrencyKey(jobType: CleanupJobType): string | undefined { + return jobType === 'cleanup-tasks' ? `cleanup:${jobType}` : undefined } /** @@ -86,7 +88,8 @@ export const CLEANUP_CONFIG = { } as const satisfies Record async function listActiveWorkspaceCleanupScopeRowsPage( - afterId: string | null + afterId: string | null, + pageSize: number ): Promise { const rows = await db .select({ @@ -104,7 +107,7 @@ async function listActiveWorkspaceCleanupScopeRowsPage( : isNull(workspace.archivedAt) ) .orderBy(asc(workspace.id)) - .limit(WORKSPACE_SCOPE_PAGE_SIZE) + .limit(pageSize) return rows.map((row) => ({ ...row, @@ -113,7 +116,8 @@ async function listActiveWorkspaceCleanupScopeRowsPage( } async function resolvePersonalPlanTypesByBilledUserId( - rows: WorkspaceCleanupScopeRow[] + rows: WorkspaceCleanupScopeRow[], + failOnLookupError: boolean ): Promise> { const billedUserIds = Array.from(new Set(rows.map((row) => row.billedAccountUserId))) const entries = await Promise.all( @@ -124,6 +128,7 @@ async function resolvePersonalPlanTypesByBilledUserId( }) return [userId, getPlanType(subscription?.plan)] as const } catch (error) { + if (failOnLookupError) throw error logger.error('Skipping cleanup for billed user after plan lookup failed', { userId, error, @@ -137,7 +142,8 @@ async function resolvePersonalPlanTypesByBilledUserId( } async function resolvePlanTypesByWorkspaceId( - rows: WorkspaceCleanupScopeRow[] + rows: WorkspaceCleanupScopeRow[], + failOnLookupError: boolean ): Promise> { /** * Without billing there are no subscription rows to read, and the per-plan @@ -156,12 +162,16 @@ async function resolvePlanTypesByWorkspaceId( } const userScopedRows = rows.filter((row) => row.workspaceMode !== WORKSPACE_MODE.ORGANIZATION) - const userPlanByBilledUserId = await resolvePersonalPlanTypesByBilledUserId(userScopedRows) + const userPlanByBilledUserId = await resolvePersonalPlanTypesByBilledUserId( + userScopedRows, + failOnLookupError + ) const entries = await Promise.all( rows.map(async (row) => { if (row.workspaceMode === WORKSPACE_MODE.ORGANIZATION) { const organizationId = isOrganizationWorkspace(row) ? row.organizationId : null if (!organizationId) { + if (failOnLookupError) throw new Error('Malformed organization workspace') logger.error('Skipping cleanup for malformed organization workspace', { workspaceId: row.id, organizationId: row.organizationId, @@ -183,6 +193,7 @@ async function resolvePlanTypesByWorkspaceId( return [row.id, getPlanType(subscription?.plan)] as const } catch (error) { + if (failOnLookupError) throw error logger.error('Skipping cleanup for organization workspace after plan lookup failed', { workspaceId: row.id, organizationId, @@ -225,7 +236,16 @@ const GLOBAL_HOUSEKEEPING_PLAN: Partial> = async function forEachCleanupChunk( jobType: CleanupJobType, - onChunk: (payload: CleanupJobPayload) => Promise + onChunk: (payload: CleanupJobPayload) => Promise, + { + shouldStop = () => false, + pageSize = WORKSPACE_SCOPE_PAGE_SIZE, + failOnLookupError = false, + }: { + shouldStop?: () => boolean + pageSize?: number + failOnLookupError?: boolean + } = {} ): Promise<{ chunkCount: number; workspaceCount: number }> { const config = CLEANUP_CONFIG[jobType] const chunkCountByPlan: Partial> = {} @@ -236,6 +256,7 @@ async function forEachCleanupChunk( let afterId: string | null = null const emitChunk = async (payload: CleanupJobPayload) => { + if (shouldStop()) return if (payload.plan === housekeepingPlan && !housekeepingAssigned) { payload.runGlobalHousekeeping = true housekeepingAssigned = true @@ -244,12 +265,12 @@ async function forEachCleanupChunk( await onChunk(payload) } - while (true) { - const rows = await listActiveWorkspaceCleanupScopeRowsPage(afterId) + while (!shouldStop()) { + const rows = await listActiveWorkspaceCleanupScopeRowsPage(afterId, pageSize) if (rows.length === 0) break afterId = rows[rows.length - 1].id - const planByWorkspaceId = await resolvePlanTypesByWorkspaceId(rows) + const planByWorkspaceId = await resolvePlanTypesByWorkspaceId(rows, failOnLookupError) for (const plan of NON_ENTERPRISE_PLANS) { const retentionHours = config.defaults[plan] @@ -294,16 +315,17 @@ async function forEachCleanupChunk( if (jobType === 'cleanup-soft-deletes' || jobType === 'cleanup-tasks') { let afterOrganizationId: string | null = null - while (true) { + while (!shouldStop()) { const organizations = await db .select({ id: organization.id, settings: organization.dataRetentionSettings }) .from(organization) .where(afterOrganizationId ? gt(organization.id, afterOrganizationId) : undefined) .orderBy(asc(organization.id)) - .limit(WORKSPACE_SCOPE_PAGE_SIZE) + .limit(pageSize) if (organizations.length === 0) break afterOrganizationId = organizations[organizations.length - 1].id for (const row of organizations) { + if (shouldStop()) break let plan: PlanCategory = 'enterprise' if (isBillingEnabled) { try { @@ -311,6 +333,7 @@ async function forEachCleanupChunk( if (!subscription) continue plan = getPlanType(subscription.plan) } catch (error) { + if (failOnLookupError) throw error logger.error('Skipping organization cleanup after plan lookup failed', { organizationId: row.id, error, @@ -463,3 +486,38 @@ export async function dispatchCleanupJobs(jobType: CleanupJobType): Promise<{ return { jobIds, jobCount: jobIds.length, chunkCount, workspaceCount } } + +/** Enqueue one job; owner discovery and all deletion happen in the worker. */ +export async function dispatchBoundedCleanup( + jobType: 'cleanup-logs' | 'cleanup-soft-deletes', + input: CleanupLimits +) { + const limits = validateCleanupLimits(jobType, input) + if (!isBillingEnabled && !isDataRetentionEnabled) throw new Error('Data retention is disabled') + if (!isTriggerAvailable()) throw new Error('Queued cleanup requires Trigger.dev') + const run = await tasks.trigger( + jobType, + { limits }, + { + maxAttempts: 1, + region: await resolveTriggerRegion(), + } + ) + return { triggered: true as const, runId: run.id, limits } +} + +/** Reuse existing cleanup functions with one budget across all workspace/organization chunks. */ +export async function runCleanupWithLimits( + jobType: 'cleanup-logs' | 'cleanup-soft-deletes', + input: CleanupLimits, + runScope: (payload: CleanupJobPayload, budgets: CleanupBudgets) => Promise +): Promise { + const limits = validateCleanupLimits(jobType, input) + if (!isBillingEnabled && !isDataRetentionEnabled) throw new Error('Data retention is disabled') + const budgets = createCleanupBudgets(limits) + await forEachCleanupChunk(jobType, (scope) => runScope(scope, budgets), { + shouldStop: () => Object.values(budgets).every((budget) => budget.remaining === 0), + pageSize: 25, + failOnLookupError: true, + }) +} diff --git a/apps/sim/lib/cleanup/batch-delete.test.ts b/apps/sim/lib/cleanup/batch-delete.test.ts index 5e25ce1307f..724e95ce39d 100644 --- a/apps/sim/lib/cleanup/batch-delete.test.ts +++ b/apps/sim/lib/cleanup/batch-delete.test.ts @@ -4,7 +4,11 @@ import { schemaMock } from '@sim/testing' import { describe, expect, it, vi } from 'vitest' -import { batchDeleteByWorkspaceAndTimestamp, chunkedBatchDelete } from '@/lib/cleanup/batch-delete' +import { + batchDeleteByWorkspaceAndTimestamp, + chunkedBatchDelete, + selectRowsByIdChunks, +} from '@/lib/cleanup/batch-delete' /** * Minimal stand-in for the drizzle client `chunkedBatchDelete` calls. Only the DELETE path is @@ -78,3 +82,72 @@ describe('chunkedBatchDelete onBatch contract', () => { expect(order[0]).toBe('onBatch') }) }) + +describe('shared cleanup row budgets', () => { + it('caps selection across ID chunks and subsequent owner scopes', async () => { + const budget = { remaining: 3 } + const select = vi.fn(async (_ids: string[], limit: number) => + [{ id: 'one' }, { id: 'two' }].slice(0, limit) + ) + expect(await selectRowsByIdChunks(['a', 'b'], select, { chunkSize: 1, budget })).toHaveLength(3) + expect(select.mock.calls.map(([, limit]) => limit)).toEqual([3, 1]) + expect(await selectRowsByIdChunks(['c'], select, { budget })).toEqual([]) + expect(select).toHaveBeenCalledTimes(2) + }) + + it('charges restored rows as attempts and uses the remaining limit for each delete batch', async () => { + const budget = { remaining: 3 } + const select = vi.fn(async (_ids: string[], limit: number) => + [{ id: 'one' }, { id: 'two' }].slice(0, limit) + ) + const options = { + tableDef: schemaMock.folder as never, + workspaceIds: ['a'], + tableName: 'folder', + dbClient: createDbClient(() => {}), + selectChunk: select, + budget, + batchSize: 2, + } + const result = await chunkedBatchDelete(options) + expect(result).toMatchObject({ deleted: 2, failed: 1 }) + expect(select.mock.calls.map(([, limit]) => limit)).toEqual([2, 1]) + await chunkedBatchDelete({ ...options, workspaceIds: ['b'] }) + expect(select).toHaveBeenCalledTimes(2) + }) + + it('stops on an error after charging selected rows', async () => { + const budget = { remaining: 2 } + const onDelete = vi.fn() + await expect( + chunkedBatchDelete({ + tableDef: schemaMock.folder as never, + workspaceIds: ['a', 'b'], + tableName: 'folder', + budget, + dbClient: createDbClient(onDelete), + selectChunk: async () => [{ id: 'one' }], + onBatch: async () => { + throw new Error('storage failed') + }, + }) + ).rejects.toThrow('storage failed') + expect(budget.remaining).toBe(1) + expect(onDelete).not.toHaveBeenCalled() + }) + + it('passes budgets through the timestamp helper', async () => { + const onDelete = vi.fn() + await batchDeleteByWorkspaceAndTimestamp({ + tableDef: schemaMock.folder as never, + workspaceIdCol: schemaMock.folder.workspaceId as never, + timestampCol: schemaMock.folder.deletedAt as never, + workspaceIds: ['a'], + retentionDate: new Date(0), + tableName: 'folder', + budget: { remaining: 0 }, + dbClient: createDbClient(onDelete, [{ id: 'one' }]), + }) + expect(onDelete).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/cleanup/batch-delete.ts b/apps/sim/lib/cleanup/batch-delete.ts index 468a1f2619f..cd6506d174c 100644 --- a/apps/sim/lib/cleanup/batch-delete.ts +++ b/apps/sim/lib/cleanup/batch-delete.ts @@ -26,7 +26,19 @@ export const DEFAULT_WORKSPACE_CHUNK_SIZE = 50 /** Bounds FK cascade trigger queue (per-statement in-memory) and bind-parameter count. */ export const DEFAULT_DELETE_CHUNK_SIZE = 1000 +export interface RowBudget { + remaining: number +} + +/** Charge selected roots before side effects, including rows that fail or are restored. */ +export function consumeRowBudget(budget: RowBudget | undefined, count: number): void { + if (!budget) return + if (count > budget.remaining) throw new Error('Cleanup selection exceeded its row budget') + budget.remaining -= count +} + export interface SelectByIdChunksOptions { + budget?: RowBudget /** Cap on rows returned across all chunks. Defaults to a full per-table cleanup budget. */ overallLimit?: number chunkSize?: number @@ -48,6 +60,7 @@ export async function selectRowsByIdChunks( { overallLimit = DEFAULT_BATCH_SIZE * DEFAULT_MAX_BATCHES_PER_TABLE, chunkSize = DEFAULT_WORKSPACE_CHUNK_SIZE, + budget, }: SelectByIdChunksOptions = {} ): Promise { if (ids.length === 0) return [] @@ -55,8 +68,10 @@ export async function selectRowsByIdChunks( const rows: T[] = [] for (const chunkIds of chunkArray(ids, chunkSize)) { if (rows.length >= overallLimit) break - const remaining = overallLimit - rows.length + const remaining = Math.min(overallLimit - rows.length, budget?.remaining ?? overallLimit) + if (remaining === 0) break const chunkRows = await query(chunkIds, remaining) + consumeRowBudget(budget, chunkRows.length) rows.push(...chunkRows) } return rows @@ -69,6 +84,7 @@ export interface TableCleanupResult { } export interface ChunkedBatchDeleteOptions { + budget?: RowBudget tableDef: PgTable workspaceIds: string[] tableName: string @@ -132,6 +148,7 @@ export interface ScopedChunkedBatchDeleteOptions /** Shares bounded deletion and side effects across explicit workspace and organization owners. */ export async function chunkedBatchDeleteByScope({ tableDef, + budget, scopeIds, tableName, selectChunk, @@ -155,7 +172,7 @@ export async function chunkedBatchDeleteByScope({ let attempted = 0 for (const [chunkIdx, chunkIds] of chunks.entries()) { - if (attempted >= totalRowLimit) { + if (attempted >= totalRowLimit || budget?.remaining === 0) { stoppedEarly = true break } @@ -167,7 +184,11 @@ export async function chunkedBatchDeleteByScope({ let rows: TRow[] = [] try { const remainingLimit = totalRowLimit - attempted - const effectiveBatchSize = Math.min(batchSize, remainingLimit) + const effectiveBatchSize = Math.min( + batchSize, + remainingLimit, + budget?.remaining ?? batchSize + ) if (effectiveBatchSize <= 0) { hasMore = false break @@ -180,6 +201,7 @@ export async function chunkedBatchDeleteByScope({ break } + consumeRowBudget(budget, rows.length) attempted += rows.length if (onBatch) await onBatch(rows) @@ -194,6 +216,7 @@ export async function chunkedBatchDeleteByScope({ hasMore = rows.length === effectiveBatchSize && attempted < totalRowLimit batchesProcessed++ } catch (error) { + if (budget) throw error // Count rows we tried to delete; SELECT-stage errors leave rows=[]. result.failed += rows.length logger.error( @@ -213,6 +236,7 @@ export async function chunkedBatchDeleteByScope({ } export interface BatchDeleteOptions { + budget?: RowBudget tableDef: PgTable workspaceIdCol: PgColumn timestampCol: PgColumn diff --git a/apps/sim/lib/cleanup/bounded-cleanup.md b/apps/sim/lib/cleanup/bounded-cleanup.md new file mode 100644 index 00000000000..c03ea1735c9 --- /dev/null +++ b/apps/sim/lib/cleanup/bounded-cleanup.md @@ -0,0 +1,18 @@ +# Manual cleanup limits + +The existing cron Lambda can call these endpoints with query parameters: + +- `/api/logs/cleanup?workflowLogs=25&jobLogs=25` +- `/api/cron/cleanup-soft-deletes?files=10&legacyFiles=10` + +Use the existing cron authentication. Each request returns HTTP 202 with `runId` and queues one job. The worker uses the existing retention rules and cleanup functions. Limits are integers from 0 to 5000; omitted types are zero. At least one positive limit is required. Calls without parameters retain scheduled dispatch. + +Log types: `workflowLogs`, `jobLogs`, `largeValues`, `legacyLargeValues`, `orphanSnapshots`, `staleReferences`, `staleDependencies`, `largeValueTombstones`. + +Soft-delete types: `workflows`, `chats`, `legacyFiles`, `files`, `knowledgeBases`, `folders`, `userTables`, `memories`, `mcpServers`, `workflowMcpServers`, `orphanKnowledgeBaseBindings`. + +Each type has one budget across all workspace and organization chunks in that job. Limits count selected root rows, including rows restored or unsuccessfully deleted after selection. Existing child cascades and attached-file cleanup still follow the selected parents; the limit is not a cap on every physical row affected by a cascade. + +Log and soft-delete tasks share a queue with concurrency one. Jobs have one attempt so automatic retries cannot reset a spent row budget. Each new API call creates a new job; inspect the returned run before repeating a call whose response was lost. + +Deploy the worker before the API. Keep schedules disabled while draining the backlog. Start with small limits for one type, inspect the job logs and database load, then repeat and increase counts gradually. Storage, billing, and concurrent-restore behavior follow the existing cleanup implementation. diff --git a/apps/sim/lib/cleanup/limits.ts b/apps/sim/lib/cleanup/limits.ts new file mode 100644 index 00000000000..f6194569ffb --- /dev/null +++ b/apps/sim/lib/cleanup/limits.ts @@ -0,0 +1,41 @@ +import type { RowBudget } from '@/lib/cleanup/batch-delete' + +export const LOG_CLEANUP_TYPES = [ + 'workflowLogs', + 'jobLogs', + 'largeValues', + 'legacyLargeValues', + 'orphanSnapshots', + 'staleReferences', + 'staleDependencies', + 'largeValueTombstones', +] as const +export const SOFT_DELETE_CLEANUP_TYPES = [ + 'workflows', + 'chats', + 'legacyFiles', + 'files', + 'knowledgeBases', + 'folders', + 'userTables', + 'memories', + 'mcpServers', + 'workflowMcpServers', + 'orphanKnowledgeBaseBindings', +] as const +export type CleanupType = + | (typeof LOG_CLEANUP_TYPES)[number] + | (typeof SOFT_DELETE_CLEANUP_TYPES)[number] +export type CleanupLimits = Partial> +export type CleanupBudgets = Record +export type LimitedCleanupPayload = { limits: CleanupLimits } + +/** One mutable budget per type, shared across every owner scope in the queued job. */ +export function createCleanupBudgets(limits: CleanupLimits): CleanupBudgets { + return Object.fromEntries( + [...LOG_CLEANUP_TYPES, ...SOFT_DELETE_CLEANUP_TYPES].map((type) => [ + type, + { remaining: limits[type] ?? 0 }, + ]) + ) as CleanupBudgets +} diff --git a/apps/sim/lib/cleanup/queue.ts b/apps/sim/lib/cleanup/queue.ts new file mode 100644 index 00000000000..06474dcf1f5 --- /dev/null +++ b/apps/sim/lib/cleanup/queue.ts @@ -0,0 +1,2 @@ +import { queue } from '@trigger.dev/sdk' +export const retentionCleanupQueue = queue({ name: 'retention-cleanup', concurrencyLimit: 1 }) diff --git a/apps/sim/lib/execution/payloads/large-value-metadata.ts b/apps/sim/lib/execution/payloads/large-value-metadata.ts index dcbc03f4ff6..efe5ab76482 100644 --- a/apps/sim/lib/execution/payloads/large-value-metadata.ts +++ b/apps/sim/lib/execution/payloads/large-value-metadata.ts @@ -9,6 +9,8 @@ import { import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { and, eq, inArray, notInArray, sql } from 'drizzle-orm' +import { consumeRowBudget } from '@/lib/cleanup/batch-delete' +import type { CleanupBudgets } from '@/lib/cleanup/limits' import { collectLargeValueKeys } from '@/lib/execution/payloads/large-execution-value' const logger = createLogger('LargeValueMetadata') @@ -50,6 +52,7 @@ export interface LargeValueMetadataPruneResult { } interface PruneLargeValueMetadataOptions { + budgets?: CleanupBudgets workspaceIds: string[] tombstonesDeletedBefore: Date batchSize?: number @@ -473,6 +476,7 @@ async function pruneDeletedLargeValueTombstones( export async function pruneLargeValueMetadata({ workspaceIds, tombstonesDeletedBefore, + budgets, batchSize = LARGE_VALUE_METADATA_PRUNE_BATCH_SIZE, maxRowsPerTable = LARGE_VALUE_METADATA_PRUNE_MAX_ROWS_PER_TABLE, dbClient = db, @@ -488,32 +492,47 @@ export async function pruneLargeValueMetadata({ workspaceIds, LARGE_VALUE_METADATA_WORKSPACE_CHUNK_SIZE )) { - const referencesRemaining = maxRowsPerTable - result.referencesDeleted + const referencesRemaining = Math.min( + maxRowsPerTable - result.referencesDeleted, + budgets?.staleReferences.remaining ?? maxRowsPerTable + ) if (referencesRemaining > 0) { - result.referencesDeleted += await pruneStaleReferences( + const deleted = await pruneStaleReferences( workspaceChunk, Math.min(batchSize, referencesRemaining), dbClient ) + consumeRowBudget(budgets?.staleReferences, deleted) + result.referencesDeleted += deleted } - const dependenciesRemaining = maxRowsPerTable - result.dependenciesDeleted + const dependenciesRemaining = Math.min( + maxRowsPerTable - result.dependenciesDeleted, + budgets?.staleDependencies.remaining ?? maxRowsPerTable + ) if (dependenciesRemaining > 0) { - result.dependenciesDeleted += await pruneDeletedParentDependencies( + const deleted = await pruneDeletedParentDependencies( workspaceChunk, Math.min(batchSize, dependenciesRemaining), dbClient ) + consumeRowBudget(budgets?.staleDependencies, deleted) + result.dependenciesDeleted += deleted } - const tombstonesRemaining = maxRowsPerTable - result.tombstonesDeleted + const tombstonesRemaining = Math.min( + maxRowsPerTable - result.tombstonesDeleted, + budgets?.largeValueTombstones.remaining ?? maxRowsPerTable + ) if (tombstonesRemaining > 0) { - result.tombstonesDeleted += await pruneDeletedLargeValueTombstones( + const deleted = await pruneDeletedLargeValueTombstones( workspaceChunk, tombstonesDeletedBefore, Math.min(batchSize, tombstonesRemaining), dbClient ) + consumeRowBudget(budgets?.largeValueTombstones, deleted) + result.tombstonesDeleted += deleted } if ( diff --git a/apps/sim/lib/logs/execution/snapshot/service.ts b/apps/sim/lib/logs/execution/snapshot/service.ts index c6cc2d22ddd..33db22dba97 100644 --- a/apps/sim/lib/logs/execution/snapshot/service.ts +++ b/apps/sim/lib/logs/execution/snapshot/service.ts @@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { generateId } from '@sim/utils/id' import { and, eq, inArray, lt, notExists, sql } from 'drizzle-orm' +import { consumeRowBudget, type RowBudget } from '@/lib/cleanup/batch-delete' import type { SnapshotService as ISnapshotService, SnapshotCreationResult, @@ -103,7 +104,7 @@ export class SnapshotService implements ISnapshotService { } /** Only invoked from the cleanup-logs background job, so it runs on the cleanup pool. */ - async cleanupOrphanedSnapshots(olderThanDays: number): Promise { + async cleanupOrphanedSnapshots(olderThanDays: number, budget?: RowBudget): Promise { const cleanupDb = dbFor('cleanup') const cutoffDate = new Date() cutoffDate.setDate(cutoffDate.getDate() - olderThanDays) @@ -115,6 +116,7 @@ export class SnapshotService implements ISnapshotService { let stoppedEarly = false for (let batch = 0; batch < MAX_BATCHES; batch++) { + if (budget?.remaining === 0) break const candidates = await cleanupDb .select({ id: workflowExecutionSnapshots.id }) .from(workflowExecutionSnapshots) @@ -129,10 +131,11 @@ export class SnapshotService implements ISnapshotService { ) ) ) - .limit(BATCH_SIZE) + .limit(Math.min(BATCH_SIZE, budget?.remaining ?? BATCH_SIZE)) if (candidates.length === 0) break + consumeRowBudget(budget, candidates.length) const ids = candidates.map((c) => c.id) const deleted = await cleanupDb .delete(workflowExecutionSnapshots) From d61006a70a3798bb6be74ee9c1f051240f7143f0 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 15 Sep 2026 12:18:27 -0700 Subject: [PATCH 04/15] fix(desktop): share browser modal and status page presentation (#7856) * fix(desktop): share browser modal and status page presentation * fix(desktop): apply initial shell theme before IPC resolves * chore(desktop): remove redundant dismissal test comment --- apps/desktop/e2e/smoke.spec.ts | 60 ++++++++++-- apps/desktop/electron-builder.yml | 2 +- apps/desktop/src/main/dialogs.test.ts | 2 +- apps/desktop/src/main/dialogs.ts | 11 ++- apps/desktop/src/main/index.ts | 2 + apps/desktop/src/main/ipc.test.ts | 4 +- apps/desktop/src/main/ipc.ts | 12 ++- apps/desktop/src/main/local-pages.ts | 4 +- apps/desktop/src/main/server-window.ts | 19 +--- apps/desktop/src/main/shell-theme.test.ts | 52 ++++++++++ apps/desktop/src/main/shell-theme.ts | 59 ++++++++++++ apps/desktop/src/main/window.test.ts | 2 +- apps/desktop/src/main/window.ts | 52 +++------- apps/desktop/src/preload/index.test.ts | 1 + apps/desktop/src/preload/index.ts | 4 + apps/desktop/src/preload/shell-theme.ts | 39 ++++++++ apps/desktop/src/preload/shell.ts | 2 + apps/desktop/src/renderer/dialog/index.tsx | 45 ++++----- apps/desktop/src/renderer/offline/index.tsx | 94 +++++++++---------- apps/desktop/src/renderer/server/index.tsx | 13 +-- .../src/renderer/server/server-modal.tsx | 24 +++-- apps/desktop/src/renderer/shell.css | 2 - apps/desktop/src/renderer/shell.test.ts | 68 ++++++++++++++ apps/desktop/src/renderer/shell.ts | 49 ++++++++-- apps/desktop/src/shared/shell.ts | 10 +- .../app/(auth)/components/auth-shell.test.tsx | 1 - apps/sim/app/(auth)/components/auth-shell.tsx | 3 +- .../chat/components/header/header.tsx | 2 +- .../comparisons/[provider]/page.test.tsx | 3 +- .../brand-icon-tile/brand-icon-tile.tsx | 3 +- .../agent-momentum/agent-momentum.test.tsx | 3 +- .../featured-customer.test.tsx | 3 +- .../footer-wordmark-loop.test.tsx | 2 +- .../footer-wordmark-loop.tsx | 3 +- .../(landing)/components/footer/footer.tsx | 3 +- .../hero-platform-stage.test.tsx | 5 +- .../(landing)/components/landing-layout.ts | 6 +- .../landing-shell/landing-shell.test.tsx | 3 +- .../components/logo-shell/logo-shell.tsx | 41 ++++---- .../components/navbar/components/index.ts | 1 - .../nav-menu-item/nav-menu-item.tsx | 3 +- .../nav-menu-chip/nav-menu-chip.test.tsx | 8 +- .../navbar/components/sim-wordmark/index.ts | 1 - .../(landing)/components/navbar/navbar.tsx | 3 +- .../platform-suite/platform-suite.test.tsx | 3 +- .../product-demo-caption.test.tsx | 3 +- .../workspace-controls.test.tsx | 3 +- .../components/security/security.test.tsx | 3 +- .../product-window/product-window.test.tsx | 3 +- .../sim/app/(landing)/customers/page.test.tsx | 5 +- .../desktop-title-bar-controller.test.tsx | 6 +- apps/sim/app/_shell/desktop-title-bar.test.ts | 3 +- apps/sim/app/_shell/desktop-title-bar.tsx | 69 +------------- apps/sim/app/f/[token]/public-file-view.tsx | 3 +- apps/sim/app/global-error.tsx | 3 +- apps/sim/app/not-found.tsx | 2 +- .../workspace-chrome/workspace-chrome.tsx | 2 +- .../settings/settings-sidebar.test.tsx | 1 - .../components/settings/settings-sidebar.tsx | 2 +- apps/sim/components/status-page/index.ts | 2 +- .../components/status-page/status-page.tsx | 39 +++----- apps/sim/lib/branding/index.ts | 6 +- apps/sim/lib/branding/wordmark.test.ts | 8 +- apps/sim/lib/branding/wordmark.ts | 23 +---- packages/desktop-bridge/src/index.ts | 7 ++ packages/desktop-bridge/src/title-bar.ts | 51 ++++++++++ .../src/components/chip-modal/chip-modal.tsx | 34 +++++-- packages/emcn/src/components/index.ts | 13 +++ .../emcn/src/components/sim-wordmark/paths.ts | 10 ++ .../components/sim-wordmark/sim-wordmark.tsx | 20 +--- .../components/status-page/status-page.tsx | 78 +++++++++++++++ packages/emcn/src/icons/index.ts | 1 - packages/emcn/src/icons/wordmark.tsx | 58 ------------ 73 files changed, 728 insertions(+), 462 deletions(-) create mode 100644 apps/desktop/src/main/shell-theme.test.ts create mode 100644 apps/desktop/src/main/shell-theme.ts create mode 100644 apps/desktop/src/preload/shell-theme.ts create mode 100644 apps/desktop/src/renderer/shell.test.ts delete mode 100644 apps/sim/app/(landing)/components/navbar/components/sim-wordmark/index.ts create mode 100644 packages/desktop-bridge/src/title-bar.ts create mode 100644 packages/emcn/src/components/sim-wordmark/paths.ts rename {apps/sim/app/(landing)/components/navbar => packages/emcn/src}/components/sim-wordmark/sim-wordmark.tsx (56%) create mode 100644 packages/emcn/src/components/status-page/status-page.tsx delete mode 100644 packages/emcn/src/icons/wordmark.tsx diff --git a/apps/desktop/e2e/smoke.spec.ts b/apps/desktop/e2e/smoke.spec.ts index 578bf6e78f0..f6799b7a835 100644 --- a/apps/desktop/e2e/smoke.spec.ts +++ b/apps/desktop/e2e/smoke.spec.ts @@ -179,6 +179,55 @@ test.describe('desktop shell smoke', () => { await expect(window.locator('#detail')).toHaveAttribute('role', 'status') }) + test('offline title-bar geometry follows native fullscreen state across reloads', async () => { + test.skip(process.platform !== 'darwin', 'The traffic-light lane is macOS-specific') + app = await launchApp('http://127.0.0.1:1') + const window = await app.firstWindow() + await expect(window.locator('#server')).toBeVisible() + await expect(window.locator('html')).toHaveAttribute('data-sim-desktop-title-bar', 'inset') + await app.evaluate(({ BrowserWindow }) => { + BrowserWindow.getAllWindows()[0].setFullScreen(true) + }) + await expect(window.locator('html')).toHaveAttribute('data-sim-desktop-title-bar', 'fullscreen') + await window.reload() + await expect(window.locator('#server')).toBeVisible() + await expect(window.locator('html')).toHaveAttribute('data-sim-desktop-title-bar', 'fullscreen') + await expect(window.locator('.desktop-title-bar-page')).toHaveCSS('padding-top', '0px') + await app.evaluate(({ BrowserWindow }) => { + BrowserWindow.getAllWindows()[0].setFullScreen(false) + }) + await expect(window.locator('html')).toHaveAttribute('data-sim-desktop-title-bar', 'inset') + }) + + test('bundled dialogs follow the app theme independently of the system theme', async () => { + app = await launchApp(origin) + const window = await app.firstWindow() + await expect(window.locator('#app')).toBeVisible() + await app.evaluate(({ nativeTheme }) => { + nativeTheme.themeSource = 'light' + }) + await window.evaluate(() => { + document.documentElement.className = 'dark' + }) + const dialogPromise = app.waitForEvent('window') + await app.evaluate(({ BrowserWindow }) => { + BrowserWindow.getAllWindows()[0].webContents.emit('unresponsive') + }) + const prompt = await dialogPromise + await expect(prompt.getByRole('dialog')).toBeVisible() + await expect(prompt.locator('html')).toHaveClass('dark') + await expect(prompt.locator('html')).toHaveCSS('color-scheme', 'dark') + await expect(prompt.locator('#dialog-message')).toHaveCSS('-webkit-font-smoothing', 'auto') + await expect(prompt.locator('#dialog-message')).toHaveCSS('font-weight', '400') + await expect(prompt.locator('#dialog-message')).toHaveCSS('font-size', '14px') + await window.evaluate(() => { + document.documentElement.className = 'light' + }) + await expect(prompt.locator('html')).toHaveClass('light') + await expect(prompt.locator('html')).toHaveCSS('color-scheme', 'light') + await prompt.getByRole('button', { name: 'Wait', exact: true }).click() + }) + test('recovery messages use an isolated EMCN dialog with a safe keyboard default', async () => { app = await launchApp('http://127.0.0.1:1') const window = await app.firstWindow() @@ -188,7 +237,9 @@ test.describe('desktop shell smoke', () => { BrowserWindow.getAllWindows()[0].webContents.emit('unresponsive') }) const prompt = await dialogPromise - await expect(prompt.getByRole('dialog', { name: 'Sim', exact: true })).toBeVisible() + await expect( + prompt.getByRole('dialog', { name: 'Sim isn’t responding', exact: true }) + ).toBeVisible() await expect(prompt.getByText('Sim isn’t responding')).toBeVisible() await expect(prompt.getByRole('button', { name: 'Wait', exact: true })).toBeFocused() await expect @@ -213,10 +264,7 @@ test.describe('desktop shell smoke', () => { win.webContents.ipc.removeHandler('shell:configuration') win.webContents.ipc.handle('shell:configuration', () => ({ title: 'Long recovery message', - message: 'Recovery details', - detail: Array.from({ length: 80 }, (_, index) => `Diagnostic detail ${index + 1}`).join( - '\n' - ), + text: Array.from({ length: 80 }, (_, index) => `Diagnostic detail ${index + 1}`).join('\n'), type: 'warning', buttons: ['Wait', 'Reload'], defaultId: 0, @@ -291,8 +339,6 @@ test.describe('desktop shell smoke', () => { }) const closed = picker.waitForEvent('close') - // The main process destroys the window on the key-down, so the key-up half - // of `press` has no target to reach; the close event is the assertion. await picker.keyboard.press('Escape').catch(() => {}) await closed expect(app.windows()).toHaveLength(1) diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index 34d390dad9c..25856fb9dc8 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -10,7 +10,7 @@ files: - dist/** - static/** - package.json - - from: ../sim/public/brand/fonts + - from: ../sim/app/_styles/fonts/season to: static filter: - SeasonSansUprightsVF.woff2 diff --git a/apps/desktop/src/main/dialogs.test.ts b/apps/desktop/src/main/dialogs.test.ts index e2915ac5925..41d3ac8b7de 100644 --- a/apps/desktop/src/main/dialogs.test.ts +++ b/apps/desktop/src/main/dialogs.test.ts @@ -121,7 +121,7 @@ describe('showShellDialog', () => { resize?.(sender(win), Number.NaN) expect(win.setContentSize).not.toHaveBeenCalled() resize?.(sender(win), 100000) - expect(win.setContentSize).toHaveBeenCalledWith(500, 820) + expect(win.setContentSize).toHaveBeenCalledWith(440, 820) respond(win, 0) await result }) diff --git a/apps/desktop/src/main/dialogs.ts b/apps/desktop/src/main/dialogs.ts index 23f96c2beac..1f2a5e60236 100644 --- a/apps/desktop/src/main/dialogs.ts +++ b/apps/desktop/src/main/dialogs.ts @@ -4,12 +4,13 @@ import { getErrorMessage } from '@sim/utils/errors' import type { MessageBoxOptions, MessageBoxReturnValue } from 'electron' import { app, BrowserWindow, dialog, nativeTheme, session } from 'electron' import { attachLocalPageProtocol, localPageUrl } from '@/main/local-pages' +import { attachShellTheme, backgroundColorFor, getShellTheme } from '@/main/shell-theme' import { attachShellWindowSizing, isShellWindowSender } from '@/main/shell-window' import { createSecureWebPreferences } from '@/main/window-preferences' import type { ShellDialogConfiguration } from '@/shared/shell' const logger = createLogger('DesktopDialogs') -const DIALOG_WIDTH = 500 +const DIALOG_WIDTH = 440 const DIALOG_PARTITION = 'shell-dialogs' interface ShellDialogOptions extends MessageBoxOptions { @@ -39,9 +40,8 @@ export function showShellDialog( buttons.findIndex((label) => /^(cancel|no|close|ok)$/i.test(label)) ) const configuration: ShellDialogConfiguration = { - title: options.title ?? 'Sim', - message: options.message, - detail: options.detail ?? '', + title: options.title ?? options.message, + text: [options.title ? options.message : '', options.detail].filter(Boolean).join('\n\n'), buttons, defaultId: options.defaultId ?? 0, cancelId, @@ -66,7 +66,7 @@ export function showShellDialog( fullscreenable: false, show: false, title: configuration.title, - backgroundColor: nativeTheme.shouldUseDarkColors ? '#1b1b1b' : '#ffffff', + backgroundColor: backgroundColorFor(getShellTheme(), nativeTheme.shouldUseDarkColors), ...(parent && !parent.isDestroyed() ? { parent, modal: true } : {}), webPreferences: createSecureWebPreferences( DIALOG_PARTITION, @@ -75,6 +75,7 @@ export function showShellDialog( ), }) const pageUrl = localPageUrl('dialog.html') + attachShellTheme(win) let settled = false const finish = (response: number) => { if (settled) return diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index b273e50f1eb..30972db3c9e 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -78,6 +78,7 @@ import { readSessionUserId, resolveStartRoute, } from '@/main/session-lifecycle' +import { setShellTheme } from '@/main/shell-theme' import { attachTelemetryPolicy } from '@/main/telemetry-policy' import { TerminalRegistry } from '@/main/terminal/registry' import { installTray, type TrayHandle } from '@/main/tray' @@ -110,6 +111,7 @@ function main(): void { const userDataPath = app.getPath('userData') const config = createConfigStore(join(userDataPath, 'settings.json')) + setShellTheme(config.get('themeBackground')) initializeAccountDataRecovery(join(userDataPath, 'account-data-teardown-required.json')) const recoveryOrigin = getAccountDataTeardownOrigin() if (isAccountDataTeardownRequired() && recoveryOrigin && !config.isPersistenceAvailable()) { diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index 83deccabb27..9ebb7d6f0d9 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -665,13 +665,15 @@ describe('registerIpcHandlers', () => { expect(deps.settings.chooseBrowserDownloadDirectory).toHaveBeenCalledTimes(1) }) - it('reports native fullscreen state only to the app origin', async () => { + it('reports native fullscreen state only to the app origin and bundled pages', async () => { const { invoke } = collectHandlers() const getWindowState = invoke.get('desktop:window-state:get') expect(await getWindowState?.(evilEvent)).toEqual({ isFullScreen: false }) expect(await getWindowState?.(appEvent)).toEqual({ isFullScreen: true }) expect(deps.getWindowState).toHaveBeenCalledWith(appSender) + expect(await getWindowState?.(localPageEvent)).toEqual({ isFullScreen: true }) + expect(deps.getWindowState).toHaveBeenCalledWith(localPageSender) }) it('restricts shell-control channels to bundled local pages', () => { diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 4586567439a..78038098bbc 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -376,6 +376,7 @@ export interface IpcDeps { * - `app-origin`: only the remote app origin (main window pages). * - `local-page`: only the bundled pages served from the shell's own scheme * (offline, server) — shell control. + * - `app-or-local-page`: read-only window state used by both hosted and bundled pages. * - `browser-page`: only the built-in browser's own tabs, identified by * WebContents rather than by URL. These carry reports from the browser * preload about untrusted pages, so they are the one inbound surface whose @@ -383,7 +384,7 @@ export interface IpcDeps { * as an instruction. * - `any`: sender-independent channels that validate their input instead. */ -type ChannelGate = 'app-origin' | 'local-page' | 'browser-page' | 'any' +type ChannelGate = 'app-origin' | 'local-page' | 'app-or-local-page' | 'browser-page' | 'any' /** * A desktop surface the user can switch off. Channels that drive one are @@ -816,7 +817,9 @@ export function registerIpcHandlers(deps: IpcDeps): void { }, 'desktop:window-state:get': { kind: 'invoke', - gate: 'app-origin', + gate: 'app-or-local-page', + deviationReason: + 'Bundled offline pages share the app title-bar geometry and need their own native fullscreen state.', passSender: true, denied: { isFullScreen: false }, handler: (sender) => deps.getWindowState(sender as WebContents), @@ -1857,6 +1860,11 @@ export function registerIpcHandlers(deps: IpcDeps): void { const senderAllowed = (event: IpcMainEvent | IpcMainInvokeEvent, gate: ChannelGate): boolean => { if (gate === 'any') return true if (gate === 'app-origin') return isAppOriginSender(event, deps.appOrigin()) + if (gate === 'app-or-local-page') { + return ( + isAppOriginSender(event, deps.appOrigin()) || isLocalPageSender(event, deps.isLocalPageUrl) + ) + } if (gate === 'browser-page') return isAgentWebContents(event.sender) return isLocalPageSender(event, deps.isLocalPageUrl) } diff --git a/apps/desktop/src/main/local-pages.ts b/apps/desktop/src/main/local-pages.ts index 4e34f2e9870..2675fe3fe49 100644 --- a/apps/desktop/src/main/local-pages.ts +++ b/apps/desktop/src/main/local-pages.ts @@ -167,13 +167,13 @@ async function readFirst(rootDirs: readonly string[], name: string): Promise { @@ -149,12 +144,7 @@ export function createServerWindow(deps: ServerWindowDeps): ServerWindowHandle { title: 'Sim Server', frame: false, show: false, - // System preference only, unlike the main window: that one pre-paints for - // the web app it is about to load, whose theme the user picked in Sim. - // This window loads a bundled page that follows `prefers-color-scheme`, - // so honouring the stored web-app theme here would pre-paint dark behind - // a page about to render light whenever the two disagree. - backgroundColor: backgroundColorFor(undefined, nativeTheme.shouldUseDarkColors), + backgroundColor: backgroundColorFor(getShellTheme(), nativeTheme.shouldUseDarkColors), // Modal only when there is a live parent to attach to. A shell whose // window is gone (or never opened, because the origin failed to load) // still has to be able to reach this. @@ -169,6 +159,7 @@ export function createServerWindow(deps: ServerWindowDeps): ServerWindowHandle { // ways out must therefore work without the page: Escape is handled here, // and a page that fails to load closes the window instead of leaving a // blank sheet nothing can dismiss. + attachShellTheme(win) const opened = win let closed = false const closeOpened = () => { diff --git a/apps/desktop/src/main/shell-theme.test.ts b/apps/desktop/src/main/shell-theme.test.ts new file mode 100644 index 00000000000..efa71b587bc --- /dev/null +++ b/apps/desktop/src/main/shell-theme.test.ts @@ -0,0 +1,52 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => import('@/test/electron-mock')) + +import type { BrowserWindow as ElectronWindow } from 'electron' +import { attachShellTheme, getShellTheme, setShellTheme } from '@/main/shell-theme' +import { BrowserWindow } from '@/test/electron-mock' + +beforeEach(() => { + setShellTheme(undefined) + vi.clearAllMocks() +}) + +describe('shell theme', () => { + it('retains the resolved app theme and updates only bundled pages', () => { + const local = new BrowserWindow({}) + const app = new BrowserWindow({}) + local.webContents.getURL.mockReturnValue('sim-shell://pages/dialog.html') + app.webContents.getURL.mockReturnValue('https://sim.example') + attachShellTheme(local as unknown as ElectronWindow) + attachShellTheme(app as unknown as ElectronWindow) + setShellTheme('dark') + expect(getShellTheme()).toBe('dark') + expect(local.webContents.send).toHaveBeenCalledWith('shell:theme-changed', 'dark') + expect(app.webContents.send).not.toHaveBeenCalled() + setShellTheme('dark') + expect(local.webContents.send).toHaveBeenCalledOnce() + local.on.mock.calls.find(([event]) => event === 'closed')?.[1]() + app.on.mock.calls.find(([event]) => event === 'closed')?.[1]() + setShellTheme('light') + expect(local.webContents.send).toHaveBeenCalledOnce() + }) + + it('rejects theme reads from foreign documents and subframes', () => { + const win = new BrowserWindow({}) + attachShellTheme(win as unknown as ElectronWindow) + const read = win.webContents.ipc.handle.mock.calls.find( + ([channel]) => channel === 'shell:get-theme' + )?.[1] + if (!read) throw new Error('Missing theme handler') + const event = { sender: win.webContents, senderFrame: win.webContents.mainFrame } + win.webContents.mainFrame.url = 'sim-shell://pages/offline.html?kind=dns' + setShellTheme('light') + expect(read(event)).toBe('light') + expect(() => read({ ...event, senderFrame: { url: event.senderFrame.url } })).toThrow( + 'Untrusted' + ) + win.webContents.mainFrame.url = 'https://untrusted.example' + expect(() => read(event)).toThrow('Untrusted') + win.on.mock.calls.find(([event]) => event === 'closed')?.[1]() + }) +}) diff --git a/apps/desktop/src/main/shell-theme.ts b/apps/desktop/src/main/shell-theme.ts new file mode 100644 index 00000000000..a1eb4afc723 --- /dev/null +++ b/apps/desktop/src/main/shell-theme.ts @@ -0,0 +1,59 @@ +import type { BrowserWindow } from 'electron' +import { isLocalPageUrl, localPageUrl } from '@/main/local-pages' +import type { ShellTheme } from '@/shared/shell' + +let theme: ShellTheme | undefined +const windows = new Set() + +export function getShellTheme(): ShellTheme | undefined { + return theme +} + +function isShellPage(url: string): boolean { + return isLocalPageUrl(url) || url === localPageUrl('dialog.html') +} + +/** Retains Sim's last resolved theme so recovery works even after its renderer stops. */ +export function setShellTheme(next: ShellTheme | undefined): void { + if (theme === next) return + theme = next + for (const win of windows) { + if (!win.isDestroyed() && isShellPage(win.webContents.getURL())) { + win.webContents.send('shell:theme-changed', theme) + } + } +} + +/** Only bundled top-level pages can read the shell's appearance. */ +export function attachShellTheme(win: BrowserWindow): void { + windows.add(win) + win.on('closed', () => windows.delete(win)) + win.webContents.ipc.handle('shell:get-theme', (event) => { + if ( + win.isDestroyed() || + event.sender !== win.webContents || + event.senderFrame !== win.webContents.mainFrame || + !isShellPage(event.senderFrame.url) + ) { + throw new Error('Untrusted shell theme sender') + } + return theme + }) +} + +/** + * Picks the pre-paint window background from the persisted web-app theme so + * dark-mode users never see a white flash before the remote page paints. + */ +export function backgroundColorFor( + theme: 'dark' | 'light' | undefined, + systemPrefersDark: boolean +): string { + if (theme === 'dark') { + return '#0c0c0c' + } + if (theme === 'light') { + return '#ffffff' + } + return systemPrefersDark ? '#0c0c0c' : '#ffffff' +} diff --git a/apps/desktop/src/main/window.test.ts b/apps/desktop/src/main/window.test.ts index 2d2ca5488be..69a02eea19d 100644 --- a/apps/desktop/src/main/window.test.ts +++ b/apps/desktop/src/main/window.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { backgroundColorFor } from '@/main/shell-theme' import { createSecureWebPreferences } from '@/main/window-preferences' vi.mock('electron', () => import('@/test/electron-mock')) @@ -7,7 +8,6 @@ import { BrowserWindow, dialog, screen, systemPreferences } from 'electron' import type { ConfigStore } from '@/main/config' import type { EventRecorder } from '@/main/observability' import { - backgroundColorFor, createMainWindow, ensureMicrophoneAccess, fitBoundsToWorkArea, diff --git a/apps/desktop/src/main/window.ts b/apps/desktop/src/main/window.ts index 52fb546115e..19481b4ff89 100644 --- a/apps/desktop/src/main/window.ts +++ b/apps/desktop/src/main/window.ts @@ -6,12 +6,11 @@ import { type ConfigStore, isSafeInternalPath, type WindowBounds } from '@/main/ import { showShellDialog } from '@/main/dialogs' import { isAppOrigin, isAuthSurfacePath } from '@/main/navigation' import type { EventRecorder } from '@/main/observability' +import { attachShellTheme, backgroundColorFor, setShellTheme } from '@/main/shell-theme' import { createSecureWebPreferences } from '@/main/window-preferences' const logger = createLogger('DesktopWindow') -const DARK_BACKGROUND = '#0c0c0c' -const LIGHT_BACKGROUND = '#ffffff' const DEFAULT_WIDTH = 1360 const DEFAULT_HEIGHT = 860 const MIN_WIDTH = 800 @@ -20,14 +19,6 @@ const WINDOW_TITLE = 'Sim' const BOUNDS_SAVE_DELAY_MS = 400 const ROUTE_SAVE_DELAY_MS = 500 -const THEME_PROBE_SCRIPT = `(() => { - try { - return document.documentElement.classList.contains('dark') - } catch { - return null - } -})()` - /** * The permission matrix: sanitized clipboard writes and microphone access for * the trusted app origin, default-deny for everything else including unknown @@ -127,23 +118,6 @@ export function setupPermissionHandlers(session: Session, getAppOrigin: () => st }) } -/** - * Picks the pre-paint window background from the persisted web-app theme so - * dark-mode users never see a white flash before the remote page paints. - */ -export function backgroundColorFor( - theme: 'dark' | 'light' | undefined, - systemPrefersDark: boolean -): string { - if (theme === 'dark') { - return DARK_BACKGROUND - } - if (theme === 'light') { - return LIGHT_BACKGROUND - } - return systemPrefersDark ? DARK_BACKGROUND : LIGHT_BACKGROUND -} - /** * Drops persisted bounds that are malformed or implausibly small so a bad * settings file can never produce an unusable window. @@ -276,6 +250,19 @@ export function createMainWindow(deps: CreateMainWindowDeps): BrowserWindow { win.show() }) + attachShellTheme(win) + win.webContents.ipc.on('shell:app-theme', (event, theme: unknown) => { + if ( + event.sender !== win.webContents || + event.senderFrame !== win.webContents.mainFrame || + !isAppOrigin(event.senderFrame.url, deps.appOrigin()) || + (theme !== 'dark' && theme !== 'light') + ) + return + deps.config.set('themeBackground', theme) + setShellTheme(theme) + }) + let boundsTimer: NodeJS.Timeout | undefined const persistBounds = () => { clearTimeout(boundsTimer) @@ -394,17 +381,6 @@ export function createMainWindow(deps: CreateMainWindowDeps): BrowserWindow { win.webContents.setZoomLevel(zoomLevel) } } - const url = win.webContents.getURL() - if (isAppOrigin(url, deps.appOrigin())) { - void win.webContents - .executeJavaScript(THEME_PROBE_SCRIPT, true) - .then((isDark) => { - if (typeof isDark === 'boolean') { - deps.config.set('themeBackground', isDark ? 'dark' : 'light') - } - }) - .catch(() => {}) - } }) let routeTimer: NodeJS.Timeout | undefined diff --git a/apps/desktop/src/preload/index.test.ts b/apps/desktop/src/preload/index.test.ts index 9e7f1a313d1..52293493d63 100644 --- a/apps/desktop/src/preload/index.test.ts +++ b/apps/desktop/src/preload/index.test.ts @@ -1,3 +1,4 @@ +/** @vitest-environment jsdom */ import type { SimDesktopApi } from '@sim/desktop-bridge' import { describe, expect, it, vi } from 'vitest' diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 34c37732078..6e6ed65a51c 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -53,6 +53,10 @@ import { type TerminalToolResponse, } from '@sim/terminal-protocol' import { contextBridge, ipcRenderer } from 'electron' +import { exposeShellTheme, observeAppTheme } from '@/preload/shell-theme' + +exposeShellTheme() +observeAppTheme() const VERSION_ARG_PREFIX = '--sim-desktop-version=' diff --git a/apps/desktop/src/preload/shell-theme.ts b/apps/desktop/src/preload/shell-theme.ts new file mode 100644 index 00000000000..d12f010555b --- /dev/null +++ b/apps/desktop/src/preload/shell-theme.ts @@ -0,0 +1,39 @@ +import { contextBridge, ipcRenderer } from 'electron' +import type { ShellTheme, ShellThemeApi } from '@/shared/shell' + +/** A read-only appearance bridge, also available on the main window's offline page. */ +export function exposeShellTheme(): void { + const api: ShellThemeApi = { + get: () => ipcRenderer.invoke('shell:get-theme'), + onChange: (callback) => { + const listener = (_event: unknown, theme: ShellTheme | undefined) => callback(theme) + ipcRenderer.on('shell:theme-changed', listener) + return () => ipcRenderer.removeListener('shell:theme-changed', listener) + }, + } + contextBridge.exposeInMainWorld('simShellTheme', api) +} + +/** Reports the actual app theme, including changes after hydration or OS theme changes. */ +export function observeAppTheme(): void { + window.addEventListener('DOMContentLoaded', () => { + if (!['https:', 'http:'].includes(location.protocol)) return + let previous: ShellTheme | undefined + const report = () => { + const root = document.documentElement + const theme = root.classList.contains('dark') + ? 'dark' + : root.classList.contains('light') + ? 'light' + : undefined + if (!theme || theme === previous) return + previous = theme + ipcRenderer.send('shell:app-theme', theme) + } + new MutationObserver(report).observe(document.documentElement, { + attributes: true, + attributeFilter: ['class'], + }) + report() + }) +} diff --git a/apps/desktop/src/preload/shell.ts b/apps/desktop/src/preload/shell.ts index 900c7993aef..4f07249d7b8 100644 --- a/apps/desktop/src/preload/shell.ts +++ b/apps/desktop/src/preload/shell.ts @@ -1,4 +1,5 @@ import { contextBridge, ipcRenderer } from 'electron' +import { exposeShellTheme } from '@/preload/shell-theme' import type { ShellWindowApi } from '@/shared/shell' const api: ShellWindowApi = { @@ -12,3 +13,4 @@ const api: ShellWindowApi = { } contextBridge.exposeInMainWorld('simShell', api) +exposeShellTheme() diff --git a/apps/desktop/src/renderer/dialog/index.tsx b/apps/desktop/src/renderer/dialog/index.tsx index ee8e252e75c..1032d4a3914 100644 --- a/apps/desktop/src/renderer/dialog/index.tsx +++ b/apps/desktop/src/renderer/dialog/index.tsx @@ -1,6 +1,12 @@ -import { ChipModalBody, ChipModalFooter, ChipModalHeader, ChipModalSurface } from '@sim/emcn' +import { + ChipModalBody, + ChipModalDescription, + ChipModalFooter, + ChipModalHeader, + ChipModalSurface, +} from '@sim/emcn' import { createRoot } from 'react-dom/client' -import { initializeShellPage, observeShellSize, shellWindow } from '@/renderer/shell' +import { initializeShellPage, mountShellModal, shellWindow } from '@/renderer/shell' import type { ShellDialogConfiguration } from '@/shared/shell' import '@/renderer/shell.css' @@ -9,21 +15,18 @@ interface ShellDialogProps { } function ShellDialog({ configuration }: ShellDialogProps) { - const { message, detail, buttons, defaultId, cancelId } = configuration + const { text, buttons, defaultId, cancelId } = configuration const primaryId = buttons.length === 1 ? 0 : buttons.findIndex((_, index) => index !== cancelId) const respond = (response: number) => shellWindow?.respond(response) const close = () => respond(cancelId) return ( { - element?.querySelector('[data-chip-modal-default-action]')?.focus() - return observeShellSize(element) - }} + ref={(element) => mountShellModal(element, close)} role='dialog' aria-modal='true' aria-labelledby='dialog-title' - aria-describedby={detail ? 'dialog-message dialog-detail' : 'dialog-message'} + aria-describedby={text ? 'dialog-message' : undefined} className='max-h-screen' > {configuration.title} -

- {message} -

- {detail ? ( -

- {detail} -

- ) : null} + {text ? {text} : null} { - document.title = configuration.title - document.addEventListener('keydown', (event) => { - if (event.key === 'Escape') shellWindow?.respond(configuration.cancelId) - }) - createRoot(container).render() -}) +void Promise.all([initializeShellPage(), shellWindow.getDialogConfiguration()]).then( + ([, configuration]) => { + document.title = configuration.title + createRoot(container).render() + } +) diff --git a/apps/desktop/src/renderer/offline/index.tsx b/apps/desktop/src/renderer/offline/index.tsx index c00417bb57c..2797d496129 100644 --- a/apps/desktop/src/renderer/offline/index.tsx +++ b/apps/desktop/src/renderer/offline/index.tsx @@ -1,7 +1,6 @@ import { useState } from 'react' -import type { SimDesktopApi } from '@sim/desktop-bridge' -import { Chip } from '@sim/emcn' -import { ArrowUpRight, RefreshCw, Server, Wordmark } from '@sim/emcn/icons' +import { observeDesktopTitleBar, type SimDesktopApi } from '@sim/desktop-bridge' +import { Chip, LogoPage, SimWordmark, StatusPageContent } from '@sim/emcn' import { createRoot } from 'react-dom/client' import { initializeShellPage } from '@/renderer/shell' import '@/renderer/shell.css' @@ -58,59 +57,58 @@ function OfflinePage({ isSimCloud }: OfflinePageProps) { } return ( -
-
- -
-
-
-

- {copy.title} -

-

{copy.message}

-
- bridge?.offlineRetry()} - > - Retry - - {isSimCloud ? ( - - Check status - - ) : null} - bridge?.server?.open()}> - Change server - -
+ + } + logo={ + + + + } + > + {actionError || detail}

-
-
-
+ } + > + bridge?.offlineRetry()}> + Retry + + {isSimCloud ? ( + + Check status + + ) : null} + bridge?.server?.open()}> + Change server + + + ) } -initializeShellPage() const container = document.getElementById('root') if (!container) throw new Error('Offline page root is missing') -const root = createRoot(container) -root.render() -void bridge?.server - ?.getConfiguration() - .then(({ isSimCloud }) => { - root.render() - }) - .catch(() => {}) +void initializeShellPage().then(() => { + observeDesktopTitleBar(document.documentElement, navigator.userAgent, bridge) + const root = createRoot(container) + root.render() + void bridge?.server + ?.getConfiguration() + .then(({ isSimCloud }) => { + root.render() + }) + .catch(() => {}) +}) diff --git a/apps/desktop/src/renderer/server/index.tsx b/apps/desktop/src/renderer/server/index.tsx index 3c7b1fdbb94..d4431319936 100644 --- a/apps/desktop/src/renderer/server/index.tsx +++ b/apps/desktop/src/renderer/server/index.tsx @@ -3,23 +3,18 @@ import { ServerModal } from '@/renderer/server/server-modal' import { initializeShellPage, shellWindow } from '@/renderer/shell' import '@/renderer/shell.css' -initializeShellPage() -document.addEventListener('keydown', (event) => { - if (event.key === 'Escape') window.close() -}) - -const server = shellWindow?.server const container = document.getElementById('root') -if (!container) throw new Error('Server modal root is missing') +if (!container || !shellWindow) throw new Error('Server modal host is unavailable') +const server = shellWindow.server const root = createRoot(container) async function renderServerModal() { try { - const configuration = await server?.getConfiguration() + const configuration = await server.getConfiguration() root.render() } catch { root.render() } } -void renderServerModal() +void initializeShellPage().then(renderServerModal) diff --git a/apps/desktop/src/renderer/server/server-modal.tsx b/apps/desktop/src/renderer/server/server-modal.tsx index 6eac19d7d76..66ce047470b 100644 --- a/apps/desktop/src/renderer/server/server-modal.tsx +++ b/apps/desktop/src/renderer/server/server-modal.tsx @@ -2,16 +2,17 @@ import { useRef, useState } from 'react' import type { DesktopServerConfiguration } from '@sim/desktop-bridge' import { ChipModalBody, + ChipModalDescription, ChipModalField, ChipModalFooter, ChipModalHeader, ChipModalSurface, } from '@sim/emcn' -import { observeShellSize } from '@/renderer/shell' +import { mountShellModal } from '@/renderer/shell' import type { ShellWindowApi } from '@/shared/shell' interface ServerModalProps { - server: ShellWindowApi['server'] | undefined + server: ShellWindowApi['server'] configuration?: DesktopServerConfiguration initialError?: string } @@ -20,9 +21,8 @@ function closeWindow() { window.close() } -function focusServerInput(element: HTMLDivElement | null) { - element?.querySelector('input')?.select() - return observeShellSize(element) +function mountServerModal(element: HTMLDivElement | null) { + return mountShellModal(element, closeWindow) } export function ServerModal({ server, configuration, initialError }: ServerModalProps) { @@ -43,10 +43,8 @@ export function ServerModal({ server, configuration, initialError }: ServerModal setError(undefined) setMessage('') try { - const result = await server?.setOrigin(origin) - if (!result) { - setError('The desktop shell is unavailable.') - } else if (!result.ok) { + const result = await server.setOrigin(origin) + if (!result.ok) { setError(result.error) } else if (result.unchanged) { setMessage('Already connected to this server.') @@ -61,12 +59,12 @@ export function ServerModal({ server, configuration, initialError }: ServerModal return ( Sim server -

+ Point this app at your own Sim deployment. Self-hosted servers must use HTTPS; localhost may use HTTP. -

+ { + vi.unstubAllGlobals() +}) + +describe('shell appearance bootstrap', () => { + it('keeps a newer theme received while the initial snapshot is pending', async () => { + let resolveInitial!: (theme: ShellTheme) => void + let update!: (theme: ShellTheme) => void + const api: ShellThemeApi = { + get: () => + new Promise((resolve) => { + resolveInitial = resolve + }), + onChange: (callback) => { + update = callback + return () => {} + }, + } + vi.stubGlobal('simShellTheme', api) + vi.stubGlobal( + 'matchMedia', + vi.fn(() => ({ matches: true, addEventListener: vi.fn() })) + ) + const ready = initializeShellPage() + expect(document.documentElement.className).toBe('dark') + expect(document.documentElement.style.colorScheme).toBe('dark') + update('light') + resolveInitial('dark') + await ready + expect(document.documentElement.className).toBe('light') + expect(document.documentElement.style.colorScheme).toBe('light') + }) + + it('follows system changes only before an app theme is known', async () => { + let update!: (theme: ShellTheme) => void + let systemChanged!: () => void + const media = { + matches: true, + addEventListener: vi.fn((_event, callback) => { + systemChanged = callback + }), + } + vi.stubGlobal( + 'matchMedia', + vi.fn(() => media) + ) + vi.stubGlobal('simShellTheme', { + get: async () => undefined, + onChange: (callback) => { + update = callback + return () => {} + }, + } satisfies ShellThemeApi) + await initializeShellPage() + expect(document.documentElement.className).toBe('dark') + media.matches = false + systemChanged() + expect(document.documentElement.className).toBe('light') + update('dark') + systemChanged() + expect(document.documentElement.className).toBe('dark') + }) +}) diff --git a/apps/desktop/src/renderer/shell.ts b/apps/desktop/src/renderer/shell.ts index 49a284719dc..9dc93c72ca7 100644 --- a/apps/desktop/src/renderer/shell.ts +++ b/apps/desktop/src/renderer/shell.ts @@ -1,17 +1,34 @@ -import type { ShellWindowApi } from '@/shared/shell' +import { focusChipModalContent } from '@sim/emcn' +import type { ShellTheme, ShellThemeApi, ShellWindowApi } from '@/shared/shell' export const shellWindow = (window as Window & { simShell?: ShellWindowApi }).simShell -/** Local windows follow the system theme independently of any reachable deployment. */ -export function initializeShellPage() { - const theme = window.matchMedia('(prefers-color-scheme: dark)') - const syncTheme = () => document.documentElement.classList.toggle('dark', theme.matches) +/** Uses Sim's resolved theme, falling back to the system before an app theme is known. */ +export async function initializeShellPage() { + const api = (window as Window & { simShellTheme?: ShellThemeApi }).simShellTheme + const system = window.matchMedia('(prefers-color-scheme: dark)') + let theme: ShellTheme | undefined + let receivedUpdate = false + const syncTheme = () => { + const dark = theme ? theme === 'dark' : system.matches + document.documentElement.classList.toggle('dark', dark) + document.documentElement.classList.toggle('light', !dark) + document.documentElement.style.colorScheme = dark ? 'dark' : 'light' + } + api?.onChange((next) => { + receivedUpdate = true + theme = next + syncTheme() + }) + syncTheme() + const initialTheme = await api?.get() + if (!receivedUpdate) theme = initialTheme syncTheme() - theme.addEventListener('change', syncTheme) + system.addEventListener('change', syncTheme) } /** Fits the native window to the complete modal, including changing inline messages. */ -export function observeShellSize(element: HTMLDivElement | null) { +function observeShellSize(element: HTMLDivElement | null) { if (!element || !shellWindow) return const resize = () => { const body = element.querySelector('[data-chip-modal-body]') @@ -30,3 +47,21 @@ export function observeShellSize(element: HTMLDivElement | null) { mutations.disconnect() } } + +/** Native host lifecycle; Escape also works when a disabled control leaves focus on the document. */ +export function mountShellModal(element: HTMLDivElement | null, dismiss: () => void) { + if (!element) return + focusChipModalContent(element) + const stopSizing = observeShellSize(element) + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') { + event.preventDefault() + dismiss() + } + } + element.ownerDocument.addEventListener('keydown', onKeyDown) + return () => { + stopSizing?.() + element.ownerDocument.removeEventListener('keydown', onKeyDown) + } +} diff --git a/apps/desktop/src/shared/shell.ts b/apps/desktop/src/shared/shell.ts index b71858f5fb9..842ef2a339d 100644 --- a/apps/desktop/src/shared/shell.ts +++ b/apps/desktop/src/shared/shell.ts @@ -1,9 +1,15 @@ import type { DesktopServerChangeResult, DesktopServerConfiguration } from '@sim/desktop-bridge' +export type ShellTheme = 'dark' | 'light' + +export interface ShellThemeApi { + get(): Promise + onChange(callback: (theme: ShellTheme | undefined) => void): () => void +} + export interface ShellDialogConfiguration { title: string - message: string - detail: string + text: string buttons: string[] defaultId: number cancelId: number diff --git a/apps/sim/app/(auth)/components/auth-shell.test.tsx b/apps/sim/app/(auth)/components/auth-shell.test.tsx index 2248773b9f6..deeb96a8f7f 100644 --- a/apps/sim/app/(auth)/components/auth-shell.test.tsx +++ b/apps/sim/app/(auth)/components/auth-shell.test.tsx @@ -14,7 +14,6 @@ vi.mock('next/link', () => ({ vi.mock('@/app/_shell/desktop-title-bar', () => ({ DesktopTitleBarLane: () => null })) vi.mock('@/app/(landing)/components/navbar/components', () => ({ LogoMark: ({ children }: { children: ReactNode }) => <>{children}, - SimWordmark: () => 'Sim', })) it('returns home through a document link so route-specific theme defaults reinitialize', () => { diff --git a/apps/sim/app/(auth)/components/auth-shell.tsx b/apps/sim/app/(auth)/components/auth-shell.tsx index d7107dfeb96..1b8cb741c0b 100644 --- a/apps/sim/app/(auth)/components/auth-shell.tsx +++ b/apps/sim/app/(auth)/components/auth-shell.tsx @@ -1,6 +1,7 @@ import type { ReactNode } from 'react' +import { SimWordmark } from '@sim/emcn' import { DesktopTitleBarLane } from '@/app/_shell/desktop-title-bar' -import { LogoMark, SimWordmark } from '@/app/(landing)/components/navbar/components' +import { LogoMark } from '@/app/(landing)/components/navbar/components' interface AuthShellProps { /** Centered content column (the form, status copy, etc.). */ diff --git a/apps/sim/app/(interfaces)/chat/components/header/header.tsx b/apps/sim/app/(interfaces)/chat/components/header/header.tsx index cfb5bfa58ca..fd96bb39cd1 100644 --- a/apps/sim/app/(interfaces)/chat/components/header/header.tsx +++ b/apps/sim/app/(interfaces)/chat/components/header/header.tsx @@ -1,9 +1,9 @@ 'use client' +import { SimWordmark } from '@sim/emcn' import Image from 'next/image' import Link from 'next/link' import { GithubIcon } from '@/components/icons' -import { SimWordmark } from '@/app/(landing)/components/navbar/components' import { useBrandConfig } from '@/ee/whitelabeling' interface ChatHeaderProps { diff --git a/apps/sim/app/(landing)/comparisons/[provider]/page.test.tsx b/apps/sim/app/(landing)/comparisons/[provider]/page.test.tsx index 906a040fdd5..6798b467cf9 100644 --- a/apps/sim/app/(landing)/comparisons/[provider]/page.test.tsx +++ b/apps/sim/app/(landing)/comparisons/[provider]/page.test.tsx @@ -5,7 +5,8 @@ import type { ReactNode } from 'react' import { renderToStaticMarkup } from 'react-dom/server' import { describe, expect, it, vi } from 'vitest' -vi.mock('@sim/emcn', () => ({ +vi.mock('@sim/emcn', async (importOriginal) => ({ + ...(await importOriginal()), cn: (...values: Array) => values.filter(Boolean).join(' '), Tooltip: { Root: ({ children }: { children: ReactNode }) => <>{children}, diff --git a/apps/sim/app/(landing)/comparisons/components/brand-icon-tile/brand-icon-tile.tsx b/apps/sim/app/(landing)/comparisons/components/brand-icon-tile/brand-icon-tile.tsx index 6c8911a805d..59f9809904b 100644 --- a/apps/sim/app/(landing)/comparisons/components/brand-icon-tile/brand-icon-tile.tsx +++ b/apps/sim/app/(landing)/comparisons/components/brand-icon-tile/brand-icon-tile.tsx @@ -1,7 +1,6 @@ import type { ComponentType, SVGProps } from 'react' -import { cn } from '@sim/emcn' +import { cn, SimWordmark } from '@sim/emcn' import type { CompetitorBrand } from '@/lib/compare/data' -import { SimWordmark } from '@/app/(landing)/components/navbar/components/sim-wordmark' export interface BrandIconTileProps { icon: ComponentType> diff --git a/apps/sim/app/(landing)/components/agent-momentum/agent-momentum.test.tsx b/apps/sim/app/(landing)/components/agent-momentum/agent-momentum.test.tsx index 1713a86b88f..3ee6730420a 100644 --- a/apps/sim/app/(landing)/components/agent-momentum/agent-momentum.test.tsx +++ b/apps/sim/app/(landing)/components/agent-momentum/agent-momentum.test.tsx @@ -4,7 +4,8 @@ import { renderToStaticMarkup } from 'react-dom/server' import { describe, expect, it, vi } from 'vitest' -vi.mock('@sim/emcn', () => ({ +vi.mock('@sim/emcn', async (importOriginal) => ({ + ...(await importOriginal()), cn: (...values: Array) => values.filter(Boolean).join(' '), })) diff --git a/apps/sim/app/(landing)/components/featured-customer/featured-customer.test.tsx b/apps/sim/app/(landing)/components/featured-customer/featured-customer.test.tsx index 65fe05ab37d..b3b05bcb278 100644 --- a/apps/sim/app/(landing)/components/featured-customer/featured-customer.test.tsx +++ b/apps/sim/app/(landing)/components/featured-customer/featured-customer.test.tsx @@ -7,7 +7,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const motionPreference = vi.hoisted(() => ({ reduced: false })) -vi.mock('@sim/emcn', () => ({ +vi.mock('@sim/emcn', async (importOriginal) => ({ + ...(await importOriginal()), Button: (props: React.ButtonHTMLAttributes) => + {downloadError && ( +

+ {downloadError.directUrl ? ( + <> + Unable to download automatically.{' '} + + Download directly + + + ) : ( + 'Unable to download this file. Please try again or request a new copy.' + )} +

+ )}
) } export function ChatFileDownloadAll({ files }: ChatFileDownloadAllProps) { const [isDownloading, setIsDownloading] = useState(false) + const [failedCount, setFailedCount] = useState(0) if (!files || files.length === 0) return null @@ -155,6 +215,8 @@ export function ChatFileDownloadAll({ files }: ChatFileDownloadAllProps) { if (isDownloading) return setIsDownloading(true) + setFailedCount(0) + let failures = 0 try { logger.info(`Initiating download for ${files.length} files`) @@ -162,8 +224,7 @@ export function ChatFileDownloadAll({ files }: ChatFileDownloadAllProps) { for (let i = 0; i < files.length; i++) { const file = files[i] try { - const url = getFileUrl(file) - await triggerDownload(url, file.name) + await triggerDownload(file) logger.info(`Downloaded file ${i + 1}/${files.length}: ${file.name}`) if (i < files.length - 1) { @@ -171,25 +232,35 @@ export function ChatFileDownloadAll({ files }: ChatFileDownloadAllProps) { } } catch (error) { logger.error(`Failed to download file ${file.name}:`, error) + failures++ } } } finally { + setFailedCount(failures) setIsDownloading(false) } } return ( - + {failedCount > 0 && ( +

+ Unable to download {failedCount} {failedCount === 1 ? 'file' : 'files'}. Please try + downloading them individually. +

)} - +
) } diff --git a/apps/sim/app/api/files/authorization.test.ts b/apps/sim/app/api/files/authorization.test.ts index a8738342a8d..e9771609833 100644 --- a/apps/sim/app/api/files/authorization.test.ts +++ b/apps/sim/app/api/files/authorization.test.ts @@ -491,3 +491,48 @@ describe('KB file live source authorization', () => { expect(get).not.toHaveBeenCalled() }) }) + +/** Execution downloads share the logs endpoint's current workspace permission check. */ +describe('execution file download authorization', () => { + const executionKey = 'execution/owner-workspace/workflow/run/image.png' + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('allows a current reader of the workspace named by the storage key', async () => { + mockGetUserEntityPermissions.mockResolvedValue('read') + await expect(verifyFileAccess(executionKey, USER_ID, undefined, 'execution')).resolves.toBe( + true + ) + expect(mockGetUserEntityPermissions).toHaveBeenCalledExactlyOnceWith( + USER_ID, + 'workspace', + 'owner-workspace' + ) + }) + + it('denies a caller without access to the file workspace', async () => { + mockGetUserEntityPermissions.mockResolvedValue(null) + await expect(verifyFileAccess(executionKey, USER_ID, undefined, 'execution')).resolves.toBe( + false + ) + }) + + it('rechecks access after membership is revoked', async () => { + mockGetUserEntityPermissions.mockResolvedValueOnce('read').mockResolvedValueOnce(null) + await expect(verifyFileAccess(executionKey, USER_ID, undefined, 'execution')).resolves.toBe( + true + ) + await expect(verifyFileAccess(executionKey, USER_ID, undefined, 'execution')).resolves.toBe( + false + ) + }) + + it('denies a malformed execution key before looking up workspace access', async () => { + await expect( + verifyFileAccess('execution/image.png', USER_ID, undefined, 'execution') + ).resolves.toBe(false) + expect(mockGetUserEntityPermissions).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/files/serve/[...path]/route.test.ts b/apps/sim/app/api/files/serve/[...path]/route.test.ts index e3937b9e78d..38c3cfcf73d 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.test.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.test.ts @@ -200,6 +200,26 @@ describe('File Serve API Route', () => { }) }) + it('requires authentication for execution downloads before reading bytes', async () => { + mockResolveStoredFileContext.mockResolvedValue('execution') + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: false, + error: 'Unauthorized', + }) + const response = await GET( + new NextRequest( + 'http://localhost/api/files/serve/execution%2Fworkspace%2Fworkflow%2Frun%2Fimage.png?context=execution' + ), + { + params: Promise.resolve({ path: ['execution/workspace/workflow/run/image.png'] }), + } + ) + expect(response.status).toBe(401) + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(mockReadFile).not.toHaveBeenCalled() + expect(storageServiceMockFns.mockDownloadFile).not.toHaveBeenCalled() + }) + it('bounds every buffered read at the shared transfer ceiling', async () => { mockIsUsingCloudStorage.mockReturnValue(true) mockResolveStoredFileContext.mockResolvedValue('copilot') From f21bf92ae00d55de192fd641fb5fd08c9d33a03b Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Tue, 15 Sep 2026 15:09:46 -0700 Subject: [PATCH 08/15] fix(agent): authorize workspace attachments through execution delegation (#7859) * fix(agent): authorize workspace attachments through execution delegation * test(agent): preserve rejection of unprefixed attachment keys --- .../handlers/agent/agent-handler.test.ts | 46 ++++ .../executor/handlers/agent/agent-handler.ts | 73 ++++--- .../agent/memory-harness.postgres.test.ts | 202 +++++++++++++++++- .../file/materialization-context.test.ts | 110 ++++++++++ .../internal/file/materialization-context.ts | 33 +++ .../file-attachments-authorization.test.ts | 97 +++++++++ .../providers/file-attachments.server.test.ts | 121 ++++++++++- apps/sim/providers/file-attachments.server.ts | 47 ++-- apps/sim/providers/index.test.ts | 24 ++- apps/sim/providers/index.ts | 4 +- 10 files changed, 703 insertions(+), 54 deletions(-) create mode 100644 apps/sim/lib/internal/file/materialization-context.test.ts create mode 100644 apps/sim/lib/internal/file/materialization-context.ts create mode 100644 apps/sim/providers/file-attachments-authorization.test.ts diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index b60638e6953..edd09b45fd2 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -953,6 +953,52 @@ describe('AgentBlockHandler', () => { expect(inputs).toEqual(rawInputs) }) + it.each([ + 'url/https://example.com/image.png', + '', + 'provider-file-id', + 'profile-pictures/avatar.png', + ])('preserves inline bytes for an actorless request with key %s', async (key) => { + mockGetProviderFromModel.mockReturnValue('openai') + await handler.execute( + { + ...mockContext, + principal: { + kind: 'system', + serviceId: 'chat', + workspaceId: 'test-workspace', + workflowId: 'test-workflow', + }, + executorDelegationOrigin: undefined, + }, + mockBlock, + { + model: 'gpt-4o', + messages: [ + { + role: 'user', + content: 'Analyze this image', + files: [ + { + id: 'file-1', + key, + name: 'image.png', + url: 'https://example.com/image.png', + size: 5, + type: 'image/png', + base64: 'aW1hZ2U=', + }, + ], + }, + ], + apiKey: 'test-api-key', + } + ) + expect(mockExecuteProviderRequest.mock.calls[0][1].messages[0].files).toEqual([ + expect.objectContaining({ base64: 'aW1hZ2U=' }), + ]) + }) + it('normalizes the persisted workspace-picker shape before provider execution', async () => { const key = 'workspace/ws-1/example.png' const hydrationSpy = vi diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index a2d1295facc..f33ff7b8ea1 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -9,6 +9,7 @@ import { selectModelSchemaInputPaths, } from '@/lib/execution/model-input-provenance' import { readAvailableCustomToolByIdOrTitleAsExecutor } from '@/lib/internal/custom-tools/read-available-by-id-or-title' +import { resolveExecutorFileMaterializationContext } from '@/lib/internal/file/materialization-context' import { discoverMcpServerToolsAsExecutor } from '@/lib/internal/mcp/discover-tools' import { readWorkflowInputFieldsForTool, @@ -31,6 +32,7 @@ import { MODEL_SUPPORTED_IMAGE_MIME_TYPES, processFilesToUserFiles, type RawFileInput, + tryInferContextFromKey, } from '@/lib/uploads/utils/file-utils' import { selectModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' @@ -63,7 +65,7 @@ import type { ToolInput, } from '@/executor/handlers/agent/types' import { parseResponseFormat } from '@/executor/handlers/shared/response-format' -import type { BlockHandler, ExecutionContext, StreamingExecution } from '@/executor/types' +import type { BlockHandler, ExecutionContext, StreamingExecution, UserFile } from '@/executor/types' import { collectBlockData } from '@/executor/utils/block-data' import { stringifyJSON } from '@/executor/utils/json' import { projectResolvedSecretDiagnosticContent } from '@/executor/utils/resolved-secret-content-projection' @@ -1481,7 +1483,6 @@ export class AgentBlockHandler implements BlockHandler { throw new Error(`File attachments are not supported for provider "${providerId}"`) } - const requestId = ctx.executionId || ctx.workflowId || 'agent-files' const nextMessages = [...messages] const inlineMaxBytes = getInlineHydrationMaxBytes(providerId) @@ -1493,36 +1494,46 @@ export class AgentBlockHandler implements BlockHandler { } const unsafeGeneratedDocumentFiles = new Set() - const hydratedFiles = await hydrateUserFilesWithBase64(message.files, { - requestId, - workspaceId: ctx.workspaceId, - workflowId: ctx.workflowId, - executionId: ctx.executionId, - largeValueExecutionIds: ctx.largeValueExecutionIds, - largeValueKeys: ctx.largeValueKeys, - fileKeys: ctx.fileKeys, - allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope, - userId: ctx.userId, - principal: ctx.principal, - logger, - maxBytes: inlineMaxBytes, - onServableFileContributors: async (file, contributors) => { - if (!ctx.workspaceId) return - for (const identity of contributors) { - const safe = await importWorkspaceFileSecretProvenanceForModelView({ - workspaceId: ctx.workspaceId, - identity, - registry: ctx.resolvedSecretTraceRegistry, - view: 'opaque', - ...(ctx.userId ? { actorUserId: ctx.userId } : {}), - }) - if (!safe) { - unsafeGeneratedDocumentFiles.add(`${file.key}:${file.id}`) - return - } - } - }, + const groups = new Map>() + message.files.forEach((file, index) => { + const workspaceFile = + ctx.principal?.kind === 'system' && tryInferContextFromKey(file.key) === 'workspace' + const group = groups.get(workspaceFile) ?? [] + group.push({ file, index }) + groups.set(workspaceFile, group) }) + const hydratedFiles = [...message.files] + await Promise.all( + [...groups.values()].map(async (group) => { + const hydrated = await hydrateUserFilesWithBase64( + group.map(({ file }) => file), + { + ...(await resolveExecutorFileMaterializationContext(ctx, group[0].file)), + logger, + maxBytes: inlineMaxBytes, + onServableFileContributors: async (file, contributors) => { + if (!ctx.workspaceId) return + for (const identity of contributors) { + const safe = await importWorkspaceFileSecretProvenanceForModelView({ + workspaceId: ctx.workspaceId, + identity, + registry: ctx.resolvedSecretTraceRegistry, + view: 'opaque', + ...(ctx.userId ? { actorUserId: ctx.userId } : {}), + }) + if (!safe) { + unsafeGeneratedDocumentFiles.add(`${file.key}:${file.id}`) + return + } + } + }, + } + ) + group.forEach(({ index }, fileIndex) => { + hydratedFiles[index] = hydrated[fileIndex] + }) + }) + ) const modelSafeHydratedFiles = hydratedFiles.flatMap((file, fileIndex) => { if (unsafeGeneratedDocumentFiles.has(`${file.key}:${file.id}`)) return [] diff --git a/apps/sim/executor/handlers/agent/memory-harness.postgres.test.ts b/apps/sim/executor/handlers/agent/memory-harness.postgres.test.ts index 7c9ebf1a3f5..b13d1b7f773 100644 --- a/apps/sim/executor/handlers/agent/memory-harness.postgres.test.ts +++ b/apps/sim/executor/handlers/agent/memory-harness.postgres.test.ts @@ -64,13 +64,22 @@ vi.mock('@/lib/api-key/byok', () => ({ import { memory, memorySecretProvenance, + resumeQueue, + workflow, + workflowExecutionLogs, + workspace, workspaceFileSecretProvenance, workspaceFiles, } from '@sim/db/schema' import { hashDurableSecretProvenanceValue } from '@/lib/execution/durable-secret-provenance' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution/execution-file-manager' -import { EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { deleteFile } from '@/lib/uploads/core/storage-service' +import { + EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, + initializeWorkspaceFileSecretProvenanceInTx, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { deleteFile, uploadFile } from '@/lib/uploads/core/storage-service' +import { insertImmutableFileMetadata } from '@/lib/uploads/server/metadata' +import { readWorkspaceFileRecordByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' import { AgentBlockHandler } from '@/executor/handlers/agent/agent-handler' import type { AgentInputs, Message } from '@/executor/handlers/agent/types' import type { ExecutionContext, StreamingExecution, UserFile } from '@/executor/types' @@ -176,7 +185,7 @@ async function executeTurn(ctx: ExecutionContext, inputs: AgentInputs): Promise< return drained.answerText } -/** Generate only the four production tables exercised here; unrelated application FKs are omitted. */ +/** Generate the production tables exercised here; unrelated application FKs are omitted. */ async function createTable(table: PgTable): Promise { if (!connection) throw new Error('Missing harness database') const dialect = new PgDialect() @@ -190,7 +199,9 @@ async function createTable(table: PgTable): Promise { column.default === undefined ? '' : ` DEFAULT ${dialect.sqlToQuery(defaultValue.inlineParams()).sql}` - return `"${column.name}" ${column.getSQLType()}${column.primary ? ' PRIMARY KEY' : ''}${column.notNull ? ' NOT NULL' : ''}${defaultSql}` + const type = + 'enumValues' in column && Array.isArray(column.enumValues) ? 'text' : column.getSQLType() + return `"${column.name}" ${type}${column.primary ? ' PRIMARY KEY' : ''}${column.notNull ? ' NOT NULL' : ''}${defaultSql}` }) await connection.unsafe(`CREATE TABLE "${config.name}" (${columns.join(', ')})`) } @@ -318,8 +329,27 @@ describe.skipIf(!databaseUrl)( memorySecretProvenance, workspaceFiles, workspaceFileSecretProvenance, + workspace, + workflow, + workflowExecutionLogs, + resumeQueue, ]) await createTable(table) + await fixture.database.insert(workspace).values({ + id: scope.workspaceId, + name: 'Attachment harness', + ownerId: scope.userId, + billedAccountUserId: scope.userId, + }) + await fixture.database.insert(workflow).values({ + id: scope.workflowId, + workspaceId: scope.workspaceId, + userId: scope.userId, + name: 'Attachment harness', + createdAt: new Date(), + updatedAt: new Date(), + lastSynced: new Date(), + }) await connection.unsafe( `CREATE UNIQUE INDEX memory_workspace_key_idx ON memory(workspace_id, key)` ) @@ -351,6 +381,170 @@ describe.skipIf(!databaseUrl)( } }) + it.each( + (['openai', 'anthropic'] as const).flatMap((provider) => + [false, true].map((streaming) => ({ provider, streaming })) + ) + )( + 'deployed chat reads remembered workspace files with $provider, streaming=$streaming', + async ({ provider, streaming }) => { + if (!fixture.database || !connection) throw new Error('Missing harness database') + vi.stubGlobal('fetch', interceptFetch) + outbound = [] + const conversationId = generateId() + const pdf = await PDFDocument.create() + pdf + .addPage() + .drawText('A workspace image-edit result can be recalled without a new upload.') + const buffer = Buffer.from(await pdf.save()) + const key = `workspace/${scope.workspaceId}/${generateId()}/result.pdf` + await uploadFile({ + file: buffer, + fileName: 'result.pdf', + contentType: 'application/pdf', + context: 'workspace', + preserveKey: true, + customKey: key, + persistMetadata: false, + }) + const record = await fixture.database.transaction(async (tx) => { + const record = await insertImmutableFileMetadata( + { + id: generateId(), + key, + userId: scope.userId, + workspaceId: scope.workspaceId, + context: 'workspace', + originalName: 'result.pdf', + contentType: 'application/pdf', + size: buffer.length, + }, + tx + ) + await initializeWorkspaceFileSecretProvenanceInTx( + tx, + record.id, + record.contentUpdatedAt, + EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE + ) + return record + }) + const file: UserFile = { + id: record.id, + name: 'result.pdf', + key, + url: '', + size: buffer.length, + type: 'application/pdf', + context: 'workspace', + } + const createChatContext = async () => { + const ctx = context(streaming) + ctx.principal = { + kind: 'system', + serviceId: 'chat', + workspaceId: scope.workspaceId, + workflowId: scope.workflowId, + } + const deploymentVersionId = generateId() + ctx.executorDelegationOrigin = { + workflowId: scope.workflowId, + executionId: ctx.executionId, + principal: ctx.principal, + currentWorkflow: { + workflowId: scope.workflowId, + mode: 'deployment', + deploymentVersionId, + }, + } + await fixture.database!.insert(workflowExecutionLogs).values({ + id: generateId(), + workflowId: scope.workflowId, + workspaceId: scope.workspaceId, + executionId: ctx.executionId!, + deploymentVersionId, + stateSnapshotId: generateId(), + level: 'info', + status: 'running', + trigger: 'chat', + startedAt: new Date(), + }) + return ctx + } + const inputs: AgentInputs = { + model: models[provider], + apiKey: apiKey(provider), + maxTokens: '128', + memoryType: 'conversation', + conversationId, + userPrompt: 'Read the attached PDF and reply exactly READY.', + } + transportReply = 'READY' + const firstContext = await createChatContext() + await expect( + readWorkspaceFileRecordByKey.execute({ + principal: firstContext.principal!, + input: { key, assertedWorkspaceId: scope.workspaceId }, + }) + ).rejects.toThrow('Principal kind system') + expect(await executeTurn(firstContext, { ...inputs, files: [file] })).toBe('READY') + expect(requestFiles(outbound[0])).toEqual([buffer.toString('base64')]) + const stored = await readConversation(conversationId) + expect(stored.data[0].files).toEqual([file]) + expect(JSON.stringify(stored)).not.toContain('base64') + + expect( + await executeTurn(await createChatContext(), { + ...inputs, + userPrompt: 'Read the earlier PDF again and reply exactly READY.', + }) + ).toBe('READY') + expect(requestFiles(outbound[1])).toEqual([buffer.toString('base64')]) + + const missingOrigin = await createChatContext() + missingOrigin.executorDelegationOrigin = undefined + await expect(executeTurn(missingOrigin, inputs)).rejects.toThrow() + const otherWorkspace = await createChatContext() + otherWorkspace.workspaceId = generateId() + await expect( + executeTurn(otherWorkspace, { ...inputs, files: [file], memoryType: 'none' }) + ).rejects.toThrow('could not be read') + const terminalRun = await createChatContext() + await connection`UPDATE workflow_execution_logs SET status = 'completed' WHERE execution_id = ${terminalRun.executionId!}` + await expect(executeTurn(terminalRun, inputs)).rejects.toThrow('active workflow execution') + const mismatchedDeployment = await createChatContext() + mismatchedDeployment.executorDelegationOrigin!.currentWorkflow = { + workflowId: scope.workflowId, + mode: 'deployment', + deploymentVersionId: generateId(), + } + await expect(executeTurn(mismatchedDeployment, inputs)).rejects.toThrow( + 'active workflow execution' + ) + await connection`UPDATE workspace_files SET deleted_at = NOW() WHERE id = ${file.id}` + await expect(executeTurn(await createChatContext(), inputs)).rejects.toThrow( + 'could not be read' + ) + expect(outbound).toHaveLength(2) + report.push({ + provider, + streaming, + workspaceAttachment: true, + stored, + controls: { + missingOrigin: 'blocked before HTTP', + differentWorkspace: 'blocked before HTTP', + terminalRun: 'blocked before HTTP', + mismatchedDeployment: 'blocked before HTTP', + deletedFile: 'blocked before HTTP', + }, + passed: true, + }) + await deleteFile({ key, context: 'workspace' }) + }, + 150_000 + ) + it.each( ( [ diff --git a/apps/sim/lib/internal/file/materialization-context.test.ts b/apps/sim/lib/internal/file/materialization-context.test.ts new file mode 100644 index 00000000000..881f3463dc4 --- /dev/null +++ b/apps/sim/lib/internal/file/materialization-context.test.ts @@ -0,0 +1,110 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionContext } from '@/executor/types' + +const { bindDelegation } = vi.hoisted(() => ({ bindDelegation: vi.fn() })) + +vi.mock('@/lib/auth/internal-delegation', () => ({ + bindInternalExecutorDelegation: bindDelegation, +})) + +import { resolveExecutorFileMaterializationContext } from '@/lib/internal/file/materialization-context' + +const workspaceFile = { key: 'workspace/workspace-1/image.png' } +const systemPrincipal = { + kind: 'system', + serviceId: 'chat', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', +} as const + +function context(): ExecutionContext { + return { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'billing-owner', + principal: systemPrincipal, + executorDelegationOrigin: { + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: systemPrincipal, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + }, + fileKeys: ['execution/workspace-1/workflow-1/prior/image.png'], + } as ExecutionContext +} + +describe('executor file materialization context', () => { + beforeEach(() => { + vi.clearAllMocks() + bindDelegation.mockResolvedValue({ kind: 'delegated', serviceId: 'executor' }) + }) + + it('binds actorless workspace reads to the current deployment without inventing a subject', async () => { + const ctx = context() + const result = await resolveExecutorFileMaterializationContext(ctx, workspaceFile) + expect(bindDelegation).toHaveBeenCalledWith( + expect.objectContaining({ + principal: systemPrincipal, + workflowId: 'workflow-1', + executionId: 'execution-1', + currentWorkflow: ctx.executorDelegationOrigin?.currentWorkflow, + }), + { audience: 'sim:workspace-files', compatibilityActorUserId: 'billing-owner' } + ) + expect(bindDelegation.mock.calls[0][0].subjectUserId).toBeUndefined() + expect(result.principal).toEqual({ kind: 'delegated', serviceId: 'executor' }) + expect(result.userId).toBeUndefined() + expect(result.fileKeys).toBe(ctx.fileKeys) + expect(ctx.principal).toBe(systemPrincipal) + }) + + it.each([ + { kind: 'session', userId: 'reader', sessionId: 'session-1' }, + { kind: 'personal_api_key', userId: 'reader', keyId: 'key-1' }, + { kind: 'workspace_api_key', workspaceId: 'workspace-1', keyId: 'key-1' }, + ] as const)('preserves the existing $kind workspace authority', async (principal) => { + const ctx = { ...context(), principal } + expect((await resolveExecutorFileMaterializationContext(ctx, workspaceFile)).principal).toBe( + principal + ) + expect(bindDelegation).not.toHaveBeenCalled() + }) + + it.each([ + 'execution/workspace-1/workflow-1/execution-1/image.png', + 'knowledge-base/document.png', + 'url/https://example.com/image.png', + '', + 'provider-file-id', + 'profile-pictures/avatar.png', + ])('does not replace the original identity for %s', async (key) => { + const ctx = context() + expect((await resolveExecutorFileMaterializationContext(ctx, { key })).principal).toBe( + systemPrincipal + ) + expect(bindDelegation).not.toHaveBeenCalled() + }) + + it('fails closed without a trusted executor origin', async () => { + const ctx = context() + ctx.executorDelegationOrigin = undefined + await expect(resolveExecutorFileMaterializationContext(ctx, workspaceFile)).rejects.toThrow() + expect(bindDelegation).not.toHaveBeenCalled() + }) + + it('propagates a failed current workflow binding without an owner fallback', async () => { + bindDelegation.mockRejectedValueOnce(new Error('Workflow binding invalid')) + await expect( + resolveExecutorFileMaterializationContext(context(), workspaceFile) + ).rejects.toThrow('Workflow binding invalid') + expect(bindDelegation).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/internal/file/materialization-context.ts b/apps/sim/lib/internal/file/materialization-context.ts new file mode 100644 index 00000000000..8c19a647522 --- /dev/null +++ b/apps/sim/lib/internal/file/materialization-context.ts @@ -0,0 +1,33 @@ +import type { ExecutionMaterializationContext } from '@/lib/execution/payloads/materialization.server' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import { tryInferContextFromKey } from '@/lib/uploads/utils/file-utils' +import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' +import type { ExecutionContext, UserFile } from '@/executor/types' + +/** Binds actorless workspace reads to the same trusted execution authority as File tools. */ +export async function resolveExecutorFileMaterializationContext( + context: ExecutionContext, + file: Pick +): Promise { + const requiresDelegation = + context.principal?.kind === 'system' && tryInferContextFromKey(file.key) === 'workspace' + const principal = requiresDelegation + ? await createExecutorPrincipalFromExecutionContext({ + context, + audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + }) + : context.principal + + return { + principal, + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + largeValueExecutionIds: context.largeValueExecutionIds, + largeValueKeys: context.largeValueKeys, + fileKeys: context.fileKeys, + allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, + userId: requiresDelegation ? undefined : context.userId, + requestId: context.executionId || context.workflowId || 'agent-files', + } +} diff --git a/apps/sim/providers/file-attachments-authorization.test.ts b/apps/sim/providers/file-attachments-authorization.test.ts new file mode 100644 index 00000000000..78e8260fcc9 --- /dev/null +++ b/apps/sim/providers/file-attachments-authorization.test.ts @@ -0,0 +1,97 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionContext, UserFile } from '@/executor/types' + +const { presign, download, metadata, permission } = vi.hoisted(() => ({ + presign: vi.fn(), + download: vi.fn(), + metadata: vi.fn(), + permission: vi.fn(), +})) + +vi.mock('@/lib/uploads', () => ({ + StorageService: { hasCloudStorage: () => true, generatePresignedDownloadUrl: presign }, + getFileMetadata: metadata, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: download, +})) + +vi.mock('@/lib/uploads/server/metadata', () => ({ + getFileMetadataByKey: metadata, +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + getUserEntityPermissions: permission, +})) + +import { resolveTrustedFileContext } from '@/lib/uploads/utils/file-utils' +import { + attachLargeFileRemoteUrls, + uploadLargeFilesToProvider, +} from '@/providers/file-attachments.server' +import type { ProviderRequest } from '@/providers/types' + +/** Authorization and key inference are real: mocking either hid this pre-existing refusal. */ +describe('provider attachment storage-key authorization', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each( + (['workspace', 'execution', 'chat', 'copilot', 'knowledge-base'] as const).flatMap((context) => + (['standalone', 'session', 'system'] as const).map((caller) => ({ context, caller })) + ) + )( + 'rejects unprefixed $context keys for $caller before reading or signing bytes', + async ({ context, caller }) => { + const file: UserFile = { + id: 'file-1', + name: 'document.pdf', + key: 'legacy-file-id/document.pdf', + url: '', + size: 10 * 1024 * 1024, + type: 'application/pdf', + context, + } + const request: ProviderRequest = { + model: 'gpt-4.1', + userId: 'billing-owner', + messages: [{ role: 'user', content: 'Read this file', files: [file] }], + } + const executionContext = + caller === 'standalone' + ? undefined + : ({ + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'billing-owner', + principal: + caller === 'session' + ? { kind: 'session', userId: 'acting-user', sessionId: 'session-1' } + : { + kind: 'system', + serviceId: 'chat', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + } as ExecutionContext) + + expect(resolveTrustedFileContext(file.key, file.context)).toBe(context) + await expect(attachLargeFileRemoteUrls(request, 'openai', executionContext)).rejects.toThrow() + expect(presign).not.toHaveBeenCalled() + + file.remoteUrl = 'https://storage.example.com/forged' + await expect( + uploadLargeFilesToProvider(request, 'openai', executionContext) + ).rejects.toThrow() + expect(download).not.toHaveBeenCalled() + expect(metadata).not.toHaveBeenCalled() + expect(permission).not.toHaveBeenCalled() + } + ) +}) diff --git a/apps/sim/providers/file-attachments.server.test.ts b/apps/sim/providers/file-attachments.server.test.ts index f88a10bf5f3..1e36a22f4b4 100644 --- a/apps/sim/providers/file-attachments.server.test.ts +++ b/apps/sim/providers/file-attachments.server.test.ts @@ -2,9 +2,11 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionContext } from '@/executor/types' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { buildOpenAIMessageContent, + getProviderFileStrategy, INLINE_ATTACHMENT_THRESHOLD_BYTES, LARGE_FILE_PATH_THRESHOLD_BYTES, } from '@/providers/attachments' @@ -13,6 +15,7 @@ import { getInlineHydrationMaxBytes, uploadLargeFilesToProvider, } from '@/providers/file-attachments.server' +import { PROVIDER_DEFINITIONS } from '@/providers/models' import { runWithProviderRuntimeContext } from '@/providers/runtime-context' import type { ProviderRequest } from '@/providers/types' @@ -21,16 +24,32 @@ const { mockGeneratePresignedDownloadUrl, mockHasCloudStorage, mockVerifyFileAccess, + mockCreateExecutorPrincipal, + mockAssertUserFileContentAccess, + mockGoogleUpload, } = vi.hoisted(() => ({ mockDownloadServableFileFromStorage: vi.fn(), mockGeneratePresignedDownloadUrl: vi.fn(), mockHasCloudStorage: vi.fn(), mockVerifyFileAccess: vi.fn(), + mockCreateExecutorPrincipal: vi.fn(), + mockAssertUserFileContentAccess: vi.fn(), + mockGoogleUpload: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mockCreateExecutorPrincipal, +})) + +vi.mock('@/lib/execution/payloads/materialization.server', () => ({ + assertUserFileContentAccess: mockAssertUserFileContentAccess, })) vi.mock('@google/genai', () => ({ FileState: { PROCESSING: 'PROCESSING', FAILED: 'FAILED' }, - GoogleGenAI: class {}, + GoogleGenAI: class { + files = { upload: mockGoogleUpload } + }, })) vi.mock('@/lib/uploads', () => ({ @@ -82,6 +101,17 @@ describe('OpenAI large-file attachment lifecycle', () => { vi.clearAllMocks() mockHasCloudStorage.mockReturnValue(true) mockVerifyFileAccess.mockResolvedValue(true) + mockCreateExecutorPrincipal.mockResolvedValue({ + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-1', + }) + mockAssertUserFileContentAccess.mockResolvedValue(undefined) + mockGoogleUpload.mockResolvedValue({ + name: 'files/harness', + uri: 'https://generativelanguage.googleapis.com/files/harness', + state: 'ACTIVE', + }) mockGeneratePresignedDownloadUrl.mockResolvedValue('https://storage.example.com/signed') mockDownloadServableFileFromStorage.mockResolvedValue({ buffer: Buffer.alloc(CSV_BYTES, 0x61), @@ -185,4 +215,93 @@ describe('OpenAI large-file attachment lifecycle', () => { expect(file?.remoteUrl).toBeUndefined() expect(file?.providerFileId).toBeUndefined() }) + + const executionContext = { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'billing-owner', + principal: { + kind: 'system', + serviceId: 'chat', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + }, + executorDelegationOrigin: { workflowId: 'workflow-1', executionId: 'execution-1' }, + } as ExecutionContext + + it.each(Object.keys(PROVIDER_DEFINITIONS))( + 'preserves %s attachment strategy while authorizing remote bytes as the execution', + async (provider) => { + const request = makeRequest(INLINE_ATTACHMENT_THRESHOLD_BYTES + 1) + await attachLargeFileRemoteUrls(request, provider, executionContext) + const largeFile = getProviderFileStrategy(provider) !== 'inline' + expect(mockCreateExecutorPrincipal).toHaveBeenCalledTimes(largeFile ? 1 : 0) + expect(mockAssertUserFileContentAccess).toHaveBeenCalledTimes(largeFile ? 1 : 0) + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledTimes(largeFile ? 1 : 0) + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + if (largeFile) { + expect(mockCreateExecutorPrincipal).toHaveBeenCalledWith({ + context: executionContext, + audience: 'sim:workspace-files', + }) + expect(mockAssertUserFileContentAccess).toHaveBeenCalledWith( + request.messages?.[0].files?.[0], + expect.objectContaining({ + principal: { kind: 'delegated', serviceId: 'executor', workspaceId: 'workspace-1' }, + userId: undefined, + executionId: 'execution-1', + }) + ) + } + } + ) + + it('rechecks current access before a Files API upload and does not fall back to the billing owner', async () => { + const request = makeRequest(CSV_BYTES) + await attachLargeFileRemoteUrls(request, 'openai', executionContext) + mockAssertUserFileContentAccess.mockRejectedValueOnce(new Error('Access revoked')) + await expect(uploadLargeFilesToProvider(request, 'openai', executionContext)).rejects.toThrow( + 'Access revoked' + ) + expect(mockCreateExecutorPrincipal).toHaveBeenCalledTimes(2) + expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it.each(['openai', 'google'])( + 'uploads an authorized actorless workspace file through %s', + async (provider) => { + const request = makeRequest(CSV_BYTES) + await attachLargeFileRemoteUrls(request, provider, executionContext) + await uploadLargeFilesToProvider(request, provider, executionContext) + expect(mockCreateExecutorPrincipal).toHaveBeenCalledTimes(2) + expect(mockAssertUserFileContentAccess).toHaveBeenCalledTimes(2) + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + const file = request.messages?.[0].files?.[0] + if (provider === 'openai') expect(file?.providerFileId).toBe('file-abc') + else + expect(file?.providerFileUri).toBe( + 'https://generativelanguage.googleapis.com/files/harness' + ) + } + ) + + it('does not mint a remote URL after execution authorization fails', async () => { + mockCreateExecutorPrincipal.mockRejectedValueOnce(new Error('Run no longer active')) + await expect( + attachLargeFileRemoteUrls(makeRequest(CSV_BYTES), 'openai', executionContext) + ).rejects.toThrow('Run no longer active') + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + }) + + it('ignores forged execution authority on an ordinary provider request', async () => { + const request = { ...makeRequest(CSV_BYTES), executionContext } + mockVerifyFileAccess.mockResolvedValueOnce(false) + await expect(attachLargeFileRemoteUrls(request, 'openai')).rejects.toThrow('not accessible') + expect(mockCreateExecutorPrincipal).not.toHaveBeenCalled() + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/providers/file-attachments.server.ts b/apps/sim/providers/file-attachments.server.ts index 39fcbf9267e..6fad9893e81 100644 --- a/apps/sim/providers/file-attachments.server.ts +++ b/apps/sim/providers/file-attachments.server.ts @@ -2,11 +2,13 @@ import { FileState, GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { assertUserFileContentAccess } from '@/lib/execution/payloads/materialization.server' +import { resolveExecutorFileMaterializationContext } from '@/lib/internal/file/materialization-context' import { StorageService } from '@/lib/uploads' import { resolveTrustedFileContext } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { verifyFileAccess } from '@/app/api/files/authorization' -import type { UserFile } from '@/executor/types' +import type { ExecutionContext, UserFile } from '@/executor/types' import { formatAttachmentSizes, getProviderAttachmentMaxBytes, @@ -72,7 +74,8 @@ export function canUseProviderLargeFilePath(providerId: ProviderId | string): bo */ export async function attachLargeFileRemoteUrls( request: ProviderRequest, - providerId: ProviderId | string + providerId: ProviderId | string, + executionContext?: ExecutionContext ): Promise { for (const file of iterateRequestFiles(request.messages)) { file.providerFileId = undefined @@ -102,16 +105,21 @@ export async function attachLargeFileRemoteUrls( continue } - if (!request.userId) { - throw new Error( - `File "${file.name}" requires an authenticated user for provider "${providerId}"` - ) - } - - const context = resolveTrustedFileContext(file.key, file.context) - const hasAccess = await verifyFileAccess(file.key, request.userId, undefined, context, false) - if (!hasAccess) { - throw new Error(`File "${file.name}" is not accessible for provider "${providerId}"`) + let context: ReturnType + if (executionContext) { + context = resolveTrustedFileContext(file.key, file.context) + await assertFileAccessForUpload(file, request.userId, executionContext) + } else { + if (!request.userId) { + throw new Error( + `File "${file.name}" requires an authenticated user for provider "${providerId}"` + ) + } + context = resolveTrustedFileContext(file.key, file.context) + const hasAccess = await verifyFileAccess(file.key, request.userId, undefined, context, false) + if (!hasAccess) { + throw new Error(`File "${file.name}" is not accessible for provider "${providerId}"`) + } } file.remoteUrl = await StorageService.generatePresignedDownloadUrl( @@ -130,7 +138,8 @@ export async function attachLargeFileRemoteUrls( */ export async function uploadLargeFilesToProvider( request: ProviderRequest, - providerId: ProviderId | string + providerId: ProviderId | string, + executionContext?: ExecutionContext ): Promise { if (getProviderFileStrategy(providerId) !== 'files-api') return @@ -142,7 +151,7 @@ export async function uploadLargeFilesToProvider( for (const group of groups) { const [representative] = group - await assertFileAccessForUpload(representative, request.userId) + await assertFileAccessForUpload(representative, request.userId, executionContext) if (providerId === 'openai') { await uploadOpenAIFile(representative, request.apiKey, maxBytes, request.abortSignal) } else if (ai) { @@ -162,11 +171,19 @@ export async function uploadLargeFilesToProvider( */ async function assertFileAccessForUpload( file: UserFile, - userId: string | undefined + userId: string | undefined, + executionContext?: ExecutionContext ): Promise { if (!file.key) { throw new Error(`File "${file.name}" has no storage key`) } + if (executionContext) { + await assertUserFileContentAccess( + file, + await resolveExecutorFileMaterializationContext(executionContext, file) + ) + return + } if (!userId) { throw new Error(`File "${file.name}" requires an authenticated user to upload`) } diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index e7219714432..5594b448d99 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -45,7 +45,7 @@ vi.mock('@/tools', () => ({ executeTool: (...args: unknown[]) => mockExecuteTool(...args), })) -import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' +import type { ExecutionContext, NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { executeProviderRequest } from '@/providers' import { executeProviderTool } from '@/providers/runtime-context' @@ -114,6 +114,28 @@ describe('executeProviderRequest — tool identities', () => { vi.clearAllMocks() }) + it('passes trusted execution context to both attachment authorization stages without serializing it', async () => { + const executionContext = { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + } as ExecutionContext + mockExecuteRequest.mockResolvedValueOnce({ content: 'ready', model: 'test-model' }) + await executeProviderRequest('anthropic', { model: 'test-model' }, { executionContext }) + expect(mockAttachLargeFileRemoteUrls).toHaveBeenCalledWith( + expect.objectContaining({ model: 'test-model' }), + 'anthropic', + executionContext + ) + expect(mockUploadLargeFilesToProvider).toHaveBeenCalledWith( + expect.objectContaining({ model: 'test-model' }), + 'anthropic', + executionContext + ) + expect(mockExecuteRequest.mock.calls[0][0]).not.toHaveProperty('executionContext') + expect(mockExecuteRequest.mock.calls[0][0]).not.toHaveProperty('principal') + }) + it('sends unique opaque ids and projects provider aliases out of the response', async () => { const tools = [ makeProviderTool('gmail_send', 'credential-a'), diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 8c7b315f677..5d1f5819a9d 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -265,8 +265,8 @@ export async function executeProviderRequest( } const response = await runWithProviderRuntimeContext(requestRuntimeContext, async () => { - await attachLargeFileRemoteUrls(modelSafeRequest, providerId) - await uploadLargeFilesToProvider(modelSafeRequest, providerId) + await attachLargeFileRemoteUrls(modelSafeRequest, providerId, runtimeContext?.executionContext) + await uploadLargeFilesToProvider(modelSafeRequest, providerId, runtimeContext?.executionContext) return provider.executeRequest(modelSafeRequest) }) From 22c149a558d8ee5293f98d405e053133b3cae0bb Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 15 Sep 2026 15:30:35 -0700 Subject: [PATCH 09/15] fix(settings): keep organization settings inline without Sim Search (#7862) * fix(settings): decouple organization access from Sim Search * fix(settings): keep organization settings inline without Sim Search --- .../settings/[section]/settings.test.tsx | 65 +++++++++++++++++++ .../settings/[section]/settings.tsx | 4 +- .../team-management/team-management.test.tsx | 29 +++++++++ .../team-management/team-management.tsx | 11 ++-- .../settings-sidebar.test.tsx | 31 ++++++++- .../settings-sidebar/settings-sidebar.tsx | 11 ++-- apps/sim/components/settings/navigation.ts | 23 ++----- .../workspace-section-access.test.ts | 28 ++++++++ .../application/workspace-section-access.ts | 5 +- 9 files changed, 170 insertions(+), 37 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.test.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.test.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.test.tsx new file mode 100644 index 00000000000..e02067414a5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.test.tsx @@ -0,0 +1,65 @@ +/** + * @vitest-environment jsdom + */ +import { act, type ComponentType, lazy, type ReactNode, Suspense } from 'react' +import { createRoot } from 'react-dom/client' +import { expect, it, vi } from 'vitest' + +vi.mock('next/dynamic', () => ({ + default: (load: () => Promise) => lazy(async () => ({ default: await load() })), +})) +vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) +vi.mock('@/lib/posthog/client', () => ({ captureEvent: vi.fn() })) +vi.mock('@/lib/auth/auth-client', () => ({ + useSession: () => ({ data: { user: { id: 'viewer-1', role: 'user' } }, isPending: false }), +})) +vi.mock('@/lib/core/config/deployment-shape', () => ({ + useDeploymentShape: () => ({ billingEnabled: false }), +})) +vi.mock('@/app/workspace/[workspaceId]/providers/workspace-host-provider', () => ({ + useWorkspaceHostContext: () => ({ + hostOrganizationId: 'organization-1', + workspace: { id: 'workspace-1' }, + }), +})) +vi.mock('@/app/workspace/[workspaceId]/settings/components/general/general', () => ({ + General: () =>
General settings
, +})) +vi.mock( + '@/app/workspace/[workspaceId]/settings/components/team-management/team-management', + () => ({ + TeamManagement: ({ organizationId }: { organizationId: string }) => ( +
Members of {organizationId}
+ ), + }) +) +vi.mock('@/app/workspace/[workspaceId]/settings/components/settings-panel', () => ({ + SettingsSectionProvider: ({ children }: { children: ReactNode }) => children, +})) +vi.mock('@/app/workspace/[workspaceId]/settings/navigation', () => ({ + getSettingsSectionMeta: () => null, +})) + +import { SettingsPage } from '@/app/workspace/[workspaceId]/settings/[section]/settings' + +it('renders the inline member roster with billing disabled, while billing stays unavailable', async () => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + const container = document.createElement('div') + const root = createRoot(container) + try { + await act(async () => { + root.render( + + + + ) + }) + expect(container).toHaveTextContent('Members of organization-1') + expect(container).not.toHaveTextContent('General settings') + + await act(async () => root.render()) + expect(container).toHaveTextContent('General settings') + } finally { + act(() => root.unmount()) + } +}) diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx index ae626e7a9d9..6fc7c8495aa 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx @@ -138,7 +138,7 @@ export function SettingsPage({ section }: SettingsPageProps) { const normalizedSection: SettingsSection = (section as string) === 'subscription' ? 'billing' : section const effectiveSection = - !billingEnabled && (normalizedSection === 'billing' || normalizedSection === 'organization') + !billingEnabled && normalizedSection === 'billing' ? 'general' : normalizedSection === 'admin' && !sessionLoading && !isAdminRole ? 'general' @@ -192,7 +192,7 @@ export function SettingsPage({ section }: SettingsPageProps) { /> )} {effectiveSection === 'teammates' && } - {billingEnabled && effectiveSection === 'organization' && organizationId && ( + {effectiveSection === 'organization' && organizationId && ( ({ + deployment: { billingEnabled: true }, mockIsAdminOrOwner: vi.fn(), mockUseOrganization: vi.fn(), mockUseOrganizationBilling: vi.fn(), @@ -22,6 +24,10 @@ vi.mock('@/lib/auth/auth-client', () => ({ useSession: () => ({ data: { user: { id: 'viewer-1', email: 'viewer' } } }), })) +vi.mock('@/lib/core/config/deployment-shape', () => ({ + useDeploymentShape: () => deployment, +})) + vi.mock('@/lib/billing/client/utils', () => ({ getSubscriptionAccessState: () => ({ hasUsableTeamAccess: false, @@ -121,6 +127,7 @@ let container: HTMLDivElement let root: Root beforeEach(() => { + deployment.billingEnabled = true ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true container = document.createElement('div') document.body.appendChild(container) @@ -145,6 +152,28 @@ afterEach(() => { }) describe('TeamManagement organization errors', () => { + it('renders members without fetching or displaying billing when billing is disabled', () => { + deployment.billingEnabled = false + mockIsAdminOrOwner.mockReturnValue(true) + mockUseOrganization.mockReturnValue({ data: { id: 'org-1' }, error: null, isLoading: false }) + mockUseOrganizationBilling.mockReturnValue({ + data: undefined, + error: new Error('Billing request failed'), + isLoading: false, + }) + + act(() => + root.render( + + ) + ) + + expect(mockUseOrganizationBilling).toHaveBeenCalledWith('org-1', { enabled: false }) + expect(container).toHaveTextContent('organization-member-lists') + expect(container).not.toHaveTextContent('Billing request failed') + expect(container).not.toHaveTextContent('team-seats-overview') + }) + it.each([ { admin: true, canInvite: false, shown: true, disabled: true }, { admin: true, canInvite: true, shown: true, disabled: false }, diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.tsx index 6bab7fe2a95..2ff61ebb00e 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.tsx @@ -6,6 +6,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useSession } from '@/lib/auth/auth-client' import { getSubscriptionAccessState } from '@/lib/billing/client/utils' +import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import { getBaseUrl } from '@/lib/core/utils/urls' import { APP_ENTRY_PATH } from '@/lib/navigation/paths' import { generateSlug, isAdminOrOwner, type Member } from '@/lib/workspaces/organization' @@ -53,6 +54,7 @@ export function TeamManagement({ canInviteMembers, }: TeamManagementProps) { const { data: session } = useSession() + const { billingEnabled } = useDeploymentShape() const { isInvitationsDisabled } = usePermissionConfig() const invitationsDisabled = canInviteMembers === undefined ? isInvitationsDisabled : !canInviteMembers @@ -71,7 +73,7 @@ export function TeamManagement({ * organization page derives its plan from organization billing, so avoid that unrelated read * on the normal first paint. */ - const shouldLoadRecoverySubscription = !isLoading && !orgError && !organization + const shouldLoadRecoverySubscription = billingEnabled && !isLoading && !orgError && !organization const { data: userSubscriptionData, isPending: isRecoverySubscriptionPending } = useSubscriptionData({ enabled: shouldLoadRecoverySubscription, @@ -89,7 +91,7 @@ export function TeamManagement({ isFetchedAfterMount: isOrganizationBillingFetchedAfterMount, isFetching: isOrganizationBillingFetching, refetch: refetchOrganizationBilling, - } = useOrganizationBilling(organizationId, { enabled: adminOrOwner }) + } = useOrganizationBilling(organizationId, { enabled: billingEnabled && adminOrOwner }) const { data: roster, @@ -148,7 +150,7 @@ export function TeamManagement({ * `client.subscription.list`, which does not reliably surface org-scoped * subscriptions. */ - const orgBilling = organizationBillingData?.data ?? null + const orgBilling = billingEnabled ? (organizationBillingData?.data ?? null) : null const orgSubscription = orgBilling ? { id: orgBilling.organizationId, @@ -367,7 +369,8 @@ export function TeamManagement({ : [] } > - {adminOrOwner && + {billingEnabled && + adminOrOwner && ((organizationBillingError || (isOrganizationBillingFetching && isOrganizationBillingFetchedAfterMount)) && organizationBillingData === undefined ? ( diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.test.tsx index 118bae39c16..4e5bb7e5e31 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.test.tsx @@ -33,7 +33,7 @@ vi.mock('@/lib/billing/client', () => ({ getSubscriptionAccessState(...args), })) vi.mock('@/lib/core/config/deployment-shape', () => ({ - useDeploymentShape: () => deployment, + useDeploymentShape: () => hostContext.deployment, getDeploymentShape: () => deployment, })) vi.mock('@/lib/desktop', () => ({ @@ -199,6 +199,7 @@ describe('workspace SettingsSidebar organization rollout', () => { renderSidebar() expect(workspaceLink('connected-accounts')).toBeNull() + expect(workspaceLink('organization')).toBeNull() }) it.each([false, undefined])( @@ -264,12 +265,38 @@ describe('workspace SettingsSidebar organization rollout', () => { renderSidebar() expect(workspaceLink('billing')).toHaveTextContent('Subscription') - for (const section of ['organization', 'usage', 'sso']) { + expect(workspaceLink('organization')).toHaveTextContent('Members') + for (const section of ['usage', 'sso']) { expect(workspaceLink(section)).toBeNull() } expectWorkspaceLinks() }) + it.each(['admin', 'member', 'external'] as const)( + 'shows permitted inline settings for a self-hosted %s with Search and billing disabled', + (role) => { + hostContext = makeHostContext(role, false) + hostContext.deployment = { ...deployment, hosted: false, billingEnabled: false } + renderSidebar() + + expect(container.querySelector('a[href^="/o/"]')).toBeNull() + expect(workspaceLink('billing')).toBeNull() + if (role === 'external') { + expect(workspaceLink('organization')).toBeNull() + } else { + expect(workspaceLink('organization')).toHaveTextContent('Members') + } + for (const section of ['connected-accounts', 'access-control', 'usage', 'sso', 'security']) { + if (role === 'admin') { + expect(workspaceLink(section)).not.toBeNull() + } else { + expect(workspaceLink(section)).toBeNull() + } + } + expectWorkspaceLinks() + } + ) + it.each([false, true])( 'keeps external workspace admins out of organization settings when rollout is %s', (enabled) => { diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.tsx index 642a8ea2679..4dc93d6ca6a 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.tsx @@ -136,7 +136,6 @@ export function SettingsSidebar({ : null const subscriptionAccess = getSubscriptionAccessState(hostContext.ownerBilling) const inboxEntitled = inboxConfig?.entitled ?? false - const hasTeamPlan = subscriptionAccess.hasUsableTeamAccess const hasEnterprisePlan = subscriptionAccess.hasUsableEnterpriseAccess const isEnterprisePlan = subscriptionAccess.isEnterprise @@ -164,6 +163,11 @@ export function SettingsSidebar({ ) { return false } + if (item.id === 'organization') { + return Boolean( + hostContext.hostOrganizationId && hostContext.viewer.isHostOrganizationMember + ) + } if (item.requiresSelfHosted && hosted) { return false } @@ -228,10 +232,6 @@ export function SettingsSidebar({ const orgAdminSatisfied = isOrgAdminOrOwner || item.allowNonOrgAdmin - if (item.requiresTeam && (!hasTeamPlan || !orgAdminSatisfied)) { - return false - } - if ( item.requiresEnterprise && (!hasEnterprisePlan || !orgAdminSatisfied) && @@ -264,7 +264,6 @@ export function SettingsSidebar({ deployment, hosted, billingEnabled, - hasTeamPlan, hasEnterprisePlan, isEnterprisePlan, subscriptionAccess.hasUsableMaxAccess, diff --git a/apps/sim/components/settings/navigation.ts b/apps/sim/components/settings/navigation.ts index 7721f1c7841..d0ef0738fbc 100644 --- a/apps/sim/components/settings/navigation.ts +++ b/apps/sim/components/settings/navigation.ts @@ -138,7 +138,6 @@ export interface UnifiedSettingsNavigationItem { section: UnifiedNavigationSection order: number hideWhenBillingDisabled?: boolean - requiresTeam?: boolean requiresEnterprise?: boolean requiresMax?: boolean requiresHosted?: boolean @@ -473,16 +472,6 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] description: 'Members and workspace access in your organization.', group: 'organization', order: 0, - hideWhenBillingDisabled: true, - requiresHosted: true, - requiresTeam: true, - /** - * A plain member sees the roster read-only — `resolveOrganizationSectionAccess` - * grants them `'view'` on this one section, and `TeamManagement` renders - * without management controls. Every other organization section stays - * admin-only. - */ - allowNonOrgAdmin: true, organizationSection: 'members', }, }, @@ -495,14 +484,10 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] group: 'organization', order: 1, /** - * Deliberately no `hideWhenBillingDisabled`, unlike Members above. - * - * The sidebar applies that filter *before* it consults `selfHostedOverride`, - * so pairing the two hid this section from exactly the deployment the - * override exists to serve: self-hosted, billing off, `USAGE_MONITORING_ENABLED` - * on. Members can carry the flag because it has no override to reach. Here the - * two gates below already answer both cases — hosted needs the plan, and - * self-hosted needs the flag. + * Do not add `hideWhenBillingDisabled`: the sidebar applies it before + * `selfHostedOverride`, which would hide usage monitoring on self-hosted + * deployments with billing disabled. Hosted deployments require the plan; + * self-hosted deployments require the feature flag. */ requiresHosted: true, requiresEnterprise: true, diff --git a/apps/sim/lib/settings/application/workspace-section-access.test.ts b/apps/sim/lib/settings/application/workspace-section-access.test.ts index ca9793b6b91..a8ca619c801 100644 --- a/apps/sim/lib/settings/application/workspace-section-access.test.ts +++ b/apps/sim/lib/settings/application/workspace-section-access.test.ts @@ -118,6 +118,7 @@ function authorize(section: Parameters describe('authorizeWorkspaceSettingsSection', () => { beforeEach(() => { vi.clearAllMocks() + mocks.deploymentShape.billingEnabled = true mocks.checkWorkspaceAccess.mockResolvedValue(PERSONAL_ACCESS) mocks.isCustomBlocksEligibleForOrganization.mockResolvedValue(true) mocks.isForkingAvailableForWorkspace.mockResolvedValue(true) @@ -249,6 +250,33 @@ describe('authorizeWorkspaceSettingsSection', () => { expect(mocks.canOpenOrganizationSettingsSection).not.toHaveBeenCalled() }) + it('allows the member roster with billing disabled while keeping billing unavailable', async () => { + mocks.deploymentShape.billingEnabled = false + mocks.checkWorkspaceAccess.mockResolvedValue(ORGANIZATION_ACCESS) + + await expect(authorize('organization')).resolves.toEqual({ allowed: true }) + expect(mocks.canOpenOrganizationSettingsSection).toHaveBeenCalledWith( + 'organization-1', + 'viewer-1', + 'members' + ) + await expect(authorize('billing')).resolves.toEqual({ + allowed: false, + disposition: 'redirect-general', + }) + }) + + it('requires current organization membership for the roster with billing disabled', async () => { + mocks.deploymentShape.billingEnabled = false + mocks.checkWorkspaceAccess.mockResolvedValue(ORGANIZATION_ACCESS) + mocks.canOpenOrganizationSettingsSection.mockResolvedValue(false) + + await expect(authorize('organization')).resolves.toEqual({ + allowed: false, + disposition: 'redirect-general', + }) + }) + it.each([ { groups: true, search: false, allowed: true }, { groups: false, search: false, allowed: false }, diff --git a/apps/sim/lib/settings/application/workspace-section-access.ts b/apps/sim/lib/settings/application/workspace-section-access.ts index 90e00290cf5..daa8201117b 100644 --- a/apps/sim/lib/settings/application/workspace-section-access.ts +++ b/apps/sim/lib/settings/application/workspace-section-access.ts @@ -77,10 +77,7 @@ async function canOpenOrganizationSection( const organizationSection = UNIFIED_TO_ORGANIZATION_SECTION[input.section] if (!organizationSection) return true const deployment = getDeploymentShape() - if ( - !deployment.billingEnabled && - (input.section === 'billing' || input.section === 'organization') - ) { + if (!deployment.billingEnabled && input.section === 'billing') { return false } if (!workspace.organizationId) { From aeec08af743a9b760dde475ee19c63ddad9971d3 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 15 Sep 2026 16:23:22 -0700 Subject: [PATCH 10/15] fix(activity): simplify outcome summaries and show three actions (#7866) --- .../agent-group/activity-stream.test.tsx | 44 +++++++++++++---- .../agent-group/tool-activity-group.test.ts | 48 ++++++++++++++----- .../agent-group/tool-activity-group.tsx | 20 +++----- 3 files changed, 78 insertions(+), 34 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/activity-stream.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/activity-stream.test.tsx index df297e5bdcb..dedae46db01 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/activity-stream.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/activity-stream.test.tsx @@ -157,24 +157,50 @@ describe.each(['mothership', 'workflow', 'browser', 'deploy'])('%s activity', (a render([tool('first'), tool('second')]) advance(100) render([tool('first'), tool('second', status)]) - const outcome = + const label = status === 'error' || status === 'rejected' - ? 'failed' - : status === 'skipped' - ? 'skipped' - : 'stopped' - expect(header()?.textContent).toBe(`Reading first · 1 ${outcome}`) + ? 'Reading first' + : `Reading first · 1 ${status === 'skipped' ? 'skipped' : 'stopped'}` + expect(header()?.textContent).toBe(label) expect(container.querySelector('[class*="shimmer"]')).not.toBeNull() advance(1000) - expect(header()?.textContent).toBe(`Reading first · 1 ${outcome}`) + expect(header()?.textContent).toBe(label) } ) - it('surfaces an earlier parallel failure while the latest call keeps working', () => { + it('keeps earlier parallel failures in history without a summary badge', () => { render([tool('first'), tool('second')]) advance(100) render([tool('first', 'error'), tool('second')]) - expect(header()?.textContent).toBe('Reading second · 1 failed') + expect(header()?.textContent).toBe('Reading second') + const trigger = container.querySelector('[role="button"]')! + act(() => trigger.click()) + expect(container.querySelector('[data-state="open"]')?.textContent).toBe( + 'Failed reading firstReading second' + ) + render([tool('first', 'error'), tool('second', 'success')], false) + expect(header()?.textContent).toBe('Read files') + expect(container.querySelector('[data-state="open"]')?.textContent).toBe( + 'Failed reading firstRead second' + ) + }) + + it('shows three distinct actions and keeps the complete history available', () => { + render( + [ + tool('first', 'success'), + { ...tool('second', 'success'), toolName: 'grep' }, + { ...tool('third', 'success'), toolName: 'terminal_run' }, + { ...tool('fourth', 'success'), toolName: 'run_workflow' }, + ], + false + ) + expect(header()?.textContent).toBe('Read files, searched files, ran commands +1 more') + const trigger = container.querySelector('[role="button"]')! + act(() => trigger.click()) + expect(container.querySelector('[data-state="open"]')?.textContent).toBe( + 'Read firstRead secondRead thirdRead fourth' + ) }) it('keeps narration from prematurely completing an open lane', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/tool-activity-group.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/tool-activity-group.test.ts index e1c23b42d1b..53f44524306 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/tool-activity-group.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/tool-activity-group.test.ts @@ -12,8 +12,14 @@ function tool(toolName: string, status: ToolCallStatus = 'success'): ToolCallDat describe('getToolActivitySummary', () => { it('caps distinct actions in order and counts the remaining categories, not repeated calls', () => { expect( - getToolActivitySummary([tool('read'), tool('terminal_run'), tool('read'), tool('grep')]) - ).toBe('Read files, ran commands +1 more') + getToolActivitySummary([ + tool('read'), + tool('terminal_run'), + tool('read'), + tool('grep'), + tool('browser_navigate'), + ]) + ).toBe('Read files, ran commands, searched files +1 more') }) it('summarizes browser navigation and interactions without repeating actions', () => { @@ -24,13 +30,16 @@ describe('getToolActivitySummary', () => { tool('browser_type'), tool('browser_navigate'), ]) - ).toBe('Navigated, read pages +1 more') + ).toBe('Navigated, read pages, entered text') }) it.each([ [['browser_navigate', 'browser_read_text'], 'Navigated, read pages'], [['browser_read_text', 'browser_navigate'], 'Read, navigated pages'], - [['browser_navigate', 'browser_read_text', 'browser_scroll'], 'Navigated, read pages +1 more'], + [ + ['browser_navigate', 'browser_read_text', 'browser_scroll'], + 'Navigated, read, scrolled pages', + ], [['browser_navigate', 'browser_type'], 'Navigated pages, entered text'], [['browser_navigate', 'browser_navigate'], 'Navigated pages'], [['read', 'browser_read_text'], 'Read files, read pages'], @@ -46,7 +55,7 @@ describe('getToolActivitySummary', () => { tool('terminal_run', 'cancelled'), tool('browser_type', 'rejected'), ]) - ).toBe('Read files · 2 failed · 1 stopped') + ).toBe('Read files · 1 stopped') }) it('does not invent actions when all calls failed or were stopped', () => { @@ -55,7 +64,22 @@ describe('getToolActivitySummary', () => { tool('apply_file_edit', 'error'), tool('terminal_run', 'interrupted'), ]) - ).toBe('Tool activity · 1 failed · 1 stopped') + ).toBe('Tool activity · 1 stopped') + }) + + it('uses a neutral summary when every call failed', () => { + expect( + getToolActivitySummary([tool('run_workflow', 'error'), tool('terminal', 'rejected')]) + ).toBe('Tool activity') + }) + + it('does not infer tool failures from workflow results', () => { + expect( + getToolActivitySummary([ + tool('read'), + { ...tool('run_workflow'), result: { success: false, error: 'Workflow run failed' } }, + ]) + ).toBe('Read files, ran workflows') }) it('keeps an individual tool’s descriptive title', () => { @@ -91,10 +115,10 @@ describe('getToolActivitySummary', () => { tool('deploy_as_api'), tool('table_rows'), ]) - ).toBe('Navigated pages, filled forms +5 more') + ).toBe('Navigated pages, filled forms, entered text +4 more') }) - it('keeps failure and interruption counts visible when action categories are capped', () => { + it('keeps interruption counts without failure badges when action categories are capped', () => { expect( getToolActivitySummary([ tool('read'), @@ -105,14 +129,14 @@ describe('getToolActivitySummary', () => { tool('wait', 'interrupted'), tool('browser_type', 'skipped'), ]) - ).toBe('Read files, searched files +2 more · 1 failed · 1 stopped · 1 skipped') + ).toBe('Read files, searched files, used the terminal +1 more · 1 stopped · 1 skipped') }) - it('uses the same outcome wording for rejected individual and grouped calls', () => { + it('keeps individual failures explicit without adding aggregate failure badges', () => { const rejected = { ...tool('terminal', 'rejected'), displayTitle: 'Running checks' } expect(getToolActivitySummary([rejected])).toBe('Failed running checks') expect(getToolActivitySummary([rejected, tool('read', 'skipped')])).toBe( - 'Tool activity · 1 failed · 1 skipped' + 'Tool activity · 1 skipped' ) }) @@ -124,7 +148,7 @@ describe('getToolActivitySummary', () => { { ...tool('deploy_as_mcp'), params: { action: 'undeploy' } }, tool('read'), ]) - ).toBe('Deployed workflows, undeployed workflows +1 more') + ).toBe('Deployed workflows, undeployed workflows, read files') }) it('describes terminal runs from their operation', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/tool-activity-group.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/tool-activity-group.tsx index 11b95a3833e..5535c0735c4 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/tool-activity-group.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/tool-activity-group.tsx @@ -10,7 +10,7 @@ import { getActivityAttentionKey } from '@/app/workspace/[workspaceId]/home/comp import { getToolIcon } from '@/app/workspace/[workspaceId]/home/components/message-content/utils' import { type ToolCallData, ToolCallStatus } from '@/app/workspace/[workspaceId]/home/types' -const MAX_SUMMARY_ACTIONS = 2 +const MAX_SUMMARY_ACTIONS = 3 /** Summarize completed actions without describing failed or skipped work as successful. */ export function getToolActivitySummary(tools: ToolCallData[]): string { @@ -31,35 +31,29 @@ export function getToolActivitySummary(tools: ToolCallData[]): string { const summaryLabel = summary ? summary[0].toUpperCase() + summary.slice(1) : 'Tool activity' return [ additionalActions > 0 ? `${summaryLabel} +${additionalActions} more` : summaryLabel, - ...getToolActivityOutcomes(tools), + ...getToolActivityInterruptions(tools), ].join(' · ') } -function getToolActivityOutcomes(tools: ToolCallData[]): string[] { - let failed = 0 +function getToolActivityInterruptions(tools: ToolCallData[]): string[] { let stopped = 0 let skipped = 0 for (const tool of tools) { - if (tool.status === ToolCallStatus.error || tool.status === ToolCallStatus.rejected) failed++ - else if (tool.status === ToolCallStatus.cancelled || tool.status === ToolCallStatus.interrupted) + if (tool.status === ToolCallStatus.cancelled || tool.status === ToolCallStatus.interrupted) stopped++ else if (tool.status === ToolCallStatus.skipped) skipped++ } - return [ - ...(failed ? [`${failed} failed`] : []), - ...(stopped ? [`${stopped} stopped`] : []), - ...(skipped ? [`${skipped} skipped`] : []), - ] + return [...(stopped ? [`${stopped} stopped`] : []), ...(skipped ? [`${skipped} skipped`] : [])] } -/** Keep earlier parallel failures visible while the latest action continues. */ +/** Keep earlier interruptions visible while the latest action continues. */ export function getActiveToolActivityTitle( label: string, tool: ToolCallData, tools: ToolCallData[] ): string { return tool.status === ToolCallStatus.executing || tool.status === ToolCallStatus.success - ? [label, ...getToolActivityOutcomes(tools)].join(' · ') + ? [label, ...getToolActivityInterruptions(tools)].join(' · ') : label } From 5bca037afc343bcaade18cc71c804c9e04069049 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 15 Sep 2026 16:40:35 -0700 Subject: [PATCH 11/15] feat(sso): open Sim from an identity provider's app dashboard (#7865) * feat(sso): open Sim from an identity provider's app dashboard * improvement(sso): skip re-authentication when signed in and limit dashboard launch to OIDC providers * improvement(sso): redirect straight to the identity provider from the launch URL * fix(sso): keep a signed-in visitor going to the app when the launch address is rate limited * fix(sso): report a failed launch sign-in on the provider's sign-in link * chore(sso): normalize the caught error when a launch sign-in fails --- .../content/docs/platform/enterprise/sso.mdx | 10 +- .../sso/launch/[providerId]/route.test.ts | 136 ++++++++++++++++++ .../(auth)/sso/launch/[providerId]/route.ts | 89 ++++++++++++ .../sso/components/sso-provider-settings.tsx | 17 ++- .../ee/sso/components/sso-settings.test.tsx | 26 +++- .../lib/auth/sso/idp-initiated-login.test.ts | 47 ++++++ apps/sim/lib/auth/sso/idp-initiated-login.ts | 51 +++++++ 7 files changed, 372 insertions(+), 4 deletions(-) create mode 100644 apps/sim/app/(auth)/sso/launch/[providerId]/route.test.ts create mode 100644 apps/sim/app/(auth)/sso/launch/[providerId]/route.ts create mode 100644 apps/sim/lib/auth/sso/idp-initiated-login.test.ts create mode 100644 apps/sim/lib/auth/sso/idp-initiated-login.ts diff --git a/apps/docs/content/docs/platform/enterprise/sso.mdx b/apps/docs/content/docs/platform/enterprise/sso.mdx index 76fc3dbb1d1..5ed4d7a7fc1 100644 --- a/apps/docs/content/docs/platform/enterprise/sso.mdx +++ b/apps/docs/content/docs/platform/enterprise/sso.mdx @@ -81,7 +81,7 @@ An organization can run several identity providers at once: Okta for `eng.acme.c ### 4. Copy the callback URL -Copy **Callback URL** for OIDC or **ACS URL (Reply URL)** for SAML. This is the endpoint that receives your identity provider's authentication response. Register it in your IdP before saving. If you set a SAML **Callback URL override** under Advanced options, the copyable ACS URL uses that override. +Copy **Callback URL** for OIDC or **ACS URL (Reply URL)** for SAML. This is the endpoint that receives your identity provider's authentication response. Register it in your IdP before saving. On Sim Cloud, `` is `www.sim.ai`; self-hosted deployments use their own domain. If you set a SAML **Callback URL override** under Advanced options, the copyable ACS URL uses that override. **OIDC providers** (Okta, Microsoft Entra ID, Google Workspace, Auth0): ``` @@ -140,7 +140,11 @@ The first time someone signs in through the new provider, Sim links it to their ``` 4. Under **Assignments**, grant access to the relevant users or groups 5. Copy the **Client ID** and **Client Secret** from the app's **General** tab -6. Copy your Okta organization domain from the account menu in the Admin Console, e.g. `dev-1234567.okta.com`. The Admin Console's `-admin` hostname is a different URL. See [Find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/). +6. To open Sim from the Okta dashboard, set **Login initiated by** to **Either Okta or App**, show the app icon to users, choose **Redirect to app to initiate login (OIDC Compliant)**, and set **Initiate login URI** to the provider's **Initiate login URL** from Sim: + ``` + https:///sso/launch/okta + ``` +7. Copy your Okta organization domain from the account menu in the Admin Console, e.g. `dev-1234567.okta.com`. The Admin Console's `-admin` hostname is a different URL. See [Find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/). **In Sim:** @@ -305,6 +309,8 @@ Once SSO is configured, users with your domain (`company.com`) can sign in throu 5. If **First sign-in** is **Automatic**, Sim adds them to the organization as a Member, growing a Team seat count or validating available fixed-seat capacity 6. They land in an accessible workspace, or see a clear no-access state until an admin grants workspace access +People can also open Sim straight from an OIDC identity provider's app dashboard, such as the Okta tile. Open **Sign-in**, select the provider, and copy its **Initiate login URL** from **Identity provider**. Set it as the app's initiate login URI in your identity provider. Sim starts sign-in through that provider without asking for an email, and only when the provider's domain is verified and the request comes from its own issuer. People who are already signed in go straight to Sim. + With **Automatic** provisioning, no invitation is required for organization membership. The join follows the organization's seat policy and does not infer a role from IdP claims: every newly provisioned user starts as a Member. Team subscriptions grow their billed seat count with membership; fixed-seat plans reject the join when capacity is full. With **Invite only**, SSO proves identity but does not create new membership or workspace access; new access must be granted separately, while existing organization membership and workspace access remain available. diff --git a/apps/sim/app/(auth)/sso/launch/[providerId]/route.test.ts b/apps/sim/app/(auth)/sso/launch/[providerId]/route.test.ts new file mode 100644 index 00000000000..a17ffeae63b --- /dev/null +++ b/apps/sim/app/(auth)/sso/launch/[providerId]/route.test.ts @@ -0,0 +1,136 @@ +/** + * @vitest-environment node + */ +import { createMockRequest, setEnvFlags } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetSession, mockSignInSSO, mockIsAllowed, mockEnforceIpRateLimit } = vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockSignInSSO: vi.fn(), + mockIsAllowed: vi.fn(), + mockEnforceIpRateLimit: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ + getSession: mockGetSession, + auth: { api: { signInSSO: mockSignInSSO } }, +})) +vi.mock('@/lib/auth/sso/idp-initiated-login', () => ({ isIdpInitiatedLoginAllowed: mockIsAllowed })) +vi.mock('@/lib/core/rate-limiter', () => ({ enforceIpRateLimit: mockEnforceIpRateLimit })) + +import { GET } from '@/app/(auth)/sso/launch/[providerId]/route' + +const context = { params: Promise.resolve({ providerId: 'acme-okta' }) } +const ISSUER = 'https://acme.okta.test' +const SIGN_IN_LINK = 'https://test.sim.ai/sso?provider=acme-okta' + +function open(search = `?iss=${encodeURIComponent(ISSUER)}`) { + return GET( + createMockRequest('GET', undefined, {}, `https://test.sim.ai/sso/launch/acme-okta${search}`), + context + ) +} + +/** Better Auth answers with the authorization URL and the signed `state` cookie for it. */ +function authorizationResponse() { + return new Response(JSON.stringify({ url: 'https://acme.okta.test/oauth2/v1/authorize?x=1' }), { + status: 200, + headers: { 'content-type': 'application/json', 'set-cookie': 'sso_state=abc; Path=/' }, + }) +} + +describe('GET /sso/launch/[providerId]', () => { + beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isSsoEnabled: true }) + mockGetSession.mockResolvedValue(null) + mockIsAllowed.mockResolvedValue(true) + mockEnforceIpRateLimit.mockResolvedValue(null) + mockSignInSSO.mockResolvedValue(authorizationResponse()) + }) + + it("redirects to the identity provider and carries Better Auth's state cookie", async () => { + const response = await open() + + expect(response.status).toBe(307) + expect(response.headers.get('location')).toBe('https://acme.okta.test/oauth2/v1/authorize?x=1') + expect(response.headers.get('set-cookie')).toContain('sso_state=abc') + expect(mockIsAllowed).toHaveBeenCalledWith('acme-okta', ISSUER) + const [{ body }] = mockSignInSSO.mock.calls[0] + expect(body.providerId).toBe('acme-okta') + expect(body).not.toHaveProperty('email') + /** The plugin appends `?error=…`, which must not corrupt the provider on the way back. */ + const retry = new URL(`${body.errorCallbackURL}?error=invalid_provider`) + expect(retry.pathname).toBe('/sso') + expect(retry.searchParams.get('provider')).toBe('acme-okta') + }) + + it('sends someone already signed in to the app without signing in again', async () => { + mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) + + const response = await open() + + expect(response.headers.get('location')).toBe('https://test.sim.ai/home') + expect(mockIsAllowed).not.toHaveBeenCalled() + expect(mockSignInSSO).not.toHaveBeenCalled() + }) + + it('keeps sending a signed-in visitor to the app when the address is rate limited', async () => { + mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) + mockEnforceIpRateLimit.mockResolvedValue(new Response(null, { status: 429 })) + + const response = await open() + + expect(response.headers.get('location')).toBe('https://test.sim.ai/home') + expect(mockEnforceIpRateLimit).not.toHaveBeenCalled() + }) + + it.each([ + ['no issuer', '', () => undefined], + [ + 'an issuer the provider does not use', + `?iss=${encodeURIComponent('https://other.test')}`, + () => mockIsAllowed.mockResolvedValue(false), + ], + ])("sends a visitor with %s to the provider's sign-in link", async (_label, search, arrange) => { + arrange() + + const response = await open(search) + + expect(response.headers.get('location')).toBe(SIGN_IN_LINK) + expect(mockSignInSSO).not.toHaveBeenCalled() + }) + + it.each([ + ['refuses', () => mockSignInSSO.mockResolvedValue(new Response('{}', { status: 400 }))], + ['throws', () => mockSignInSSO.mockRejectedValue(new Error('network'))], + ])("reports the failure on the provider's sign-in link when sign-in %s", async (_l, arrange) => { + arrange() + + const response = await open() + + const failure = new URL(response.headers.get('location') ?? '') + expect(failure.pathname).toBe('/sso') + expect(failure.searchParams.get('error')).toBe('sso_failed') + expect(failure.searchParams.get('provider')).toBe('acme-okta') + }) + + it('sends a rate-limited visitor to the sign-in link before any lookup', async () => { + mockEnforceIpRateLimit.mockResolvedValue(new Response(null, { status: 429 })) + + const response = await open() + + expect(response.headers.get('location')).toBe(SIGN_IN_LINK) + expect(mockIsAllowed).not.toHaveBeenCalled() + expect(mockSignInSSO).not.toHaveBeenCalled() + }) + + it('leaves SSO off when the deployment has not enabled it', async () => { + setEnvFlags({ isSsoEnabled: false }) + + const response = await open() + + expect(response.headers.get('location')).toBe('https://test.sim.ai/login') + expect(mockEnforceIpRateLimit).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/(auth)/sso/launch/[providerId]/route.ts b/apps/sim/app/(auth)/sso/launch/[providerId]/route.ts new file mode 100644 index 00000000000..86afda73121 --- /dev/null +++ b/apps/sim/app/(auth)/sso/launch/[providerId]/route.ts @@ -0,0 +1,89 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { auth, getSession } from '@/lib/auth' +import { isIdpInitiatedLoginAllowed } from '@/lib/auth/sso/idp-initiated-login' +import { isSsoEnabled } from '@/lib/core/config/env-flags' +import { enforceIpRateLimit } from '@/lib/core/rate-limiter' +import { getBaseUrl } from '@/lib/core/utils/urls' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { DEFAULT_POST_AUTH_ROUTE } from '@/app/(auth)/auth-redirect' + +const logger = createLogger('SSOLaunchRoute') + +type RouteContext = { params: Promise<{ providerId: string }> } + +/** + * The initiate login URL an identity provider's app dashboard opens (OpenID Connect third-party + * initiated login). The dashboard adds its issuer as `iss`, so the URL carries no query of its own. + * + * Sign-in starts here rather than on the sign-in page: the visitor arrives to be sent onward, and a + * redirect spares them a page load and a hydration wait first. Someone already signed in goes + * straight to the app, so a link cannot replace their session. Anything else — an unknown issuer, a + * provider this deployment does not serve, a refused sign-in — falls back to the provider's ordinary + * sign-in link, which asks for an email. + */ +export const GET = withRouteHandler(async (request: NextRequest, context: RouteContext) => { + const { providerId } = await context.params + const signInLink = new URL( + `/sso?provider=${encodeURIComponent(providerId)}`, + getBaseUrl() + ).toString() + if (!isSsoEnabled) return NextResponse.redirect(new URL('/login', getBaseUrl()).toString()) + + const session = await getSession() + if (session?.user) { + return NextResponse.redirect(new URL(DEFAULT_POST_AUTH_ROUTE, getBaseUrl()).toString()) + } + + /** Admitted per address, after the session, so a busy shared address never strands a signed-in visitor. */ + const rateLimited = await enforceIpRateLimit('sso-launch', request, { + maxTokens: 30, + refillRate: 30, + refillIntervalMs: 60_000, + }) + if (rateLimited) return NextResponse.redirect(signInLink) + + const issuer = request.nextUrl.searchParams.get('iss') + if (!issuer || !(await isIdpInitiatedLoginAllowed(providerId, issuer))) { + return NextResponse.redirect(signInLink) + } + + /** + * A failed sign-in returns to the provider's sign-in link with the error. `callbackUrl` comes + * last because the SSO plugin appends its own error with a raw `?`, which runs into whichever + * parameter is last — there it is harmless, on `provider` it would corrupt the retry. + */ + const errorCallbackURL = new URL( + `/sso?error=sso_failed&provider=${encodeURIComponent(providerId)}&callbackUrl=${encodeURIComponent(DEFAULT_POST_AUTH_ROUTE)}`, + getBaseUrl() + ).toString() + /** A sign-in that never starts is a failure, so it carries the error rather than a blank form. */ + let signIn: Response + try { + signIn = await auth.api.signInSSO({ + body: { providerId, callbackURL: DEFAULT_POST_AUTH_ROUTE, errorCallbackURL }, + headers: request.headers, + asResponse: true, + }) + } catch (error) { + logger.error('SSO sign-in could not be started', { providerId, error: toError(error) }) + return NextResponse.redirect(errorCallbackURL) + } + const payload = (await signIn.json().catch(() => null)) as { url?: string } | null + if (!signIn.ok || !payload?.url) { + logger.error('SSO sign-in did not return an authorization URL', { + providerId, + status: signIn.status, + }) + return NextResponse.redirect(errorCallbackURL) + } + + const response = NextResponse.redirect(payload.url) + /** Better Auth's signed `state` cookie has to reach the browser before the identity provider does. */ + const signInHeaders = signIn.headers as Headers & { getSetCookie?: () => string[] } + for (const cookie of signInHeaders.getSetCookie?.() ?? []) { + response.headers.append('set-cookie', cookie) + } + return response +}) diff --git a/apps/sim/ee/sso/components/sso-provider-settings.tsx b/apps/sim/ee/sso/components/sso-provider-settings.tsx index e01275de70b..2bfd5af853e 100644 --- a/apps/sim/ee/sso/components/sso-provider-settings.tsx +++ b/apps/sim/ee/sso/components/sso-provider-settings.tsx @@ -542,6 +542,8 @@ export function SsoProviderSettings({ ? [{ text: 'Delete', variant: 'destructive', onSelect: onDelete } satisfies SettingsAction] : []), ] + const isOidcProvider = (existingProvider.providerType ?? 'oidc') === 'oidc' + const encodedProviderId = encodeURIComponent(existingProvider.providerId ?? '') const providerCallbackUrl = (existingProvider.providerType === 'saml' && readProviderConfigString(existingProvider.samlConfig, 'callbackUrl')) || @@ -593,11 +595,24 @@ export function SsoProviderSettings({ )} + {isOidcProvider && ( + + +

+ Configure this in your identity provider to open Sim from its app dashboard +

+ + )} + {onMakePrimary && (

diff --git a/apps/sim/ee/sso/components/sso-settings.test.tsx b/apps/sim/ee/sso/components/sso-settings.test.tsx index f9f3fae3009..bbd66655693 100644 --- a/apps/sim/ee/sso/components/sso-settings.test.tsx +++ b/apps/sim/ee/sso/components/sso-settings.test.tsx @@ -794,7 +794,11 @@ describe('SSO provider list', () => { describe('SSO primary provider', () => { /** An organization moving one domain's sign-in from one identity provider to another. */ - function renderMigration(searchParams = '', okta: Record = {}) { + function renderMigration( + searchParams = '', + okta: Record = {}, + entra: Record = {} + ) { mockUseSSOProviders.mockReturnValue({ data: { providers: [ @@ -804,6 +808,7 @@ describe('SSO primary provider', () => { providerId: 'acme-entra', domainVerified: true, isPrimary: true, + ...entra, }, { ...provider('org-a'), @@ -848,6 +853,25 @@ describe('SSO primary provider', () => { expect(container.querySelector('#sso-test-link')).toBeNull() }) + it("shows an OIDC provider's initiate login URL for its identity provider's app dashboard", () => { + renderMigration() + openProvider('acme-entra') + + expect(container).toHaveTextContent('Initiate login URL') + const link = new URL( + container.querySelector('#sso-initiate-login-url')?.value ?? '' + ) + expect(link.pathname).toBe('/sso/launch/acme-entra') + expect(link.search).toBe('') + }) + + it('shows no initiate login URL on a SAML provider', () => { + renderMigration('', {}, { providerType: 'saml' }) + openProvider('acme-entra') + + expect(container.querySelector('#sso-initiate-login-url')).toBeNull() + }) + it('offers a test sign-in link and Make primary on a provider waiting beside the primary', () => { renderMigration() openProvider('acme-okta') diff --git a/apps/sim/lib/auth/sso/idp-initiated-login.test.ts b/apps/sim/lib/auth/sso/idp-initiated-login.test.ts new file mode 100644 index 00000000000..2ebcbeb41d6 --- /dev/null +++ b/apps/sim/lib/auth/sso/idp-initiated-login.test.ts @@ -0,0 +1,47 @@ +/** + * @vitest-environment node + */ +import { dbChainMock, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@sim/db', () => ({ ...dbChainMock, ...schemaMock })) + +import { isIdpInitiatedLoginAllowed } from '@/lib/auth/sso/idp-initiated-login' + +function queueProvider(issuer: string) { + queueTableRows(schemaMock.ssoProvider, [{ issuer }]) +} + +describe('isIdpInitiatedLoginAllowed', () => { + beforeEach(() => { + resetDbChainMock() + }) + + it.each([ + ['the issuer it is configured with', 'https://acme.okta.test', 'https://acme.okta.test'], + ['that issuer with a trailing slash', 'https://acme.okta.test', 'https://acme.okta.test/'], + [ + 'the organization URL of its custom authorization server', + 'https://acme.okta.test/oauth2/default', + 'https://acme.okta.test', + ], + ])('allows a provider opened by %s', async (_label, configured, opened) => { + queueProvider(configured) + await expect(isIdpInitiatedLoginAllowed('acme-okta', opened)).resolves.toBe(true) + }) + + it.each([ + ['another identity provider', 'https://attacker.example.test'], + ['a value that is not a URL', 'not-a-url'], + ])('refuses a link opened by %s', async (_label, opened) => { + queueProvider('https://acme.okta.test') + await expect(isIdpInitiatedLoginAllowed('acme-okta', opened)).resolves.toBe(false) + }) + + it('refuses a provider that is unknown, unverified, or SAML', async () => { + queueTableRows(schemaMock.ssoProvider, []) + await expect(isIdpInitiatedLoginAllowed('acme-okta', 'https://acme.okta.test')).resolves.toBe( + false + ) + }) +}) diff --git a/apps/sim/lib/auth/sso/idp-initiated-login.ts b/apps/sim/lib/auth/sso/idp-initiated-login.ts new file mode 100644 index 00000000000..67957c7dcc2 --- /dev/null +++ b/apps/sim/lib/auth/sso/idp-initiated-login.ts @@ -0,0 +1,51 @@ +import { db, ssoProvider } from '@sim/db' +import { and, eq, isNull } from 'drizzle-orm' + +/** Issuers compare without trailing slashes, which identity providers add or drop freely. */ +function normalizeIssuer(issuer: string): string { + return issuer.trim().replace(/\/+$/, '') +} + +/** The issuer's origin, so an Okta custom authorization server matches its organization URL. */ +function issuerOrigin(issuer: string): string | null { + try { + return new URL(issuer).origin + } catch { + return null + } +} + +/** + * Whether an identity provider's app dashboard may start sign-in through this provider + * (OpenID Connect third-party initiated login). + * + * The dashboard opens the provider's initiate login URL with its own issuer in `iss`. It is + * honored only for a domain-verified OIDC provider configured with that issuer, or one on the + * same host — Okta sends the organization URL even for a provider registered against a custom + * authorization server under it. The gate is defense in depth: a crafted link can then only + * reach an identity provider this deployment already registered, never an attacker's own, and + * Better Auth re-checks the provider before it issues the authorization request. + */ +export async function isIdpInitiatedLoginAllowed( + providerId: string, + issuer: string +): Promise { + const [provider] = await db + .select({ issuer: ssoProvider.issuer }) + .from(ssoProvider) + .where( + and( + eq(ssoProvider.providerId, providerId), + eq(ssoProvider.domainVerified, true), + isNull(ssoProvider.samlConfig) + ) + ) + .limit(1) + if (!provider) return false + + const configured = normalizeIssuer(provider.issuer) + const opened = normalizeIssuer(issuer) + if (configured === opened) return true + const configuredOrigin = issuerOrigin(configured) + return configuredOrigin !== null && configuredOrigin === issuerOrigin(opened) +} From 1e7c3dcfd970877231130100b9bd35a708735b96 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 15 Sep 2026 16:55:57 -0700 Subject: [PATCH 12/15] fix(cleanup): preserve log rows when file deletion fails (#7869) --- .../sim/background/cleanup-logs-files.test.ts | 111 ++++++++++++++++++ apps/sim/background/cleanup-logs.ts | 2 +- 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 apps/sim/background/cleanup-logs-files.test.ts diff --git a/apps/sim/background/cleanup-logs-files.test.ts b/apps/sim/background/cleanup-logs-files.test.ts new file mode 100644 index 00000000000..ebcb62fbc23 --- /dev/null +++ b/apps/sim/background/cleanup-logs-files.test.ts @@ -0,0 +1,111 @@ +/** + * @vitest-environment node + */ + +import { dbChainMockFns, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockDeleteFiles, mockDeleteFileMetadata } = vi.hoisted(() => ({ + mockDeleteFiles: vi.fn(), + mockDeleteFileMetadata: vi.fn(), +})) + +vi.mock('@trigger.dev/sdk', () => ({ + task: vi.fn((config) => config), + queue: vi.fn((config) => config), +})) +vi.mock('@/lib/billing/cleanup-dispatcher', () => ({ runCleanupWithLimits: vi.fn() })) +vi.mock('@/lib/execution/payloads/large-value-metadata', () => ({ + LIVE_PAUSED_REFERENCE_STATUSES: ['paused', 'partially_resumed', 'cancelling'], + markLargeValuesDeleted: vi.fn(), + pruneLargeValueMetadata: vi.fn(async () => ({ + referencesDeleted: 0, + dependenciesDeleted: 0, + tombstonesDeleted: 0, + })), + unreferencedLargeValuePredicate: vi.fn(), +})) +vi.mock('@/lib/logs/execution/snapshot/service', () => ({ + snapshotService: { cleanupOrphanedSnapshots: vi.fn() }, +})) +vi.mock('@/lib/uploads', () => ({ + isUsingCloudStorage: vi.fn(() => true), + StorageService: { deleteFiles: mockDeleteFiles }, +})) +vi.mock('@/lib/uploads/server/metadata', () => ({ + deleteFileMetadata: mockDeleteFileMetadata, +})) + +import { createCleanupBudgets } from '@/lib/cleanup/limits' +import { runCleanupLogs } from '@/background/cleanup-logs' + +const payload = { + label: 'free/1', + plan: 'free' as const, + retentionHours: 720, + workspaceIds: ['workspace-1'], +} +const rows = [ + { id: 'log-1', files: [{ key: 'file-a' }, { key: 'file-b' }] }, + { id: 'log-2', files: [{ key: 'file-c' }] }, +] + +describe('bounded log cleanup file failures', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValueOnce(rows) + dbChainMockFns.returning.mockResolvedValueOnce(rows.map(({ id }) => ({ id }))) + mockDeleteFiles.mockImplementation(async (keys: string[]) => ({ + deleted: keys.length, + failed: [], + })) + mockDeleteFileMetadata.mockResolvedValue(true) + }) + + it.each([ + { + name: 'the storage request throws', + fail: () => mockDeleteFiles.mockRejectedValueOnce(new Error('storage unavailable')), + }, + { + name: 'storage returns a partial failure', + fail: () => + mockDeleteFiles.mockResolvedValueOnce({ + deleted: 1, + failed: [{ key: 'file-b', error: 'storage unavailable' }], + }), + }, + { + name: 'metadata deletion throws', + fail: () => mockDeleteFileMetadata.mockRejectedValueOnce(new Error('database unavailable')), + }, + ])('keeps the entire log batch retryable when $name', async ({ fail }) => { + fail() + const budgets = createCleanupBudgets({ workflowLogs: 2 }) + + await expect(runCleanupLogs(payload, budgets)).rejects.toThrow('Log file cleanup failed') + + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + expect(mockDeleteFiles).toHaveBeenCalledTimes(1) + expect(budgets.workflowLogs.remaining).toBe(0) + + dbChainMockFns.limit.mockResolvedValueOnce(rows) + await runCleanupLogs(payload, createCleanupBudgets({ workflowLogs: 2 })) + + expect(mockDeleteFiles).toHaveBeenNthCalledWith(2, ['file-a', 'file-b'], 'execution') + expect(mockDeleteFiles).toHaveBeenNthCalledWith(3, ['file-c'], 'execution') + expect(dbChainMockFns.delete).toHaveBeenCalledExactlyOnceWith(schemaMock.workflowExecutionLogs) + }) + + it('deletes log rows only after every file and its metadata succeeds', async () => { + await runCleanupLogs(payload, createCleanupBudgets({ workflowLogs: 2 })) + + expect(mockDeleteFiles).toHaveBeenCalledTimes(2) + expect(mockDeleteFileMetadata).toHaveBeenCalledTimes(3) + expect(dbChainMockFns.delete).toHaveBeenCalledExactlyOnceWith(schemaMock.workflowExecutionLogs) + const deleteOrder = dbChainMockFns.delete.mock.invocationCallOrder[0] + expect(mockDeleteFiles.mock.invocationCallOrder.at(-1)).toBeLessThan(deleteOrder) + expect(mockDeleteFileMetadata.mock.invocationCallOrder.at(-1)).toBeLessThan(deleteOrder) + }) +}) diff --git a/apps/sim/background/cleanup-logs.ts b/apps/sim/background/cleanup-logs.ts index e46127a8e6a..c26b2f2a76d 100644 --- a/apps/sim/background/cleanup-logs.ts +++ b/apps/sim/background/cleanup-logs.ts @@ -429,6 +429,7 @@ async function cleanupWorkflowExecutionLogs( onBatch: async (rows) => { for (const row of rows) { await deleteExecutionFiles(row.files, fileStats) + if (budget && fileStats.filesDeleteFailed) throw new Error('Log file cleanup failed') } }, batchSize: WORKFLOW_LOG_CLEANUP_BATCH_SIZE, @@ -486,7 +487,6 @@ export async function runCleanupLogs( logger.info( `[${label}] workflow_execution_logs files: ${workflowResults.filesDeleted}/${workflowResults.filesTotal} deleted, ${workflowResults.filesDeleteFailed} failed` ) - if (budgets && workflowResults.filesDeleteFailed) throw new Error('Log file cleanup failed') const largeValueResults = await cleanupLargeExecutionValues( workspaceIds, retentionDate, From 8e14e4d7ab765105cbc7d270626872f3ad096321 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Tue, 15 Sep 2026 17:07:00 -0700 Subject: [PATCH 13/15] fix(provenance): align file matching and report withheld attachments (#7867) * fix(provenance): stop silently dropping model attachments * fix(provenance): keep model turns running after attachment refusal * fix(provenance): apply literal policy before file classification --- .../handlers/agent/agent-handler.test.ts | 122 ++++++++----- .../executor/handlers/agent/agent-handler.ts | 10 +- .../lib/copilot/request/lifecycle/run.test.ts | 113 ++++++++++-- apps/sim/lib/copilot/request/lifecycle/run.ts | 45 +++-- .../mounted-file-secret-provenance.test.ts | 46 +++++ .../mounted-file-secret-provenance.ts | 19 +- .../execute-request.test.ts | 166 +++++++++++++----- .../lib/function-execution/execute-request.ts | 1 + apps/sim/lib/uploads/utils/model-input.ts | 9 + apps/sim/providers/index.test.ts | 104 ++++++++--- apps/sim/providers/index.ts | 16 +- 11 files changed, 491 insertions(+), 160 deletions(-) diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index edd09b45fd2..b5b61ec6c1d 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -1040,55 +1040,87 @@ describe('AgentBlockHandler', () => { } }) - it('omits only a generated document whose embedded contributor is not model-safe', async () => { - const key = 'workspace/ws-1/report.pdf' - mockContext.workspaceId = 'ws-1' - const hydrationSpy = vi - .spyOn(userFileBase64, 'hydrateUserFilesWithBase64') - .mockImplementationOnce(async (files, options) => { - await options.onServableFileContributors?.(files[0], [ - { - fileId: 'image-1', - key: 'workspace/ws-1/image-1.png', - context: 'workspace', - contentUpdatedAt: new Date('2026-08-06T00:00:00.000Z'), - }, - ]) - return files.map((file) => ({ ...file, base64: 'JVBERi0=' })) - }) - mockImportWorkspaceFileSecretProvenanceForModelView.mockResolvedValueOnce(false) + it.each([ + { safe: false, includeSafeFile: false }, + { safe: false, includeSafeFile: true }, + { safe: true, includeSafeFile: true }, + ])( + 'continues after document contributor admission (safe=$safe, mixed=$includeSafeFile)', + async ({ safe, includeSafeFile }) => { + const key = 'workspace/ws-1/report.pdf' + mockContext.workspaceId = 'ws-1' + const hydrationSpy = vi + .spyOn(userFileBase64, 'hydrateUserFilesWithBase64') + .mockImplementationOnce(async (files, options) => { + await options.onServableFileContributors?.(files[0], [ + { + fileId: 'image-1', + key: 'workspace/ws-1/image-1.png', + context: 'workspace', + contentUpdatedAt: new Date('2026-08-06T00:00:00.000Z'), + }, + ]) + return files.map((file) => ({ ...file, base64: 'JVBERi0=' })) + }) + mockImportWorkspaceFileSecretProvenanceForModelView.mockResolvedValueOnce(safe) - try { - mockGetProviderFromModel.mockReturnValue('openai') + try { + mockGetProviderFromModel.mockReturnValue('openai') - await handler.execute(mockContext, mockBlock, { - model: 'gpt-4o', - userPrompt: 'Analyze this document', - files: [ - { - id: 'file-1', - name: 'report.pdf', - path: `/api/files/serve/${encodeURIComponent(key)}?context=workspace`, - key, - size: 128, - type: 'text/x-python-pdf', - }, - ], - apiKey: 'test-api-key', - }) - - expect(mockExecuteProviderRequest.mock.calls[0][1].messages.at(-1)?.files).toEqual([]) - expect(mockImportWorkspaceFileSecretProvenanceForModelView).toHaveBeenCalledWith( - expect.objectContaining({ - workspaceId: mockContext.workspaceId, - view: 'opaque', - identity: expect.objectContaining({ fileId: 'image-1' }), + await handler.execute(mockContext, mockBlock, { + model: 'gpt-4o', + userPrompt: 'Analyze this document', + files: [ + { + id: 'file-1', + name: 'report.pdf', + path: `/api/files/serve/${encodeURIComponent(key)}?context=workspace`, + key, + size: 128, + type: 'text/x-python-pdf', + }, + ...(includeSafeFile + ? [ + { + id: 'file-2', + name: 'safe.pdf', + path: '/safe.pdf', + key: 'workspace/ws-1/safe.pdf', + size: 128, + type: 'application/pdf', + }, + ] + : []), + ], + apiKey: 'test-api-key', }) - ) - } finally { - hydrationSpy.mockRestore() + + expect(mockExecuteProviderRequest).toHaveBeenCalledOnce() + const sent = mockExecuteProviderRequest.mock.calls[0][1].messages.at(-1) + expect(sent.files.map((file: { id: string }) => file.id)).toEqual([ + ...(safe ? ['file-1'] : []), + ...(includeSafeFile ? ['file-2'] : []), + ]) + if (safe) { + expect(sent.content).toBe('Analyze this document') + } else { + expect(sent.content).toMatch( + /^Analyze this document\n\nAttachment error: 1 requested file attachment was not provided/ + ) + expect(JSON.stringify(sent)).not.toContain(key) + } + expect(mockImportWorkspaceFileSecretProvenanceForModelView).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: mockContext.workspaceId, + view: 'opaque', + identity: expect.objectContaining({ fileId: 'image-1' }), + }) + ) + } finally { + hydrationSpy.mockRestore() + } } - }) + ) it('should reject files for providers without attachment support', async () => { const inputs = { diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index f33ff7b8ea1..d96aeed8f74 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -34,7 +34,10 @@ import { type RawFileInput, tryInferContextFromKey, } from '@/lib/uploads/utils/file-utils' -import { selectModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' +import { + appendUnavailableAttachmentNotice, + selectModelBoundFileInputPaths, +} from '@/lib/uploads/utils/model-input' import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' import { resolveCustomBlockToolBinding } from '@/lib/workflows/custom-blocks/operations' import { @@ -1602,8 +1605,13 @@ export class AgentBlockHandler implements BlockHandler { ) } + const omittedCount = hydratedFiles.length - modelSafeHydratedFiles.length nextMessages[messageIndex] = { ...message, + content: + omittedCount > 0 + ? appendUnavailableAttachmentNotice(message.content, omittedCount) + : message.content, files: modelSafeHydratedFiles, } } diff --git a/apps/sim/lib/copilot/request/lifecycle/run.test.ts b/apps/sim/lib/copilot/request/lifecycle/run.test.ts index 5f8f74b27a1..655067607cb 100644 --- a/apps/sim/lib/copilot/request/lifecycle/run.test.ts +++ b/apps/sim/lib/copilot/request/lifecycle/run.test.ts @@ -665,31 +665,110 @@ describe('runCopilotLifecycle', () => { expect(sent.fileAttachments).toEqual([{ name: 'TOKEN.txt', key: 'safe-key' }]) }) - it('omits only unsafe durable attachments before the initial Go request', async () => { - const unsafe = { id: 'wf-unsafe', name: 'unsafe.txt', key: 'workspace/ws-1/unsafe.txt' } - const safe = { id: 'wf-safe', name: 'safe.txt', key: 'workspace/ws-1/safe.txt' } - mockFilterModelSafeWorkspaceFileAttachments.mockResolvedValueOnce([safe]) - let capturedRequestBody = '' - mockRunStreamLoop.mockImplementationOnce(async (_url: string, request: RequestInit) => { - capturedRequestBody = String(request.body) - }) - - await runCopilotLifecycle( - { + it.each([ + { key: 'attachments', includeSafeFile: false }, + { key: 'attachments', includeSafeFile: true }, + { key: 'fileAttachments', includeSafeFile: false }, + { key: 'fileAttachments', includeSafeFile: true }, + ])( + 'continues with an error notice for refused $key (mixed=$includeSafeFile)', + async ({ key, includeSafeFile }) => { + const unsafe = { + id: 'wf-private', + name: 'private-filename.txt', + key: 'private-storage-key', + base64: 'private-bytes', + } + const safe = { id: 'wf-safe', name: 'safe.txt', key: 'workspace/ws-1/safe.txt' } + const safeFiles = includeSafeFile ? [safe] : [] + mockFilterModelSafeWorkspaceFileAttachments.mockResolvedValueOnce(safeFiles) + const onError = vi.fn() + mockRunStreamLoop.mockImplementationOnce(async (_url, _request, context) => { + context.accumulatedContent = 'I can continue with the available inputs.' + context.completionStatus = MothershipStreamV1CompletionStatus.complete + }) + const payload = { message: 'Review files', - fileAttachments: [unsafe, safe], + [key]: [...safeFiles, unsafe], workspaceId: 'ws-1', messageId: 'stream-file-provenance', - }, - { + } + const originalPayload = structuredClone(payload) + + const result = await runCopilotLifecycle(payload, { userId: 'user-1', workspaceId: 'ws-1', executionContext: { userId: 'user-1', workflowId: '', workspaceId: 'ws-1' }, + onError, + }) + + expect(result).toMatchObject({ + success: true, + content: 'I can continue with the available inputs.', + }) + expect(onError).not.toHaveBeenCalled() + expect(mockRunStreamLoop).toHaveBeenCalledOnce() + const sent = JSON.parse(String(mockRunStreamLoop.mock.calls[0][1].body)) + expect(sent.message).toMatch( + /^Review files\n\nAttachment error: 1 requested file attachment was not provided/ + ) + expect(sent[key] ?? []).toEqual(safeFiles) + expect(JSON.stringify(sent)).not.toContain('private-') + expect(mockFilterModelSafeWorkspaceFileAttachments).toHaveBeenCalledWith( + [...safeFiles, unsafe], + { workspaceId: 'ws-1' } + ) + expect(payload).toEqual(originalPayload) + } + ) + + it.each(['messages', 'both', 'attachment-only', 'system-only'])( + 'reports combined attachment refusals in %s payloads without changing history', + async (shape) => { + const history = { + role: 'assistant', + content: 'Previous response', + tool_calls: [{ id: 'existing-call' }], + } + const messages = + shape === 'system-only' + ? [{ role: 'system', content: 'System context' }] + : [history, { role: 'user', content: 'Review files' }] + const payload = { + ...(shape === 'both' ? { message: 'Review files' } : {}), + ...(shape === 'attachment-only' ? {} : { messages }), + attachments: [{ key: 'private-first-file' }], + fileAttachments: [{ key: 'private-second-file' }], } - ) + const original = structuredClone(payload) + mockFilterModelSafeWorkspaceFileAttachments + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) + mockRunStreamLoop.mockResolvedValueOnce(undefined) - expect(JSON.parse(capturedRequestBody).fileAttachments).toEqual([safe]) - }) + const result = await runCopilotLifecycle(payload, { + userId: 'user-1', + workspaceId: 'ws-1', + executionContext: { userId: 'user-1', workflowId: '', workspaceId: 'ws-1' }, + }) + + expect(result.success).toBe(true) + const sent = JSON.parse(String(mockRunStreamLoop.mock.calls[0][1].body)) + const notice = 'Attachment error: 2 requested file attachments were not provided' + if (shape === 'both' || shape === 'attachment-only') expect(sent.message).toContain(notice) + if (shape !== 'attachment-only') { + expect(sent.messages[0]).toEqual(messages[0]) + expect(sent.messages.at(-1)).toMatchObject({ + role: 'user', + content: expect.stringContaining(notice), + }) + } + expect(sent).not.toHaveProperty('attachments') + expect(sent).not.toHaveProperty('fileAttachments') + expect(JSON.stringify(sent)).not.toContain('private-') + expect(payload).toEqual(original) + } + ) it('rejects when durable attachment provenance cannot be verified', async () => { mockFilterModelSafeWorkspaceFileAttachments.mockRejectedValueOnce(new Error('db unavailable')) diff --git a/apps/sim/lib/copilot/request/lifecycle/run.ts b/apps/sim/lib/copilot/request/lifecycle/run.ts index d9b090fd48a..ba50ca1ef5e 100644 --- a/apps/sim/lib/copilot/request/lifecycle/run.ts +++ b/apps/sim/lib/copilot/request/lifecycle/run.ts @@ -4,7 +4,7 @@ import type { PermissionType } from '@sim/platform-authz/workspace' import { getErrorMessage, toError } from '@sim/utils/errors' import { interruptibleSleep, sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { omit } from '@sim/utils/object' +import { isPlainRecord, omit } from '@sim/utils/object' import { workspaceSearchFiltersSchema } from '@/lib/api/contracts/knowledge/search' import { type AttributedBillingRequestEnvelope, @@ -72,6 +72,7 @@ import { env } from '@/lib/core/config/env' import { isCopilotToolPermissionsEnabled, isHosted } from '@/lib/core/config/env-flags' import { isWorkspaceCapabilityWithheld } from '@/lib/permission-groups/capability-assertions' import { filterModelSafeWorkspaceFileAttachments } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { appendUnavailableAttachmentNotice } from '@/lib/uploads/utils/model-input' import type { ExecutorDelegationOrigin } from '@/executor/types' import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -95,11 +96,12 @@ class CopilotModelContentProjectionError extends Error { } } -async function omitUnsafeInitialCopilotAttachments( +async function prepareInitialCopilotAttachmentsForModel( payload: Record, workspaceId?: string ): Promise> { let projected = payload + let omittedCount = 0 for (const key of ['attachments', 'fileAttachments'] as const) { if (!Object.hasOwn(projected, key)) continue const attachments = projected[key] @@ -129,6 +131,7 @@ async function omitUnsafeInitialCopilotAttachments( } if (safeAttachments.length === attachments.length) continue + omittedCount += attachments.length - safeAttachments.length logger.warn('Omitting Copilot attachments with unsafe secret provenance', { attachmentCount: attachments.length, omittedCount: attachments.length - safeAttachments.length, @@ -136,14 +139,36 @@ async function omitUnsafeInitialCopilotAttachments( projected = safeAttachments.length > 0 ? { ...projected, [key]: safeAttachments } : omit(projected, [key]) } - return projected -} + if (omittedCount === 0) return projected -async function filterInitialCopilotAttachmentsForModel( - payload: Record, - workspaceId?: string -): Promise> { - return omitUnsafeInitialCopilotAttachments(payload, workspaceId) + if (typeof projected.message === 'string') { + projected = { + ...projected, + message: appendUnavailableAttachmentNotice(projected.message, omittedCount), + } + } + if (Array.isArray(projected.messages)) { + const messages: unknown[] = [...projected.messages] + let notified = false + for (let index = messages.length - 1; index >= 0; index--) { + const message = messages[index] + if (!isPlainRecord(message) || message.role !== 'user' || typeof message.content !== 'string') + continue + messages[index] = { + ...message, + content: appendUnavailableAttachmentNotice(message.content, omittedCount), + } + notified = true + break + } + if (!notified) { + messages.push({ role: 'user', content: appendUnavailableAttachmentNotice('', omittedCount) }) + } + projected = { ...projected, messages } + } else if (typeof projected.message !== 'string') { + projected = { ...projected, message: appendUnavailableAttachmentNotice('', omittedCount) } + } + return projected } async function ensureModelEgressRegistry( @@ -412,7 +437,7 @@ export async function runCopilotLifecycle( }), } } - const modelSafeRequestPayload = await filterInitialCopilotAttachmentsForModel( + const modelSafeRequestPayload = await prepareInitialCopilotAttachmentsForModel( requestPayload, lifecycleOptions.workspaceId ) diff --git a/apps/sim/lib/execution/mounted-file-secret-provenance.test.ts b/apps/sim/lib/execution/mounted-file-secret-provenance.test.ts index 77be6d392a0..8a4dccf5a4e 100644 --- a/apps/sim/lib/execution/mounted-file-secret-provenance.test.ts +++ b/apps/sim/lib/execution/mounted-file-secret-provenance.test.ts @@ -84,6 +84,52 @@ describe('mounted file output provenance scanner', () => { expect(scanner?.hasSecrets).toBe(true) }) + it.each(['false', 'hunter2', '""""'])( + 'excludes short plaintext %j before escaping it', + async (plaintext) => { + encryptionMockFns.mockDecryptSecret.mockResolvedValue({ decrypted: plaintext }) + + const scanner = await createMountedFileSecretProvenanceScanner({ + version: 1, + complete: true, + entries: [{ encryptedValue: 'encrypted-short' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + + expect(scanner?.scan(Buffer.from(JSON.stringify(plaintext)))).toEqual({ + status: 'exact', + entries: [], + }) + expect(scanner?.hasSecrets).toBe(false) + } + ) + + it('protects an eight-character literal alongside excluded short entries', async () => { + encryptionMockFns.mockDecryptSecret.mockImplementation(async (value: string) => ({ + decrypted: value === 'encrypted-short' ? 'false' : 'hunter22', + })) + + const scanner = await createMountedFileSecretProvenanceScanner({ + version: 1, + complete: true, + entries: [{ encryptedValue: 'encrypted-short' }, { encryptedValue: 'encrypted-boundary' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + + expect(scanner?.hasSecrets).toBe(true) + expect(scanner?.scan(Buffer.from('false hunter22'))).toEqual({ + status: 'exact', + entries: [ + { + name: 'MOUNTED_FILE_SECRET', + encryptedValue: 'encrypted-boundary', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + ], + }) + }) + it('classifies outputs unknown when authenticated mount provenance cannot be inspected', async () => { const incomplete = await createMountedFileSecretProvenanceScanner({ version: 1, diff --git a/apps/sim/lib/execution/mounted-file-secret-provenance.ts b/apps/sim/lib/execution/mounted-file-secret-provenance.ts index 44ddfddc870..15ec17b42fa 100644 --- a/apps/sim/lib/execution/mounted-file-secret-provenance.ts +++ b/apps/sim/lib/execution/mounted-file-secret-provenance.ts @@ -5,6 +5,7 @@ import { createResolvedSecretMatcher, scanResolvedSecretString, } from '@/executor/utils/resolved-secret-content-projection' +import { isNonIdentifyingSecretLiteral } from '@/executor/utils/resolved-secret-match-policy' import type { ResolvedSecretTraceProvenanceV1 } from '@/executor/utils/resolved-secret-trace-registry' const MAX_MOUNTED_FILE_SECRET_MATCH_EVENTS = 1_000_000 @@ -12,11 +13,10 @@ const ANONYMOUS_MOUNTED_FILE_SECRET_NAME = 'MOUNTED_FILE_SECRET' export interface MountedFileSecretProvenanceScanner { /** - * True when the envelope attested to any secret material, whether or not it could be turned into - * a scannable literal. False therefore means the mount carried nothing to leak — which lets - * callers classify content this scanner cannot soundly scan (binary bytes) instead of failing - * closed. Entries that fail to yield plaintext keep this true: losing the ability to scan them - * makes the mount less classifiable, not more. + * True when the envelope carries material protected by the shared literal policy, or an entry + * cannot be inspected. Successfully decrypted short values do not taint derived binary files. + * Entries that fail to yield plaintext keep this true: losing the ability to scan them makes + * the mount less classifiable, not more. */ hasSecrets: boolean scan(buffer: Buffer): WorkspaceFileSecretProvenance @@ -42,12 +42,17 @@ export async function createMountedFileSecretProvenanceScanner( if (!provenance.complete) return UNKNOWN_MOUNTED_FILE_SECRET_PROVENANCE_SCANNER if (!provenance.scope?.userId) return undefined - const hasSecrets = provenance.entries.length > 0 + let hasSecrets = false const entriesByScanLiteral = new Map>() try { for (const entry of provenance.entries) { const { decrypted: plaintext } = await decryptSecret(entry.encryptedValue) - if (!plaintext) continue + if (!plaintext) { + hasSecrets = true + continue + } + if (isNonIdentifyingSecretLiteral(plaintext)) continue + hasSecrets = true const fileEntry: WorkspaceFileSecretProvenanceEntry = { name: entry.name || ANONYMOUS_MOUNTED_FILE_SECRET_NAME, encryptedValue: entry.encryptedValue, diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index 36ceb0a0e59..30f47ef2387 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -1018,6 +1018,67 @@ describe('Function execution request', () => { ) }) + it('excludes short compiled plaintext before JSON escaping even with a protected secret in scope', async () => { + const shortValue = '""""' + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { + '/home/user/short.json': JSON.stringify({ value: shortValue }), + '/home/user/protected.txt': 'hunter22', + }, + }) + + const response = await POST( + createMockRequest('POST', { + code: 'print({{SHORT_VALUE}}, {{API_KEY}})', + language: 'python', + workspaceId: 'workspace-1', + envVars: { SHORT_VALUE: shortValue, API_KEY: 'hunter22' }, + outputs: { + files: [ + { + path: 'files/short.json', + sandboxPath: '/home/user/short.json', + mimeType: 'application/json', + }, + { + path: 'files/protected.txt', + sandboxPath: '/home/user/protected.txt', + mimeType: 'text/plain', + }, + ], + }, + }) + ) + + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: expect.objectContaining({ path: 'files/short.json' }), + secretProvenance: { status: 'exact', entries: [] }, + }) + ) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: expect.objectContaining({ path: 'files/protected.txt' }), + secretProvenance: { + status: 'exact', + entries: [ + { + name: 'API_KEY', + encryptedValue: 'encrypted:hunter22', + sourceUserId: 'user-123', + sourceWorkspaceId: 'workspace-1', + }, + ], + }, + }) + ) + }) + it('classifies exports exact-empty when the only compiled secret is exempt, still reporting its name', async () => { envFlagsMock.isRemoteSandboxEnabled = true mockExecuteInSandbox.mockResolvedValueOnce({ @@ -1369,58 +1430,69 @@ describe('Function execution request', () => { ) }) - it('keeps a binary export unknown when a mounted input file carried a secret', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/small.jpg': '/9j/4AAQ' }, - }) + it.each([ + { plaintext: 'mounted-secret', expectedStatus: 'unknown' }, + { plaintext: 'false', expectedStatus: 'exact' }, + { plaintext: '""""', expectedStatus: 'exact' }, + ])( + 'classifies binary exports $expectedStatus with mounted plaintext $plaintext', + async ({ plaintext, expectedStatus }) => { + mockDecryptSecret.mockResolvedValueOnce({ decrypted: plaintext }) + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/small.jpg': '/9j/4AAQ' }, + }) - const response = await POST( - createMockRequest( - 'POST', - { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/small.jpg', - sandboxPath: '/home/user/small.jpg', - mimeType: 'image/jpeg', - }, - ], - }, - [PRIVATE_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - selections: [ - { - key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, - provenance: { - version: 1, - complete: true, - entries: [{ encryptedValue: 'encrypted:mounted-secret' }], - scope: { userId: 'user-123', workspaceId: 'workspace-1' }, + const response = await POST( + createMockRequest( + 'POST', + { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/small.jpg', + sandboxPath: '/home/user/small.jpg', + mimeType: 'image/jpeg', }, - }, - ], + ], + }, + [PRIVATE_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: true, + selections: [ + { + key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, + provenance: { + version: 1, + complete: true, + entries: [{ encryptedValue: 'encrypted:mounted-secret' }], + scope: { userId: 'user-123', workspaceId: 'workspace-1' }, + }, + }, + ], + }, }, - }, - { - [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, - } + { + [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, + } + ) ) - ) - expect(response.status).toBe(200) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ secretProvenance: { status: 'unknown' } }) - ) - }) + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + secretProvenance: + expectedStatus === 'exact' ? { status: 'exact', entries: [] } : { status: 'unknown' }, + }) + ) + } + ) it('marks binary exports unknown without failing the Function execution', async () => { envFlagsMock.isRemoteSandboxEnabled = true diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index 020b7b32f9d..de66f8dbfe9 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -2452,6 +2452,7 @@ export async function executeFunctionRequest( * owner's provenance and the file still locks. */ if (routeContext.unredactedSecretNames.has(name)) continue + if (isNonIdentifyingSecretLiteral(plaintext)) continue const scanLiterals = new Set([plaintext, JSON.stringify(plaintext).slice(1, -1)]) for (const scanLiteral of scanLiterals) { const names = routeContext.outputSecretNamesByScanLiteral.get(scanLiteral) ?? [] diff --git a/apps/sim/lib/uploads/utils/model-input.ts b/apps/sim/lib/uploads/utils/model-input.ts index 736de9c7eaf..214f0a3f459 100644 --- a/apps/sim/lib/uploads/utils/model-input.ts +++ b/apps/sim/lib/uploads/utils/model-input.ts @@ -149,3 +149,12 @@ export function applyProjectedModelVisibleFileNames( } return { ...original, name: projected.name } } + +/** Reports withheld attachments without exposing their unverified names, locators, or contents. */ +export function appendUnavailableAttachmentNotice( + content: string | null | undefined, + omittedCount: number +): string { + const notice = `Attachment error: ${omittedCount} requested file attachment${omittedCount === 1 ? ' was' : 's were'} not provided because file safety checks failed. Their contents are unavailable. Continue with the available inputs and explain any resulting limitation.` + return content ? `${content}\n\n${notice}` : notice +} diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 5594b448d99..c254ef97f1c 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -963,35 +963,85 @@ describe('executeProviderRequest — caller-prepared model input', () => { }) }) - it('omits only unsafe durable files before any provider attachment processing', async () => { - const unsafe = { - id: 'wf-unsafe', - name: 'unsafe.txt', - url: '/unsafe', - size: 10, - type: 'text/plain', - key: 'workspace/ws-1/unsafe.txt', - } - const safe = { - id: 'wf-safe', - name: 'safe.txt', - url: '/safe', - size: 10, - type: 'text/plain', - key: 'workspace/ws-1/safe.txt', - } - mockFilterModelSafeWorkspaceFileAttachments.mockResolvedValueOnce([safe]) + it.each([ + { stream: false, includeSafeFile: false }, + { stream: false, includeSafeFile: true }, + { stream: true, includeSafeFile: false }, + { stream: true, includeSafeFile: true }, + ])( + 'continues with an attachment error notice (stream=$stream, mixed=$includeSafeFile)', + async ({ stream, includeSafeFile }) => { + const unsafe = { + id: 'wf-unsafe', + name: 'private-filename.txt', + url: '/private-file-url', + size: 10, + type: 'text/plain', + key: 'workspace/ws-1/private-storage-key.txt', + base64: 'private-file-bytes', + } + const safe = { + ...unsafe, + id: 'wf-safe', + name: 'safe.txt', + url: '/safe', + key: 'safe-key', + base64: 'safe-bytes', + } + const safeFiles = includeSafeFile ? [safe] : [] + mockFilterModelSafeWorkspaceFileAttachments.mockResolvedValueOnce(safeFiles) + const messages = [ + { role: 'user' as const, content: 'Earlier context' }, + { + role: 'user' as const, + content: includeSafeFile ? 'Review files' : null, + files: [...safeFiles, unsafe], + }, + ] + const originalMessages = structuredClone(messages) + if (stream) { + mockExecuteRequest.mockResolvedValueOnce({ + stream: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('ok')) + controller.close() + }, + }), + execution: { success: true, output: { content: 'ok' } }, + }) + } - await executeProviderRequest('openai', { - model: 'test-model', - workspaceId: 'ws-1', - messages: [{ role: 'user', content: 'Review files', files: [unsafe, safe] }], - }) + const response = await executeProviderRequest('openai', { + model: 'test-model', + workspaceId: 'ws-1', + userId: 'user-1', + stream, + messages, + }) - expect(mockAttachLargeFileRemoteUrls.mock.calls[0][0].messages[0].files).toEqual([safe]) - expect(mockUploadLargeFilesToProvider.mock.calls[0][0].messages[0].files).toEqual([safe]) - expect(mockExecuteRequest.mock.calls[0][0].messages[0].files).toEqual([safe]) - }) + if (stream) { + expect(await new Response((response as StreamingExecution).stream).text()).toBe('ok') + } else { + expect(response).toMatchObject({ content: 'ok' }) + } + expect(mockFilterModelSafeWorkspaceFileAttachments).toHaveBeenCalledWith( + [...safeFiles, unsafe], + { workspaceId: 'ws-1', actorUserId: 'user-1' } + ) + const sent = mockExecuteRequest.mock.calls[0][0] + expect(sent.messages[0]).toEqual(messages[0]) + expect(sent.messages[1].content).toContain( + 'Attachment error: 1 requested file attachment was not provided' + ) + expect(sent.messages[1].content).toContain('Continue with the available inputs') + if (includeSafeFile) expect(sent.messages[1].content).toMatch(/^Review files\n\n/) + expect(sent.messages[1].files ?? []).toEqual(safeFiles) + expect(JSON.stringify(sent)).not.toContain('private-') + expect(mockAttachLargeFileRemoteUrls.mock.calls[0][0]).toBe(sent) + expect(mockUploadLargeFilesToProvider.mock.calls[0][0]).toBe(sent) + expect(messages).toEqual(originalMessages) + } + ) it('fails explicitly when file provenance lookup is unavailable', async () => { mockFilterModelSafeWorkspaceFileAttachments.mockRejectedValueOnce(new Error('db unavailable')) diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 5d1f5819a9d..c7e5f8b2eb0 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors' import { getApiKeyWithBYOK } from '@/lib/api-key/byok' import { env, envNumber } from '@/lib/core/config/env' import { filterModelSafeWorkspaceFileAttachments } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { appendUnavailableAttachmentNotice } from '@/lib/uploads/utils/model-input' import type { StreamingExecution } from '@/executor/types' import { applyModelCostPolicy, @@ -41,9 +42,7 @@ import { const logger = createLogger('Providers') -async function omitUnsafeProviderFileAttachments( - request: ProviderRequest -): Promise { +async function prepareProviderFileAttachments(request: ProviderRequest): Promise { const attachments = (request.messages ?? []).flatMap((message) => message.files ?? []) if (attachments.length === 0) return request @@ -72,7 +71,13 @@ async function omitUnsafeProviderFileAttachments( messages: request.messages?.map((message) => { if (!message.files) return message const files = message.files.filter((file) => safe.has(file)) - return { ...message, ...(files.length > 0 ? { files } : { files: undefined }) } + const omittedCount = message.files.length - files.length + if (omittedCount === 0) return message + return { + ...message, + content: appendUnavailableAttachmentNotice(message.content, omittedCount), + files: files.length > 0 ? files : undefined, + } }), } } @@ -236,8 +241,7 @@ export async function executeProviderRequest( sanitizedRequest.responseFormat = undefined } - const provenanceSafeRequest = await omitUnsafeProviderFileAttachments(sanitizedRequest) - const modelSafeRequest = provenanceSafeRequest + const modelSafeRequest = await prepareProviderFileAttachments(sanitizedRequest) const toolIdentities = assignProviderToolIdentities(modelSafeRequest.tools) const failedFunctionToolCost = { total: 0 } const requestRuntimeContext: ProviderRuntimeContext = { From 21291d7272fcf67f95054b4f9aaff7a1dfd5c31f Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Tue, 15 Sep 2026 17:14:26 -0700 Subject: [PATCH 14/15] fix(files): hydrate chat attachments and reject blank download URLs (#7870) * fix(files): hydrate chat attachments and reject blank download URLs * fix(files): preserve concealed canonical file failures --- .../message/components/file-download.test.tsx | 27 ++ .../message/components/file-download.tsx | 20 +- .../agent/memory-harness.postgres.test.ts | 72 ++++- .../payloads/file-secret-provenance.test.ts | 12 +- .../payloads/materialization.server.test.ts | 166 +++++++++- .../payloads/materialization.server.ts | 26 +- .../function-execution/sandbox-mounts.test.ts | 12 +- apps/sim/lib/internal/file/operations.test.ts | 48 +-- ...tored-workspace-file-record-by-key.test.ts | 284 ++++++++++++++++++ ...ead-stored-workspace-file-record-by-key.ts | 89 ++++++ .../file-attachments-authorization.test.ts | 37 ++- 11 files changed, 744 insertions(+), 49 deletions(-) create mode 100644 apps/sim/lib/workspace-files/application/read-stored-workspace-file-record-by-key.test.ts create mode 100644 apps/sim/lib/workspace-files/application/read-stored-workspace-file-record-by-key.ts diff --git a/apps/sim/app/(interfaces)/chat/components/message/components/file-download.test.tsx b/apps/sim/app/(interfaces)/chat/components/message/components/file-download.test.tsx index 5928480b4b8..0a1bf029446 100644 --- a/apps/sim/app/(interfaces)/chat/components/message/components/file-download.test.tsx +++ b/apps/sim/app/(interfaces)/chat/components/message/components/file-download.test.tsx @@ -85,6 +85,13 @@ describe('ChatFileDownload', () => { ) }) + it.each(['', ' \t\n'])('uses the serve route to preview files with a blank URL (%j)', (url) => { + const container = renderFile({ ...imageFile, base64: undefined, url }) + expect(container.querySelector('img')?.getAttribute('src')).toBe( + '/api/files/serve/execution%2Fgenerated.png?context=execution' + ) + }) + it('keeps a download available when an image preview fails', () => { const container = renderFile(imageFile) act(() => container.querySelector('img')!.dispatchEvent(new Event('error'))) @@ -154,6 +161,26 @@ describe('chat file downloads', () => { expect(downloadedNames).toEqual(['generated.png']) }) + it.each(['', ' \t\n'].flatMap((url) => [false, true].map((stored) => ({ url, stored }))))( + 'never downloads the chat page for a blank URL (%j)', + async ({ url, stored }) => { + if (stored) fetchMock.mockResolvedValueOnce(new Response(null, { status: 401 })) + fetchMock.mockResolvedValue(new Response('Chat page')) + const container = renderFile({ + ...imageFile, + base64: undefined, + key: stored ? imageFile.key : 'url/external', + url, + }) + await clickDownload(container) + expect(fetchMock).toHaveBeenCalledTimes(stored ? 1 : 0) + expect(createObjectURL).not.toHaveBeenCalled() + expect(downloadedNames).toEqual([]) + expect(container.querySelector('a')).toBeNull() + expect(container.querySelector('[role="alert"]')?.textContent).toContain('Unable to download') + } + ) + it.each([false, true])( 'offers a safe browser download when an external host blocks CORS (stored=%s)', async (stored) => { diff --git a/apps/sim/app/(interfaces)/chat/components/message/components/file-download.tsx b/apps/sim/app/(interfaces)/chat/components/message/components/file-download.tsx index 1da92cb8959..1a2c3b318ea 100644 --- a/apps/sim/app/(interfaces)/chat/components/message/components/file-download.tsx +++ b/apps/sim/app/(interfaces)/chat/components/message/components/file-download.tsx @@ -68,10 +68,17 @@ function isImageFile(mimeType: string): boolean { return mimeType.startsWith('image/') } +function getExternalFileUrl(file: ChatFile): string | null { + const url = file.url?.trim() + return url && isSafeHttpUrl(url) ? url : null +} + function getFileUrl(file: ChatFile): string { if (file.base64) return `data:${file.type};base64,${file.base64}` - if (isSafeHttpUrl(file.url)) return file.url - return `/api/files/serve/${encodeURIComponent(file.key)}?context=${file.context || 'execution'}` + return ( + getExternalFileUrl(file) ?? + `/api/files/serve/${encodeURIComponent(file.key)}?context=${file.context || 'execution'}` + ) } async function triggerDownload(file: ChatFile): Promise { @@ -88,11 +95,10 @@ async function triggerDownload(file: ChatFile): Promise { const storageContext = tryInferContextFromKey(file.key) const hasStorageKey = storageContext !== null + const externalUrl = getExternalFileUrl(file) const url = hasStorageKey ? `/api/files/serve/${encodeURIComponent(file.key)}?context=${encodeURIComponent(storageContext)}` - : isSafeHttpUrl(file.url) - ? file.url - : null + : externalUrl if (!url) throw new Error('File has no download URL') /** The same serve route as execution logs resolves current storage access on each click. */ @@ -103,10 +109,10 @@ async function triggerDownload(file: ChatFile): Promise { } else { response = await fetchExternalFile(url) } - if (hasStorageKey && response.status === 401 && isSafeHttpUrl(file.url)) { + if (hasStorageKey && response.status === 401 && externalUrl) { await response.body?.cancel() /** Public chat visitors may only have the file access already delivered in the response. */ - response = await fetchExternalFile(file.url) + response = await fetchExternalFile(externalUrl) } if (!response.ok) { await response.body?.cancel() diff --git a/apps/sim/executor/handlers/agent/memory-harness.postgres.test.ts b/apps/sim/executor/handlers/agent/memory-harness.postgres.test.ts index b13d1b7f773..a55c6f45dfa 100644 --- a/apps/sim/executor/handlers/agent/memory-harness.postgres.test.ts +++ b/apps/sim/executor/handlers/agent/memory-harness.postgres.test.ts @@ -72,6 +72,7 @@ import { workspaceFiles, } from '@sim/db/schema' import { hashDurableSecretProvenanceValue } from '@/lib/execution/durable-secret-provenance' +import { StorageService } from '@/lib/uploads' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution/execution-file-manager' import { EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, @@ -84,7 +85,13 @@ import { AgentBlockHandler } from '@/executor/handlers/agent/agent-handler' import type { AgentInputs, Message } from '@/executor/handlers/agent/types' import type { ExecutionContext, StreamingExecution, UserFile } from '@/executor/types' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { INLINE_ATTACHMENT_THRESHOLD_BYTES } from '@/providers/attachments' +import { + attachLargeFileRemoteUrls, + uploadLargeFilesToProvider, +} from '@/providers/file-attachments.server' import { createAgentStreamPump } from '@/providers/stream-pump' +import type { ProviderRequest } from '@/providers/types' import type { SerializedBlock } from '@/serializer/types' const databaseUrl = process.env.AGENT_MEMORY_TEST_DATABASE_URL @@ -382,12 +389,14 @@ describe.skipIf(!databaseUrl)( }) it.each( - (['openai', 'anthropic'] as const).flatMap((provider) => - [false, true].map((streaming) => ({ provider, streaming })) + (['workspace', 'mothership'] as const).flatMap((storageContext) => + (['openai', 'anthropic'] as const).flatMap((provider) => + [false, true].map((streaming) => ({ storageContext, provider, streaming })) + ) ) )( - 'deployed chat reads remembered workspace files with $provider, streaming=$streaming', - async ({ provider, streaming }) => { + 'deployed chat reads remembered $storageContext files with $provider, streaming=$streaming', + async ({ storageContext, provider, streaming }) => { if (!fixture.database || !connection) throw new Error('Missing harness database') vi.stubGlobal('fetch', interceptFetch) outbound = [] @@ -414,7 +423,7 @@ describe.skipIf(!databaseUrl)( key, userId: scope.userId, workspaceId: scope.workspaceId, - context: 'workspace', + context: storageContext, originalName: 'result.pdf', contentType: 'application/pdf', size: buffer.length, @@ -487,6 +496,57 @@ describe.skipIf(!databaseUrl)( input: { key, assertedWorkspaceId: scope.workspaceId }, }) ).rejects.toThrow('Principal kind system') + + const strictWorkspaceRead = readWorkspaceFileRecordByKey.execute({ + principal: { + kind: 'workspace_api_key', + workspaceId: scope.workspaceId, + keyId: 'harness-key', + }, + input: { key, assertedWorkspaceId: scope.workspaceId }, + }) + if (storageContext === 'mothership') { + await expect(strictWorkspaceRead).rejects.toMatchObject({ code: 'not_found' }) + } else { + await expect(strictWorkspaceRead).resolves.toMatchObject({ file: { id: record.id } }) + } + + /** Exercise large-file authorization with real metadata and delegation before model dispatch. */ + const largeFile = { ...file, size: INLINE_ATTACHMENT_THRESHOLD_BYTES + 1 } + const largeRequest: ProviderRequest = { + model: models[provider], + apiKey: apiKey(provider), + userId: scope.userId, + messages: [{ role: 'user', content: 'Read the attachment', files: [largeFile] }], + } + const cloudStorage = vi.spyOn(StorageService, 'hasCloudStorage').mockReturnValue(true) + const presign = vi + .spyOn(StorageService, 'generatePresignedDownloadUrl') + .mockResolvedValue('https://storage.example.com/signed') + try { + await attachLargeFileRemoteUrls(largeRequest, provider, firstContext) + expect(presign).toHaveBeenCalledWith(key, 'workspace', 3600) + expect(largeFile.remoteUrl).toBe('https://storage.example.com/signed') + if (provider === 'openai') { + const upload = vi.fn(async (url: string, init?: RequestInit) => { + expect(url).toBe('https://api.openai.com/v1/files') + expect(init?.body).toBeInstanceOf(FormData) + const body = init!.body as FormData + const uploaded = body.get('file') as File + expect(Buffer.from(await uploaded.arrayBuffer())).toEqual(buffer) + return Response.json({ id: 'file-harness' }) + }) + vi.stubGlobal('fetch', upload) + await uploadLargeFilesToProvider(largeRequest, provider, firstContext) + expect(upload).toHaveBeenCalledOnce() + expect(largeFile.providerFileId).toBe('file-harness') + } + } finally { + cloudStorage.mockRestore() + presign.mockRestore() + vi.stubGlobal('fetch', interceptFetch) + } + expect(await executeTurn(firstContext, { ...inputs, files: [file] })).toBe('READY') expect(requestFiles(outbound[0])).toEqual([buffer.toString('base64')]) const stored = await readConversation(conversationId) @@ -529,7 +589,7 @@ describe.skipIf(!databaseUrl)( report.push({ provider, streaming, - workspaceAttachment: true, + storageContext, stored, controls: { missingOrigin: 'blocked before HTTP', diff --git a/apps/sim/lib/execution/payloads/file-secret-provenance.test.ts b/apps/sim/lib/execution/payloads/file-secret-provenance.test.ts index 0afa418d05b..046c1f7cb83 100644 --- a/apps/sim/lib/execution/payloads/file-secret-provenance.test.ts +++ b/apps/sim/lib/execution/payloads/file-secret-provenance.test.ts @@ -7,9 +7,15 @@ const { metadata, readWorkspaceFile } = vi.hoisted(() => ({ })) vi.mock('@/lib/uploads/server/metadata', () => ({ getFileMetadataByKey: metadata })) -vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', () => ({ - readWorkspaceFileRecordByKey: { execute: readWorkspaceFile }, -})) +vi.mock( + '@/lib/workspace-files/application/read-stored-workspace-file-record-by-key', + async (importOriginal) => ({ + ...(await importOriginal< + typeof import('@/lib/workspace-files/application/read-stored-workspace-file-record-by-key') + >()), + readStoredWorkspaceFileRecordByKey: { execute: readWorkspaceFile }, + }) +) import { resolveStoredFileProvenanceSource } from '@/lib/execution/payloads/file-secret-provenance' diff --git a/apps/sim/lib/execution/payloads/materialization.server.test.ts b/apps/sim/lib/execution/payloads/materialization.server.test.ts index c7208738e04..42a5df4c621 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.test.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.test.ts @@ -19,14 +19,23 @@ vi.mock('@/app/api/files/authorization', () => ({ verifyFileAccess: mockVerifyFileAccess, })) -vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', () => ({ - readWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileByKey }, -})) +vi.mock( + '@/lib/workspace-files/application/read-stored-workspace-file-record-by-key', + async (importOriginal) => ({ + ...(await importOriginal< + typeof import('@/lib/workspace-files/application/read-stored-workspace-file-record-by-key') + >()), + readStoredWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileByKey }, + }) +) +import { OrchestrationError } from '@/lib/core/orchestration/types' import { + assertUserFileContentAccess, readUserFileContent, readUserFileContentWithContributors, } from '@/lib/execution/payloads/materialization.server' +import { StoredWorkspaceFileUnavailableError } from '@/lib/workspace-files/application/read-stored-workspace-file-record-by-key' import type { UserFile } from '@/executor/types' const PDF_SOURCE = Buffer.from('from reportlab.pdfgen import canvas') @@ -41,6 +50,17 @@ const generatedPdf: UserFile = { key: 'workspace/2f1d8c3e-5b6a-4c7d-8e9f-0a1b2c3d4e5f/1700000000000-abc1234-report.pdf', } +const delegatedReader = { + kind: 'delegated' as const, + serviceId: 'copilot' as const, + subjectUserId: 'reader', + workspaceId: 'workspace-1', + delegationId: 'read-1', + audience: 'sim:function-executions', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), +} + describe('readUserFileContent', () => { beforeEach(() => { vi.clearAllMocks() @@ -264,4 +284,144 @@ describe('readUserFileContent', () => { }, }) }) + + it.each([undefined, 'workspace', 'mothership'] as const)( + 'resolves chat-upload ownership canonically with descriptor context %s', + async (context) => { + const principal = { + kind: 'workspace_api_key' as const, + workspaceId: 'workspace-1', + keyId: 'key-1', + } + await assertUserFileContentAccess( + { key: 'workspace/workspace-1/upload.png', context }, + { principal, workspaceId: 'workspace-1' } + ) + + expect(mockReadWorkspaceFileByKey).toHaveBeenCalledWith({ + principal, + input: { + key: 'workspace/workspace-1/upload.png', + assertedWorkspaceId: 'workspace-1', + }, + }) + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + } + ) + + it.each([ + { userId: 'billing-owner' }, + { + userId: 'billing-owner', + workspaceId: 'workspace-1', + principal: { kind: 'workspace_api_key' as const, workspaceId: 'workspace-1', keyId: 'key-1' }, + }, + ])('never uses a user fallback to accept a mothership context alias', async (options) => { + mockReadWorkspaceFileByKey.mockRejectedValue( + new OrchestrationError('not_found', 'File not found') + ) + + await expect( + assertUserFileContentAccess( + { key: 'workspace/workspace-1/upload.png', context: 'mothership' }, + options + ) + ).rejects.toThrow('Chat upload access requires canonical file authorization.') + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + }) + + it.each( + ([undefined, 'workspace', 'mothership'] as const).flatMap((context) => + (['forbidden', 'not_found'] as const).flatMap((code) => + [{ fileId: 'file-1' }, { chatId: 'chat-1' }, { fileId: 'file-1', chatId: 'chat-1' }].map( + (resourceScope) => ({ context, code, resourceScope }) + ) + ) + ) + )( + 'preserves delegated file/chat limits after canonical rejection: %j', + async ({ context, code, resourceScope }) => { + const principal = { ...delegatedReader, resourceScope } + mockReadWorkspaceFileByKey.mockRejectedValue(new OrchestrationError(code, 'Denied')) + + await expect( + assertUserFileContentAccess( + { key: 'workspace/workspace-1/upload.png', context }, + { principal, workspaceId: 'workspace-1', userId: 'billing-owner' } + ) + ).rejects.toMatchObject({ code }) + expect(mockReadWorkspaceFileByKey).toHaveBeenCalledWith( + expect.objectContaining({ + principal: expect.objectContaining({ resourceScope: principal.resourceScope }), + }) + ) + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + } + ) + + it.each( + ([undefined, 'workspace'] as const).flatMap((context) => + [{ kind: 'session' as const, userId: 'reader', sessionId: 'session-1' }, delegatedReader].map( + (principal) => ({ context, principal }) + ) + ) + )( + 'retains legacy authorization for an unscoped caller with absent metadata: %j', + async ({ context, principal }) => { + mockReadWorkspaceFileByKey.mockRejectedValue( + new OrchestrationError('not_found', 'File not found') + ) + await expect( + assertUserFileContentAccess( + { key: 'workspace/workspace-1/legacy.png', context }, + { principal, workspaceId: 'workspace-1', userId: 'reader' } + ) + ).resolves.toBeUndefined() + expect(mockVerifyFileAccess).toHaveBeenCalledWith( + 'workspace/workspace-1/legacy.png', + 'reader', + undefined, + 'workspace', + false, + { knowledgeAccess: undefined } + ) + } + ) + + it.each( + ([undefined, 'workspace', 'mothership'] as const).flatMap((context) => + [ + { kind: 'session' as const, userId: 'reader', sessionId: 'session-1' }, + delegatedReader, + { ...delegatedReader, resourceScope: { fileId: 'file-1', chatId: 'chat-1' } }, + ].map((principal) => ({ context, principal })) + ) + )( + 'never falls back or reads bytes for a known unavailable binding: %j', + async ({ context, principal }) => { + const unavailable = new StoredWorkspaceFileUnavailableError() + mockReadWorkspaceFileByKey.mockRejectedValue(unavailable) + await expect( + readUserFileContent( + { ...generatedPdf, key: 'workspace/workspace-1/upload.png', context }, + { principal, workspaceId: 'workspace-1', userId: 'reader', encoding: 'base64' } + ) + ).rejects.toBe(unavailable) + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + } + ) + + it.each([ + 'execution/workspace-1/workflow-1/run-1/file.png', + 'profile-pictures/file.png', + 'assistant/org-1/file.png', + ])('does not extend the mothership alias to %s', async (key) => { + await expect( + assertUserFileContentAccess({ key, context: 'mothership' }, { workspaceId: 'workspace-1' }) + ).rejects.toThrow() + expect(mockReadWorkspaceFileByKey).not.toHaveBeenCalled() + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/execution/payloads/materialization.server.ts b/apps/sim/lib/execution/payloads/materialization.server.ts index b0e6eb3483f..8e8035a90f7 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.ts @@ -29,7 +29,10 @@ import { } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { rebindWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' -import { readWorkspaceFileRecordByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' +import { + readStoredWorkspaceFileRecordByKey, + StoredWorkspaceFileUnavailableError, +} from '@/lib/workspace-files/application/read-stored-workspace-file-record-by-key' import type { UserFile } from '@/executor/types' const logger = createLogger('ExecutionPayloadMaterialization') @@ -266,7 +269,11 @@ function getVerifiedStorageContext(file: Pick): Sto } const inferredContext = inferContextFromKey(file.key) - if (file.context && file.context !== inferredContext) { + if ( + file.context && + file.context !== inferredContext && + !(inferredContext === 'workspace' && file.context === 'mothership') + ) { throw new Error('File context does not match its storage key.') } @@ -305,7 +312,7 @@ export async function assertUserFileContentAccess( }) : options.principal try { - await readWorkspaceFileRecordByKey.execute({ + await readStoredWorkspaceFileRecordByKey.execute({ principal, input: { key: file.key, @@ -315,9 +322,22 @@ export async function assertUserFileContentAccess( return } catch (error) { if (!(error instanceof OrchestrationError && error.code === 'not_found')) throw error + if (error instanceof StoredWorkspaceFileUnavailableError) throw error + /** Legacy storage metadata cannot prove a delegated file or chat identity. */ + if ( + options.principal.kind === 'delegated' && + (options.principal.resourceScope?.fileId !== undefined || + options.principal.resourceScope?.chatId !== undefined) + ) { + throw error + } } } + if (context === 'workspace' && file.context === 'mothership') { + throw new Error('Chat upload access requires canonical file authorization.') + } + if (!options.userId) { throw new Error('File access requires an authenticated user.') } diff --git a/apps/sim/lib/function-execution/sandbox-mounts.test.ts b/apps/sim/lib/function-execution/sandbox-mounts.test.ts index 6557a35cb0d..f988bd4ef2f 100644 --- a/apps/sim/lib/function-execution/sandbox-mounts.test.ts +++ b/apps/sim/lib/function-execution/sandbox-mounts.test.ts @@ -32,9 +32,15 @@ vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadServableFileFromStorage: mockDownloadServableFileFromStorage, })) -vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', () => ({ - readWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileRecordByKey }, -})) +vi.mock( + '@/lib/workspace-files/application/read-stored-workspace-file-record-by-key', + async (importOriginal) => ({ + ...(await importOriginal< + typeof import('@/lib/workspace-files/application/read-stored-workspace-file-record-by-key') + >()), + readStoredWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileRecordByKey }, + }) +) vi.mock('@/lib/uploads/server/metadata', () => ({ getFileMetadataByKey: mockGetFileMetadataByKey, diff --git a/apps/sim/lib/internal/file/operations.test.ts b/apps/sim/lib/internal/file/operations.test.ts index 3cffe984456..8230dcc4aec 100644 --- a/apps/sim/lib/internal/file/operations.test.ts +++ b/apps/sim/lib/internal/file/operations.test.ts @@ -1019,6 +1019,7 @@ describe('file manage folder wiring', () => { describe('file manage operations', () => { beforeEach(() => { vi.clearAllMocks() + mockGetFileMetadataByKey.mockResolvedValue(null) hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ success: true, userId: 'user-1', @@ -1150,16 +1151,31 @@ describe('file manage operations', () => { }, ] - beforeEach(() => { - mockResolveWorkspaceFileReference.mockResolvedValue(workspaceFile('document')) - mockGetFileMetadataByKey.mockResolvedValue({ - id: contributor.fileId, - key: contributor.key, - context: contributor.context, + function mockContributorMetadata(overrides: { userId?: string; workspaceId?: string } = {}) { + const document = { + id: 'document', + key: 'workspace/workspace-1/document.txt', + context: 'workspace', workspaceId: 'workspace-1', userId: 'user-1', contentUpdatedAt: CONTENT_UPDATED_AT, - }) + } + const image = { + ...document, + id: contributor.fileId, + key: contributor.key, + ...overrides, + } + const records = new Map([ + [document.key, document], + [image.key, image], + ]) + mockGetFileMetadataByKey.mockImplementation(async (key: string) => records.get(key) ?? null) + } + + beforeEach(() => { + mockResolveWorkspaceFileReference.mockResolvedValue(workspaceFile('document')) + mockContributorMetadata() mockDownloadServableFileFromStorage.mockResolvedValue({ buffer: Buffer.from('rendered image content'), contentType: 'text/plain', @@ -1301,14 +1317,7 @@ describe('file manage operations', () => { it.each(['write', 'compress'] as const)( '%s retains the secret owner guard for rendered contributors', async (operation) => { - mockGetFileMetadataByKey.mockResolvedValue({ - id: contributor.fileId, - key: contributor.key, - context: contributor.context, - workspaceId: 'workspace-1', - userId: 'other-user', - contentUpdatedAt: CONTENT_UPDATED_AT, - }) + mockContributorMetadata({ userId: 'other-user' }) mockGetBoundWorkspaceFileSecretProvenance.mockImplementation( async (_workspaceId: string, identity: { fileId: string }) => ({ status: 'exact', @@ -1326,14 +1335,7 @@ describe('file manage operations', () => { ) it('refuses a rendered contributor whose canonical scope differs', async () => { - mockGetFileMetadataByKey.mockResolvedValue({ - id: contributor.fileId, - key: contributor.key, - context: contributor.context, - workspaceId: 'other-workspace', - userId: 'user-1', - contentUpdatedAt: CONTENT_UPDATED_AT, - }) + mockContributorMetadata({ workspaceId: 'other-workspace' }) mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ status: 'exact', entries: [] }) const response = await POST(renderedRequest('write')) diff --git a/apps/sim/lib/workspace-files/application/read-stored-workspace-file-record-by-key.test.ts b/apps/sim/lib/workspace-files/application/read-stored-workspace-file-record-by-key.test.ts new file mode 100644 index 00000000000..8f67639c8d4 --- /dev/null +++ b/apps/sim/lib/workspace-files/application/read-stored-workspace-file-record-by-key.test.ts @@ -0,0 +1,284 @@ +/** @vitest-environment node */ +import type { DelegatedPrincipal, Principal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + metadata: vi.fn(), + workspace: vi.fn(), + permission: vi.fn(), +})) + +vi.mock('@/lib/uploads/server/metadata', () => ({ getFileMetadataByKey: mocks.metadata })) +vi.mock('@/lib/uploads/contexts/workspace', () => ({ loadActiveWorkspaceContext: mocks.workspace })) +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (permission: string) => ['read', 'write', 'admin'].includes(permission), + resolveEffectiveWorkspacePermission: mocks.permission, +})) + +import { + readStoredWorkspaceFileRecordByKey, + StoredWorkspaceFileUnavailableError, +} from '@/lib/workspace-files/application/read-stored-workspace-file-record-by-key' + +const input = { + key: 'workspace/workspace-1/upload.png', + assertedWorkspaceId: 'workspace-1', +} +const workspace = { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner', +} +const file = { + id: 'file-1', + key: input.key, + context: 'mothership', + workspaceId: 'workspace-1', + organizationId: null, + chatId: 'chat-1', + deletedAt: null, + userId: 'uploader', +} +const session = { kind: 'session', userId: 'reader', sessionId: 'session-1' } as const + +function executor(overrides: Partial = {}): DelegatedPrincipal { + return { + kind: 'delegated', + serviceId: 'executor', + workspaceId: workspace.workspaceId, + delegationId: 'delegation-1', + audience: 'sim:workspace-files', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system', + serviceId: 'chat', + workspaceId: workspace.workspaceId, + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + }, + ...overrides, + } +} + +describe('readStoredWorkspaceFileRecordByKey', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.metadata.mockResolvedValue(file) + mocks.workspace.mockResolvedValue(workspace) + mocks.permission.mockResolvedValue('read') + }) + + it.each(['workspace', 'mothership'] as const)( + 'authorizes canonical %s bytes with the actual current workspace member', + async (context) => { + const record = { ...file, context } + mocks.metadata.mockResolvedValue(record) + + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal: session, input }) + ).resolves.toEqual({ file: record }) + expect(mocks.metadata).toHaveBeenCalledWith(input.key) + expect(mocks.metadata).toHaveBeenNthCalledWith(1, input.key, undefined, { + includeDeleted: true, + }) + expect(mocks.metadata).toHaveBeenCalledTimes(2) + expect(mocks.permission).toHaveBeenCalledWith('reader', 'workspace-1', null, undefined, { + forUpdate: undefined, + }) + } + ) + + it.each([ + { kind: 'personal_api_key', userId: 'reader', keyId: 'key-1' }, + { kind: 'workspace_api_key', workspaceId: 'workspace-1', keyId: 'key-1' }, + ])( + 'preserves the $kind authority without substituting the uploader or billing owner', + async (principal) => { + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal, input }) + ).resolves.toEqual({ + file, + }) + if (principal.kind === 'personal_api_key') { + expect(mocks.permission).toHaveBeenCalledWith('reader', 'workspace-1', null, undefined, { + forUpdate: undefined, + }) + } else { + expect(mocks.permission).not.toHaveBeenCalled() + } + } + ) + + it('admits a deployment executor through workspace authority without consulting a human', async () => { + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal: executor(), input }) + ).resolves.toEqual({ file }) + expect(mocks.permission).not.toHaveBeenCalled() + }) + + it.each([{ fileId: 'file-1' }, { chatId: 'chat-1' }, { fileId: 'file-1', chatId: 'chat-1' }])( + 'retains a matching narrower delegation: %j', + async (resourceScope) => { + await expect( + readStoredWorkspaceFileRecordByKey.execute({ + principal: executor({ resourceScope }), + input, + }) + ).resolves.toEqual({ file }) + } + ) + + it('keeps workspace files available to a chat-scoped delegate', async () => { + mocks.metadata.mockResolvedValue({ ...file, context: 'workspace', chatId: null }) + await expect( + readStoredWorkspaceFileRecordByKey.execute({ + principal: executor({ resourceScope: { chatId: 'chat-1' } }), + input, + }) + ).resolves.toMatchObject({ file: { context: 'workspace' } }) + }) + + it.each>([ + { resourceScope: { fileId: 'other-file' } }, + { resourceScope: { chatId: 'other-chat' } }, + { resourceScope: { fileId: 'file-1', chatId: 'other-chat' } }, + { workspaceId: 'other-workspace' }, + { audience: 'other-audience' }, + { expiresAt: new Date(0) }, + { delegationContext: undefined }, + ])('rejects invalid delegation before reading content metadata: %j', async (overrides) => { + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal: executor(overrides), input }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.metadata).toHaveBeenCalledTimes(1) + expect(mocks.permission).not.toHaveBeenCalled() + }) + + it('does not broaden a chat-scoped delegation when an attachment has no chat binding', async () => { + mocks.metadata.mockResolvedValue({ ...file, chatId: null }) + await expect( + readStoredWorkspaceFileRecordByKey.execute({ + principal: executor({ resourceScope: { chatId: 'chat-1' } }), + input, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + }) + + it('rechecks the real human behind an executor and denies revoked membership', async () => { + mocks.permission.mockResolvedValue(null) + const principal = executor({ + subjectUserId: 'reader', + delegationContext: { + ...executor().delegationContext!, + principal: session, + }, + }) + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal, input }) + ).rejects.toMatchObject({ + code: 'forbidden', + }) + expect(mocks.permission).toHaveBeenCalledWith('reader', 'workspace-1', null, undefined, { + forUpdate: undefined, + }) + expect(mocks.metadata).toHaveBeenCalledTimes(1) + }) + + it('rejects raw system identity before loading metadata', async () => { + await expect( + readStoredWorkspaceFileRecordByKey.execute({ + principal: { + kind: 'system', + serviceId: 'chat', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + input, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.metadata).not.toHaveBeenCalled() + }) + + it('keeps absent initial metadata distinguishable for legacy access', async () => { + mocks.metadata.mockResolvedValue(null) + const read = readStoredWorkspaceFileRecordByKey.execute({ principal: session, input }) + await expect(read).rejects.toMatchObject({ code: 'not_found' }) + await expect(read).rejects.not.toBeInstanceOf(StoredWorkspaceFileUnavailableError) + expect(mocks.workspace).not.toHaveBeenCalled() + expect(mocks.permission).not.toHaveBeenCalled() + }) + + it('conceals canonical unavailability as not_found', () => { + expect(new StoredWorkspaceFileUnavailableError()).toMatchObject({ + code: 'not_found', + message: 'File not found', + }) + }) + + it.each([ + { ...file, deletedAt: new Date() }, + { ...file, workspaceId: 'other-workspace' }, + { ...file, workspaceId: null }, + { ...file, organizationId: 'organization-1' }, + { ...file, context: 'execution' }, + { ...file, context: 'profile-pictures' }, + { ...file, key: 'workspace/workspace-1/different.png' }, + ])('refuses legacy fallback for invalid canonical metadata: %j', async (record) => { + mocks.metadata.mockResolvedValue(record) + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal: session, input }) + ).rejects.toBeInstanceOf(StoredWorkspaceFileUnavailableError) + expect(mocks.workspace).not.toHaveBeenCalled() + expect(mocks.permission).not.toHaveBeenCalled() + }) + + it.each([null, { ...workspace, workspaceId: 'other-workspace' }])( + 'rejects an inactive or changed workspace: %j', + async (record) => { + mocks.workspace.mockResolvedValue(record) + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal: session, input }) + ).rejects.toBeInstanceOf(StoredWorkspaceFileUnavailableError) + expect(mocks.permission).not.toHaveBeenCalled() + } + ) + + it.each([ + null, + { ...file, id: 'replacement-file' }, + { ...file, key: 'workspace/workspace-1/new.png' }, + { ...file, deletedAt: new Date() }, + { ...file, workspaceId: 'other-workspace' }, + { ...file, context: 'workspace' }, + { ...file, chatId: 'other-chat' }, + ])('rejects an identity changed during authorization: %j', async (record) => { + mocks.metadata.mockResolvedValueOnce(file).mockResolvedValueOnce(record) + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal: session, input }) + ).rejects.toBeInstanceOf(StoredWorkspaceFileUnavailableError) + expect(mocks.permission).toHaveBeenCalledOnce() + }) + + it.each([ + '', + 'legacy-unprefixed.png', + 'assistant/org-1/file.png', + 'execution/workspace-1/file.png', + ])('rejects a storage key outside the shared workspace bucket: %s', async (key) => { + await expect( + readStoredWorkspaceFileRecordByKey.execute({ principal: session, input: { ...input, key } }) + ).rejects.toMatchObject({ code: 'not_found' }) + expect(mocks.metadata).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/workspace-files/application/read-stored-workspace-file-record-by-key.ts b/apps/sim/lib/workspace-files/application/read-stored-workspace-file-record-by-key.ts new file mode 100644 index 00000000000..acdb813194e --- /dev/null +++ b/apps/sim/lib/workspace-files/application/read-stored-workspace-file-record-by-key.ts @@ -0,0 +1,89 @@ +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + type ActiveWorkspaceContext, + loadActiveWorkspaceContext, +} from '@/lib/uploads/contexts/workspace' +import { type FileMetadataRecord, getFileMetadataByKey } from '@/lib/uploads/server/metadata' +import { isWorkspaceScopedContext } from '@/lib/uploads/shared/types' +import { tryInferContextFromKey } from '@/lib/uploads/utils/file-utils' +import { workspaceFileDelegationPolicy } from '@/lib/workspace-files/application/authorization' +import { fileOperations } from '@/lib/workspace-files/application/operations' + +interface ReadStoredWorkspaceFileByKeyInput { + key: string + assertedWorkspaceId: string +} + +interface StoredWorkspaceFileContext extends ActiveWorkspaceContext { + fileId: string + file: FileMetadataRecord +} + +/** Conceals a known unavailable binding without permitting legacy storage authorization. */ +export class StoredWorkspaceFileUnavailableError extends OrchestrationError { + constructor() { + super('not_found', 'File not found') + this.name = 'StoredWorkspaceFileUnavailableError' + } +} + +/** + * Authorizes stored bytes under the shared workspace tenancy of files and chat uploads. + * Canonical metadata determines ownership; workspace-file CRUD remains workspace-only. + */ +export const readStoredWorkspaceFileRecordByKey = defineAuthorizedWorkspaceUseCase({ + operation: fileOperations.readContent, + async resolveContext({ + input, + }: { + input: ReadStoredWorkspaceFileByKeyInput + }): Promise { + if (tryInferContextFromKey(input.key) !== 'workspace') { + throw new OrchestrationError('not_found', 'File not found') + } + const file = await getFileMetadataByKey(input.key, undefined, { includeDeleted: true }) + if (!file) throw new OrchestrationError('not_found', 'File not found') + if ( + !file.workspaceId || + file.deletedAt || + file.organizationId || + file.key !== input.key || + !isWorkspaceScopedContext(file.context) || + file.workspaceId !== input.assertedWorkspaceId + ) { + throw new StoredWorkspaceFileUnavailableError() + } + const workspace = await loadActiveWorkspaceContext(file.workspaceId) + if (!workspace || workspace.workspaceId !== file.workspaceId) { + throw new StoredWorkspaceFileUnavailableError() + } + return { ...workspace, fileId: file.id, file } + }, + authorizationOptions: { + delegation: { + audience: workspaceFileDelegationPolicy.audience, + isWithinScope: (principal, context) => + workspaceFileDelegationPolicy.isWithinScope(principal, context) && + (context.file.context !== 'mothership' || + principal.resourceScope?.chatId === undefined || + principal.resourceScope.chatId === context.file.chatId), + }, + }, + async execute({ input, context }): Promise<{ file: FileMetadataRecord }> { + const file = await getFileMetadataByKey(input.key) + if ( + !file || + file.deletedAt || + file.organizationId || + file.id !== context.fileId || + file.key !== input.key || + file.workspaceId !== context.workspaceId || + file.context !== context.file.context || + file.chatId !== context.file.chatId + ) { + throw new StoredWorkspaceFileUnavailableError() + } + return { file } + }, +}) diff --git a/apps/sim/providers/file-attachments-authorization.test.ts b/apps/sim/providers/file-attachments-authorization.test.ts index 78e8260fcc9..ebbfd6e45e2 100644 --- a/apps/sim/providers/file-attachments-authorization.test.ts +++ b/apps/sim/providers/file-attachments-authorization.test.ts @@ -38,7 +38,7 @@ import type { ProviderRequest } from '@/providers/types' /** Authorization and key inference are real: mocking either hid this pre-existing refusal. */ describe('provider attachment storage-key authorization', () => { beforeEach(() => { - vi.clearAllMocks() + vi.resetAllMocks() }) it.each( @@ -94,4 +94,39 @@ describe('provider attachment storage-key authorization', () => { expect(permission).not.toHaveBeenCalled() } ) + + it('allows standalone provider reads of authorized mothership attachments in workspace storage', async () => { + const file: UserFile = { + id: 'attachment-1', + name: 'document.pdf', + key: 'workspace/workspace-1/attachment-1/document.pdf', + url: '', + size: 10 * 1024 * 1024, + type: 'application/pdf', + context: 'workspace', + } + metadata.mockResolvedValue({ + id: file.id, + key: file.key, + workspaceId: 'workspace-1', + userId: 'uploader', + context: 'mothership', + deletedAt: null, + }) + permission.mockResolvedValue('read') + presign.mockResolvedValue('https://storage.example.com/signed') + + await attachLargeFileRemoteUrls( + { + model: 'gpt-4.1', + userId: 'reader', + messages: [{ role: 'user', content: 'Read the attachment', files: [file] }], + }, + 'openai' + ) + + expect(permission).toHaveBeenCalledWith('reader', 'workspace', 'workspace-1') + expect(presign).toHaveBeenCalledWith(file.key, 'workspace', 3600) + expect(file.remoteUrl).toBe('https://storage.example.com/signed') + }) }) From f4845ccefbcbce0b8e39c4f56df2f9d6b3779363 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:18:04 -0700 Subject: [PATCH 15/15] fix(tables): rework lock settings as Table Security and gate locked actions (#7853) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tables): block schema-locked column edits and rework lock settings as Table Security * fix(tables): read-only cell editors and add-row form for update-locked tables - Update-locked tables open cell editors read-only; the expanded editor disables Save with the reason on hover, and empty cells that can't be edited open nothing - New row opens the Add Row form when only updates are locked - Row form reads and writes values by column id and uses one date-and-time picker - ChipDatePicker follows InsideModalContext so its calendar is clickable in modals, and gains showTime/timeLabel in single mode * fix(tables): open the add-row form for required columns and gate Save on required fields - New row, Shift+Enter, and Insert row open the Add Row form at their position when updates are locked or any column is required - Add Row and Update Row stay disabled until every required field has a value - Add mode accepts an insert position; Shift+Enter anchors to the neighbor row id * fix(tables): explain denied actions where the user meets them Clicking a column header opened the full column editor on a schema-locked table: every field was editable and Save only failed once the server refused it. The header click is the primary way into that panel, so it now opens read-only — values stay readable and selectable, a disabled `

` makes the controls inert, and Save carries the lock reason. Clicking a checkbox cell on an update-locked table did nothing at all, while the keyboard paths explained themselves; it now raises the same notice. The column menu disabled only "Edit column" while "Insert column left/right" and "Delete column" stayed live and explained the lock after the click. All four are disabled now, each with a tooltip. A disabled `DropdownMenuItem` sets `pointer-events: none`, so the tooltip wraps the row rather than the item. "Hide column" is untouched: hiding a workflow output is a metadata change no lock covers. Notices now speak the modal's Allow/Deny vocabulary and name the row that denies the action, and the tooltip strings live in `lock-copy` instead of being written out at each call site. Drops the "This table is append-only" copy, which no path could reach once New row and Shift+Enter started opening the add-row form. Co-Authored-By: Claude Opus 5 * fix(tables): write only what the row form changed, and report failures once The add/edit row form sent every column on every save. An untouched empty column was written as `null`, so a no-op edit still bumped the row, and an insert filled in nulls for columns the user never opened. It now sends only the fields the user touched, and in edit mode only those whose value actually differs — a save with nothing changed closes without a write. Checkboxes stay the exception on insert: they always carry a concrete boolean, so a required one the user never clicked still reaches the server as `false`. A rejected write also arrived twice: the modal rendered the message inline and the mutation toasted the same sentence. Row mutations take an opt-in `suppressErrorToast` so the form owns its own failure; the cache self-heal on a 423 still runs, only its toast is dropped. Co-Authored-By: Claude Opus 5 * refactor(tables): drop the Table Security switch and its device-local store "Enable Table Security" had no server representation: enabled-with-everything- allowed and never-configured both save four `false` flags, so the difference lived only in the browser that set it. Anyone else — another device, another admin — saw the table as unconfigured, and the per-action choices behind the switch were remembered per device too. The modal now always shows the four Allow/Deny rows, mapped one-to-one onto the server flags, so an unconfigured table opens on four `Allow`s and every viewer sees the same state. That removes the reason for the preference store, which is deleted along with its helpers and test. Stale `table-security-preferences` keys are left where they are; nothing reads them. Co-Authored-By: Claude Opus 5 * fix(tables): stage only the Table Security rows an admin moves The modal reset its draft only when it opened, so a lock another admin changed while it sat open went stale behind it: the controls kept rendering the old values and Save submitted all four flags, overwriting the newer state. Only the rows this admin moves are staged now. Every other row keeps rendering the authoritative value, so a concurrent change shows up in the open modal instead of hiding behind it, and Save sends just that patch — the route already takes a partial — so an untouched row can't carry a stale flag over someone else's change. A row both admins moved is the one real conflict, and there the explicit choice wins. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- .../column-config-sidebar.tsx | 243 +++++++++------ .../column-dropdown/column-dropdown.test.tsx | 31 +- .../column-dropdown/column-dropdown.tsx | 44 +-- .../components/context-menu/context-menu.tsx | 3 +- .../lock-settings-modal.test.tsx | 148 +++++++++ .../lock-settings-modal.tsx | 127 ++++---- .../components/row-modal/row-modal.test.tsx | 291 ++++++++++++++++-- .../components/row-modal/row-modal.tsx | 191 +++++++++--- .../table-grid/cells/cell-content.tsx | 4 + .../cells/expanded-cell-popover.test.tsx | 148 +++++++++ .../cells/expanded-cell-popover.tsx | 34 +- .../table-grid/cells/inline-editors.test.ts | 63 ++++ .../table-grid/cells/inline-editors.tsx | 62 ++-- .../components/table-grid/data-row.tsx | 5 + .../table-grid/headers/column-header-menu.tsx | 8 + .../headers/workflow-group-meta-cell.tsx | 92 ++++-- .../components/table-grid/table-grid.tsx | 148 ++++++--- .../table-grid/table-primitives.tsx | 49 ++- .../tables/[tableId]/lock-copy.ts | 112 +++---- .../[workspaceId]/tables/[tableId]/table.tsx | 36 ++- .../[workspaceId]/tables/[tableId]/types.ts | 7 + apps/sim/hooks/queries/tables.ts | 52 +++- .../chip-date-picker/chip-date-picker.tsx | 22 +- 23 files changed, 1481 insertions(+), 439 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.test.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-config-sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-config-sidebar.tsx index 87b813c62ba..c2d80f94d36 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-config-sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-config-sidebar.tsx @@ -1,7 +1,17 @@ 'use client' import { useState } from 'react' -import { Button, ChipCombobox, ChipInput, cn, FieldDivider, Label, Switch, toast } from '@sim/emcn' +import { + Button, + ChipCombobox, + ChipInput, + cn, + FieldDivider, + Label, + Switch, + Tooltip, + toast, +} from '@sim/emcn' import { X } from '@sim/emcn/icons' import { toError } from '@sim/utils/errors' import { findValidationIssue, isValidationError } from '@/lib/api/client/errors' @@ -59,6 +69,15 @@ interface ColumnConfigSidebarProps { /** Notify parent of a rename so it can rewrite local `columnOrder` / * `columnWidths` keys that reference the old name. */ onColumnRename?: (oldName: string, newName: string) => void + /** + * Opens the panel for reading only — every field is inert and Save is + * disabled behind {@link readOnlyReason}. The header click that opens this + * sidebar is a primary affordance, so a schema-locked (or read-only) table + * shows the column's settings rather than swallowing the click. + */ + readOnly?: boolean + /** Why saving is unavailable; surfaced on the disabled Save button. */ + readOnlyReason?: string } /** @@ -109,6 +128,8 @@ function ColumnConfigBody({ workspaceId, tableId, onColumnRename, + readOnly, + readOnlyReason, }: ColumnConfigBodyProps) { const updateColumn = useUpdateColumn({ workspaceId, tableId }) const addColumn = useAddTableColumn({ workspaceId, tableId }) @@ -154,6 +175,8 @@ function ColumnConfigBody({ } async function handleSave() { + // Belt and braces: the button is disabled, and the server refuses too. + if (readOnly) return if (!trimmedName) { setShowValidation(true) return @@ -254,118 +277,136 @@ function ColumnConfigBody({
-
- Column name - { - setNameInput(e.target.value) - if (nameError) setNameError(null) - }} - spellCheck={false} - autoComplete='off' - error={Boolean((showValidation && !trimmedName) || nameError)} - aria-invalid={(showValidation && !trimmedName) || nameError ? true : undefined} - /> - {showValidation && !trimmedName && } - {nameError && !(showValidation && !trimmedName) && } -
- - {config.mode === 'edit' && ( - <> - -
- Type - option.type !== 'workflow') - .map((option) => ({ - label: option.label, - value: option.type, - icon: option.icon, - disabled: option.disabledReason !== undefined, - }))} - value={typeInput} - onChange={(v) => setTypeInput(v as ColumnDefinition['type'])} - placeholder='Select type' - maxHeight={300} - /> -
- - )} + {/* `disabled` on the fieldset reaches every native control inside, + including the comboboxes' trigger buttons; `contents` keeps the + existing layout. Values stay readable and selectable. */} +
+
+ Column name + { + setNameInput(e.target.value) + if (nameError) setNameError(null) + }} + spellCheck={false} + autoComplete='off' + error={Boolean((showValidation && !trimmedName) || nameError)} + aria-invalid={(showValidation && !trimmedName) || nameError ? true : undefined} + /> + {showValidation && !trimmedName && } + {nameError && !(showValidation && !trimmedName) && } +
- {wantsCurrency && ( - <> - -
- Currency - -
- - )} + {config.mode === 'edit' && ( + <> + +
+ Type + option.type !== 'workflow') + .map((option) => ({ + label: option.label, + value: option.type, + icon: option.icon, + disabled: option.disabledReason !== undefined, + }))} + value={typeInput} + onChange={(v) => setTypeInput(v as ColumnDefinition['type'])} + placeholder='Select type' + maxHeight={300} + /> +
+ + )} - {wantsOptions && ( - <> - -
- Options - { - setOptionsInput(next) - if (optionsError) setOptionsError(null) - }} - /> - {optionsError && } -
- -
- - setMultipleInput(!!v)} - /> -
- - )} + {wantsCurrency && ( + <> + +
+ Currency + +
+ + )} - {/* Select columns don't expose a unique constraint. */} - {!wantsOptions && ( - <> - -
+ {wantsOptions && ( + <> + +
+ Options + { + setOptionsInput(next) + if (optionsError) setOptionsError(null) + }} + /> + {optionsError && } +
+
- + setUniqueInput(!!v)} + id='column-sidebar-multiple' + checked={multipleInput} + onCheckedChange={(v) => setMultipleInput(!!v)} />
-
- - )} + + )} + + {/* Select columns don't expose a unique constraint. */} + {!wantsOptions && ( + <> + +
+
+ + setUniqueInput(!!v)} + /> +
+
+ + )} +
- + {readOnly ? ( + + + + + + + {readOnlyReason && {readOnlyReason}} + + ) : ( + + )}
) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.test.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.test.tsx index e4321a7eb59..c2e5bcb3886 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.test.tsx @@ -23,6 +23,36 @@ afterEach(() => { }) describe('ColumnDropdown', () => { + it('keeps a schema-locked trigger focusable for its explanation without opening a menu', () => { + const onPickType = vi.fn() + act(() => { + root.render( + + ) + }) + const trigger = container.querySelector('button')! + expect(trigger.getAttribute('aria-disabled')).toBe('true') + expect(trigger.disabled).toBe(false) + act(() => { + trigger.focus() + trigger.click() + }) + expect(document.querySelector('[role="tooltip"]')?.textContent).toContain( + 'Changing the table schema is disabled in Table Security.' + ) + expect(document.querySelector('[role="menu"]')).toBeNull() + expect(onPickType).not.toHaveBeenCalled() + }) + it('lists Enrichments as a regular entry after the column options', () => { const onPickEnrichment = vi.fn() @@ -37,7 +67,6 @@ describe('ColumnDropdown', () => { onPickWorkflow={vi.fn()} onPickEnrichment={onPickEnrichment} blocked={false} - onBlocked={vi.fn()} /> ) }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.tsx index 1f4ab32cef4..4fc8addc7ee 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.tsx @@ -10,12 +10,15 @@ import { DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger, - Plus, Tooltip, } from '@sim/emcn' -import { Sparkles } from '@sim/emcn/icons' +import { Lock, Plus, Sparkles } from '@sim/emcn/icons' import type { ColumnDefinition } from '@/lib/table' -import { type ColumnTypeOption, columnTypeOptionsForTable } from '../column-config-sidebar' +import { + type ColumnTypeOption, + columnTypeOptionsForTable, +} from '@/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar' +import { LOCK_TOOLTIPS } from '@/app/workspace/[workspaceId]/tables/[tableId]/lock-copy' const CELL_HEADER = 'border-[var(--border)] border-r border-b bg-[var(--bg)] px-2 py-[7px] text-left align-middle' @@ -30,14 +33,8 @@ interface ColumnDropdownProps { onPickType: (type: ColumnDefinition['type']) => void onPickWorkflow: () => void onPickEnrichment: () => void - /** - * When true, the trigger stays visible and clickable but opens nothing — it - * calls {@link onBlocked} instead. Used when the table is schema-locked: - * hiding the control leaves the user guessing, so it stays and explains. - * Paired required so `blocked` can never be set without a handler. - */ + /** A schema lock disables the action and explains why on hover or focus. */ blocked: boolean - onBlocked: () => void } interface ColumnTypeMenuItemProps { @@ -88,37 +85,46 @@ export function ColumnDropdown({ onPickWorkflow, onPickEnrichment, blocked, - onBlocked, }: ColumnDropdownProps) { + const Icon = blocked ? Lock : Plus const triggerButton = trigger === 'header' ? ( ) : ( ) if (blocked) { + const lockedTrigger = ( + + {triggerButton} + {LOCK_TOOLTIPS.schema} + + ) return trigger === 'inline-header' ? ( - {triggerButton} + {lockedTrigger} ) : ( - triggerButton + lockedTrigger ) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/context-menu/context-menu.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/context-menu/context-menu.tsx index 3bdc488b998..343c91d44c1 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/context-menu/context-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/context-menu/context-menu.tsx @@ -65,8 +65,7 @@ interface ContextMenuProps { disableInsert?: boolean /** * Duplicate is a one-shot insert carrying the copied row's data, so it needs - * only the insert lock — unlike the blank-row inserts above it, which also - * need the update lock to be fillable. + * only the insert lock. */ disableDuplicate?: boolean disableDelete?: boolean diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal.test.tsx new file mode 100644 index 00000000000..029a34b05af --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal.test.tsx @@ -0,0 +1,148 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { type TableLocks, UNLOCKED_TABLE_LOCKS } from '@/lib/table/types' +import { LockSettingsModal } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal' + +const { mutateAsync } = vi.hoisted(() => ({ mutateAsync: vi.fn() })) +vi.mock('@/hooks/queries/tables', () => ({ + useUpdateTableLocks: () => ({ mutateAsync, isPending: false }), +})) + +const LABELS = ['Inserting Rows', 'Updating Rows', 'Deleting Rows', 'Changing Table Schema'] +let container: HTMLDivElement +let root: Root +const onClose = vi.fn() + +function render(locks: TableLocks = UNLOCKED_TABLE_LOCKS, isOpen = true) { + act(() => { + root.render( + + ) + }) +} + +function getPermission(label: string, choice: 'Deny' | 'Allow'): HTMLButtonElement { + const group = document.querySelector(`[role="radiogroup"][aria-label="${label}"]`) + const button = [ + ...(group?.querySelectorAll('button[role="radio"]') ?? []), + ].find((element) => element.textContent === choice) + if (!button) throw new Error(`Missing permission: ${label} ${choice}`) + return button +} + +function selectPermission(label: string, choice: 'Deny' | 'Allow') { + act(() => getPermission(label, choice).click()) +} + +function getSave(): HTMLButtonElement { + const button = [...document.querySelectorAll('button')].find( + (element) => element.textContent === 'Save' + ) + if (!button) throw new Error('Missing Save button') + return button +} + +function save() { + act(() => getSave().click()) +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + vi.clearAllMocks() + mutateAsync.mockReturnValue(new Promise(() => {})) + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) +}) + +afterEach(() => { + act(() => root.unmount()) + container.remove() +}) + +describe('Table Security', () => { + it('always shows the four rows and starts an unconfigured table on Allow', () => { + render() + for (const label of LABELS) { + expect(getPermission(label, 'Allow').getAttribute('aria-checked')).toBe('true') + expect(getPermission(label, 'Deny').getAttribute('aria-checked')).toBe('false') + } + // Nothing staged yet, so there is nothing to save. + expect(getSave().disabled).toBe(true) + }) + + it('mirrors the server locks, with Deny meaning a set lock', () => { + render({ insertLocked: true, updateLocked: false, deleteLocked: true, schemaLocked: false }) + expect(getPermission('Inserting Rows', 'Deny').getAttribute('aria-checked')).toBe('true') + expect(getPermission('Deleting Rows', 'Deny').getAttribute('aria-checked')).toBe('true') + expect(getPermission('Updating Rows', 'Allow').getAttribute('aria-checked')).toBe('true') + expect(getPermission('Changing Table Schema', 'Allow').getAttribute('aria-checked')).toBe( + 'true' + ) + }) + + it('saves only the rows the admin moved', () => { + render() + selectPermission('Inserting Rows', 'Deny') + selectPermission('Changing Table Schema', 'Deny') + expect(getSave().disabled).toBe(false) + save() + + // A partial patch: the untouched rows are absent, so a concurrent change to + // one of them survives this save. + expect(mutateAsync.mock.calls[0][0]).toEqual({ + tableId: 'table-1', + locks: { insertLocked: true, schemaLocked: true }, + }) + }) + + it('follows a lock changed elsewhere while open without staging it', () => { + render() + selectPermission('Inserting Rows', 'Deny') + + // Another admin denies updates while this modal is open; the realtime + // refetch lands as a new `locks` prop. + render({ ...UNLOCKED_TABLE_LOCKS, updateLocked: true }) + expect(getPermission('Updating Rows', 'Deny').getAttribute('aria-checked')).toBe('true') + expect(getPermission('Inserting Rows', 'Deny').getAttribute('aria-checked')).toBe('true') + + save() + expect(mutateAsync.mock.calls[0][0]).toEqual({ + tableId: 'table-1', + locks: { insertLocked: true }, + }) + }) + + it('treats a row already matching the server as nothing to save', () => { + render({ ...UNLOCKED_TABLE_LOCKS, deleteLocked: true }) + selectPermission('Deleting Rows', 'Allow') + expect(getSave().disabled).toBe(false) + selectPermission('Deleting Rows', 'Deny') + expect(getSave().disabled).toBe(true) + }) + + it('keeps the modal open when the save fails and discards the draft on reopen', async () => { + mutateAsync.mockRejectedValueOnce(new Error('Admin access required to change table locks')) + render() + selectPermission('Updating Rows', 'Deny') + await act(async () => { + getSave().click() + }) + expect(onClose).not.toHaveBeenCalled() + + render(UNLOCKED_TABLE_LOCKS, false) + render() + expect(getPermission('Updating Rows', 'Allow').getAttribute('aria-checked')).toBe('true') + expect(getSave().disabled).toBe(true) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal.tsx index b30531933ca..e72cdce34ad 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/lock-settings-modal/lock-settings-modal.tsx @@ -1,30 +1,32 @@ 'use client' -import { useId, useState } from 'react' +import { useState } from 'react' import { + ChipButtonGroup, + ChipButtonGroupItem, ChipModal, ChipModalBody, + ChipModalField, ChipModalFooter, ChipModalHeader, - Label, - Switch, Tooltip, } from '@sim/emcn' import { CircleInfo, Lock } from '@sim/emcn/icons' -import type { TableLocks } from '@/lib/table' -import { - describeLocks, - LOCK_FIELDS, -} from '@/app/workspace/[workspaceId]/tables/[tableId]/lock-copy' +import type { TableLocks } from '@/lib/table/types' +import { LOCK_FIELDS } from '@/app/workspace/[workspaceId]/tables/[tableId]/lock-copy' import { useUpdateTableLocks } from '@/hooks/queries/tables' -function locksEqual(a: TableLocks, b: TableLocks): boolean { - return ( - a.schemaLocked === b.schemaLocked && - a.insertLocked === b.insertLocked && - a.updateLocked === b.updateLocked && - a.deleteLocked === b.deleteLocked - ) +/** + * The rows the admin actually moved, relative to the locks the server holds + * right now. Everything absent from this patch is left alone by the save. + */ +function changedLocks(overrides: Partial, locks: TableLocks): Partial { + const changed: Partial = {} + for (const field of LOCK_FIELDS) { + const next = overrides[field.key] + if (next !== undefined && next !== locks[field.key]) changed[field.key] = next + } + return changed } interface LockSettingsModalProps { @@ -36,10 +38,19 @@ interface LockSettingsModalProps { } /** - * Admin-only panel to toggle a table's four mutation locks. Changes are staged - * locally and applied on Save (one request); the server re-checks admin and - * rejects a `write`-only caller with a 403 surfaced as a toast. Gated at the - * call site on `canAdmin`. + * Admin-only panel that sets a table's four mutation locks, one Allow/Deny row + * each. The rows mirror the server flags exactly — `Deny` is a set lock — so a + * table nobody has configured opens on four `Allow`s and every viewer sees the + * same state. + * + * Only the rows this admin moved are staged; every other row keeps rendering + * the authoritative value, so a lock another admin changes while this modal is + * open shows up here instead of going stale behind it. Save sends just that + * patch (the route takes a partial), so it can't carry a stale flag over + * someone else's newer change — a row both admins moved is the only real + * conflict, and there this admin's explicit choice wins. The server re-checks + * admin and rejects a `write`-only caller with a 403 surfaced as a toast. + * Gated at the call site on `canAdmin`. */ export function LockSettingsModal({ isOpen, @@ -48,65 +59,77 @@ export function LockSettingsModal({ tableId, locks, }: LockSettingsModalProps) { - const idPrefix = useId() const updateLocks = useUpdateTableLocks(workspaceId) - // Stage edits locally; reset to the server value each time the modal opens. - const [draft, setDraft] = useState(locks) + // Stage only the rows this admin moved; clear them each time the modal opens. + const [overrides, setOverrides] = useState>({}) const [prevOpen, setPrevOpen] = useState(isOpen) if (prevOpen !== isOpen) { setPrevOpen(isOpen) - if (isOpen) setDraft(locks) + if (isOpen) setOverrides({}) } - const dirty = !locksEqual(draft, locks) - const summary = describeLocks(draft) + const changed = changedLocks(overrides, locks) + const dirty = Object.keys(changed).length > 0 - const handleSave = () => { + const handleSave = async () => { if (!dirty) { onClose() return } - updateLocks.mutate({ tableId, locks: draft }, { onSuccess: () => onClose() }) + try { + await updateLocks.mutateAsync({ tableId, locks: changed }) + } catch { + return + } + onClose() } return ( - !open && onClose()} srTitle='Table locks'> + !open && onClose()} srTitle='Table Security'> - Table locks + Table Security -

- {summary.name} — {summary.detail} -

- {LOCK_FIELDS.map((field) => { - const fieldId = `${idPrefix}-${field.kind}` - return ( -
-
- + {LOCK_FIELDS.map((field) => ( + + {field.label} - {/* Not `asChild`: the hint is each lock's only explanation, so + {/* Not `asChild`: the hint is each row's only explanation, so the trigger must be a focusable button for keyboard users. */} - +

{field.hint}

 -
- - setDraft((prev) => ({ ...prev, [field.key]: checked })) - } - /> -
- ) - })} + + } + > + + setOverrides((prev) => ({ ...prev, [field.key]: value === 'deny' })) + } + > + Deny + Allow + + + ))} ({ - mockToastError: vi.fn(), - mockUseTimezoneState: vi.fn(), - mockUpdateRow: vi.fn(), - mockDeleteRow: vi.fn(), - mockDeleteRows: vi.fn(), - })) +const { + mockToastError, + mockUseTimezoneState, + mockCreateRow, + mockUpdateRow, + mockDeleteRow, + mockDeleteRows, +} = vi.hoisted(() => ({ + mockToastError: vi.fn(), + mockUseTimezoneState: vi.fn(), + mockCreateRow: vi.fn(), + mockUpdateRow: vi.fn(), + mockDeleteRow: vi.fn(), + mockDeleteRows: vi.fn(), +})) vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), @@ -23,6 +30,7 @@ vi.mock('@/hooks/queries/general-settings', () => ({ useTimezoneState: mockUseTimezoneState, })) vi.mock('@/hooks/queries/tables', () => ({ + useCreateTableRow: () => ({ mutateAsync: mockCreateRow, isPending: false }), useUpdateTableRow: () => ({ mutateAsync: mockUpdateRow, isPending: false }), useDeleteTableRow: () => ({ mutateAsync: mockDeleteRow, isPending: false }), useDeleteTableRows: () => ({ mutateAsync: mockDeleteRows, isPending: false }), @@ -35,11 +43,12 @@ vi.mock('@sim/emcn', () => { createElement('button', { type: 'button', ...props }, children), ChipConfirmModal: passthrough, ChipDatePicker: ({ value, onChange }: { value?: string; onChange: (value: string) => void }) => - createElement( - 'button', - { type: 'button', 'data-testid': 'date', onClick: () => onChange(value ?? '2026-11-01') }, - value - ), + createElement('input', { + 'data-testid': 'date', + value: value ?? '', + onChange: (event: { currentTarget: { value: string } }) => + onChange(event.currentTarget.value), + }), ChipModal: passthrough, ChipModalBody: passthrough, ChipModalError: passthrough, @@ -80,13 +89,6 @@ vi.mock('@sim/emcn', () => { 'Update Row' ), ChipModalHeader: passthrough, - ChipTimePicker: ({ value, onChange }: { value?: string; onChange: (value: string) => void }) => - createElement('input', { - 'data-testid': 'time', - value: value ?? '', - onChange: (event: { currentTarget: { value: string } }) => - onChange(event.currentTarget.value), - }), Label: passthrough, toast: { error: mockToastError }, } @@ -113,6 +115,154 @@ function changeInput(input: HTMLInputElement, value: string) { input.dispatchEvent(new Event('input', { bubbles: true })) } +describe('RowModal add mode', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateRow.mockResolvedValue(undefined) + mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', status: 'ready' }) + }) + + it('inserts the complete row under column ids in one request without updating', async () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const props = { + mode: 'add' as const, + isOpen: true, + onClose: vi.fn(), + table: { + id: 'table-3', + name: 'People', + schema: { columns: [{ id: 'col_name', name: 'Name', type: 'string' as const }] }, + }, + onSuccess: vi.fn(), + } + + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + act(() => root.render(createElement(RowModal, props))) + + const nameInput = container.querySelector('[data-testid="modal-input"]') + expect(nameInput?.value).toBe('') + act(() => changeInput(nameInput as HTMLInputElement, 'Ada')) + const submit = container.querySelector('[data-testid="submit"]') + await act(async () => submit?.click()) + + expect(mockCreateRow).toHaveBeenCalledWith({ data: { col_name: 'Ada' } }) + expect(mockUpdateRow).not.toHaveBeenCalled() + expect(props.onSuccess).toHaveBeenCalledTimes(1) + + act(() => root.unmount()) + container.remove() + }) + + it('inserts the row at the requested position', async () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const props = { + mode: 'add' as const, + isOpen: true, + onClose: vi.fn(), + table: { + id: 'table-3', + name: 'People', + schema: { + columns: [{ id: 'col_name', name: 'Name', type: 'string' as const, required: true }], + }, + }, + insertAt: { afterRowId: 'row-1' }, + onSuccess: vi.fn(), + } + + act(() => root.render(createElement(RowModal, props))) + + const nameInput = container.querySelector('[data-testid="modal-input"]') + act(() => changeInput(nameInput as HTMLInputElement, 'Ada')) + const submit = container.querySelector('[data-testid="submit"]') + await act(async () => submit?.click()) + + expect(mockCreateRow).toHaveBeenCalledWith({ data: { col_name: 'Ada' }, afterRowId: 'row-1' }) + act(() => root.unmount()) + container.remove() + }) + + it('keeps Add Row disabled until every required field has a value', () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const props = { + mode: 'add' as const, + isOpen: true, + onClose: vi.fn(), + table: { + id: 'table-3', + name: 'People', + schema: { + columns: [ + { id: 'col_name', name: 'Name', type: 'string' as const, required: true }, + { id: 'col_notes', name: 'Notes', type: 'string' as const }, + { id: 'col_active', name: 'Active', type: 'boolean' as const, required: true }, + ], + }, + }, + onSuccess: vi.fn(), + } + + act(() => root.render(createElement(RowModal, props))) + + const submit = () => container.querySelector('[data-testid="submit"]') + const nameInput = container.querySelectorAll('[data-testid="modal-input"]')[0] + expect(submit()?.disabled).toBe(true) + + act(() => changeInput(nameInput, 'Ada')) + expect(submit()?.disabled).toBe(false) + + act(() => changeInput(nameInput, '')) + expect(submit()?.disabled).toBe(true) + + act(() => root.unmount()) + container.remove() + }) +}) + +describe('RowModal column ids', () => { + beforeEach(() => { + vi.clearAllMocks() + mockUpdateRow.mockResolvedValue(undefined) + mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', status: 'ready' }) + }) + + it('shows and saves edit values stored under the column id', async () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const props = { + mode: 'edit' as const, + isOpen: true, + onClose: vi.fn(), + table: { + id: 'table-4', + name: 'People', + schema: { columns: [{ id: 'col_name', name: 'Name', type: 'string' as const }] }, + }, + row: { ...row, data: { col_name: 'Ada' } }, + onSuccess: vi.fn(), + } + + act(() => root.render(createElement(RowModal, props))) + + const nameInput = container.querySelector('[data-testid="modal-input"]') + expect(nameInput?.value).toBe('Ada') + act(() => changeInput(nameInput as HTMLInputElement, 'Grace')) + const submit = container.querySelector('[data-testid="submit"]') + await act(async () => submit?.click()) + + expect(mockUpdateRow).toHaveBeenCalledWith({ rowId: 'row-1', data: { col_name: 'Grace' } }) + act(() => root.unmount()) + container.remove() + }) +}) + describe('RowModal expiration editing', () => { beforeEach(() => { vi.clearAllMocks() @@ -136,7 +286,9 @@ describe('RowModal expiration editing', () => { ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true act(() => root.render(createElement(RowModal, props))) - expect(container.querySelector('[data-testid="time"]')?.value).toBe('01:00') + expect(container.querySelector('[data-testid="date"]')?.value).toBe( + '2026-11-01T01:00:00' + ) expect(container.querySelector('[data-testid="submit"]')?.disabled).toBe( false ) @@ -153,9 +305,9 @@ describe('RowModal expiration editing', () => { }) act(() => root.render(createElement(RowModal, props))) - const timeInput = container.querySelector('[data-testid="time"]') - expect(timeInput?.value).toBe('01:00') - act(() => changeInput(timeInput as HTMLInputElement, '01:30')) + const dateInput = container.querySelector('[data-testid="date"]') + expect(dateInput?.value).toBe('2026-11-01T01:00:00') + act(() => changeInput(dateInput as HTMLInputElement, '2026-11-01T01:30')) const submit = container.querySelector('[data-testid="submit"]') await act(async () => submit?.click()) @@ -193,7 +345,7 @@ describe('RowModal expiration editing', () => { expect(container.querySelector('[aria-label="Edit starts_at"]')?.textContent).toBe( 'Loading timezone…' ) - expect(container.querySelector('[data-testid="time"]')).toBeNull() + expect(container.querySelector('[data-testid="date"]')).toBeNull() mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', @@ -201,7 +353,7 @@ describe('RowModal expiration editing', () => { }) act(() => root.render(createElement(RowModal, props))) - expect(container.querySelector('[data-testid="time"]')).not.toBeNull() + expect(container.querySelector('[data-testid="date"]')).not.toBeNull() act(() => root.unmount()) container.remove() }) @@ -226,7 +378,9 @@ describe('RowModal expiration editing', () => { act(() => root.render(createElement(RowModal, props))) - expect(container.querySelector('[data-testid="time"]')?.value).toBe('01:00') + expect(container.querySelector('[data-testid="date"]')?.value).toBe( + '2026-11-01T01:00:00' + ) expect(container.querySelector('[data-testid="submit"]')?.disabled).toBe( false ) @@ -235,7 +389,7 @@ describe('RowModal expiration editing', () => { container.remove() }) - it('keeps unrelated fields editable and omits blocked date values from the update', async () => { + it('sends only the edited field and omits blocked date values from the update', async () => { mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', savedTimezone: 'Mars/Olympus', @@ -283,10 +437,10 @@ describe('RowModal expiration editing', () => { act(() => changeInput(nameInput as HTMLInputElement, 'Grace')) await act(async () => submit?.click()) - expect(mockUpdateRow).toHaveBeenCalledWith({ - rowId: 'row-1', - data: { name: 'Grace', expires_at: row.data.expires_at }, - }) + // Only the edited field is sent: the untouched TTL would otherwise be + // rewritten with the same value (and re-stamped through the picker), and the + // timezone-blocked date is dropped entirely. + expect(mockUpdateRow).toHaveBeenCalledWith({ rowId: 'row-1', data: { name: 'Grace' } }) expect(props.onSuccess).toHaveBeenCalledTimes(1) expect(mockToastError).not.toHaveBeenCalled() @@ -294,3 +448,76 @@ describe('RowModal expiration editing', () => { container.remove() }) }) + +describe('RowModal payload', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateRow.mockResolvedValue(undefined) + mockUpdateRow.mockResolvedValue(undefined) + mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', status: 'ready' }) + }) + + it('closes without a write when the edit changes nothing', async () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const props = { + mode: 'edit' as const, + isOpen: true, + onClose: vi.fn(), + table: { + id: 'table-5', + name: 'People', + schema: { columns: [{ id: 'col_name', name: 'Name', type: 'string' as const }] }, + }, + row: { ...row, data: { col_name: 'Ada' } }, + onSuccess: vi.fn(), + } + + act(() => root.render(createElement(RowModal, props))) + const submit = container.querySelector('[data-testid="submit"]') + await act(async () => submit?.click()) + + expect(mockUpdateRow).not.toHaveBeenCalled() + expect(props.onSuccess).toHaveBeenCalledTimes(1) + + act(() => root.unmount()) + container.remove() + }) + + it('omits untouched columns on insert but still sends toggles', async () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const props = { + mode: 'add' as const, + isOpen: true, + onClose: vi.fn(), + table: { + id: 'table-6', + name: 'People', + schema: { + columns: [ + { id: 'col_name', name: 'Name', type: 'string' as const }, + { id: 'col_notes', name: 'Notes', type: 'string' as const }, + { id: 'col_done', name: 'Done', type: 'boolean' as const }, + ], + }, + }, + onSuccess: vi.fn(), + } + + act(() => root.render(createElement(RowModal, props))) + const nameInput = container.querySelector('[data-testid="modal-input"]') + act(() => changeInput(nameInput as HTMLInputElement, 'Ada')) + const submit = container.querySelector('[data-testid="submit"]') + await act(async () => submit?.click()) + + // `col_notes` was never touched, so it stays absent instead of being written + // as null; a checkbox always carries a concrete boolean. + expect(mockCreateRow).toHaveBeenCalledWith({ data: { col_name: 'Ada', col_done: false } }) + + act(() => root.unmount()) + container.remove() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx index 94939e31e28..9da75292fae 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx @@ -12,7 +12,6 @@ import { ChipModalField, ChipModalFooter, ChipModalHeader, - ChipTimePicker, Label, toast, } from '@sim/emcn' @@ -20,17 +19,26 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useParams } from 'next/navigation' import type { ColumnDefinition, TableInfo, TableRow } from '@/lib/table' +import { getColumnId } from '@/lib/table/column-keys' import { columnTypeOf } from '@/lib/table/column-types' import { resolveCurrencyCode } from '@/lib/table/currency' +import { isEmptyCellValue } from '@/lib/table/deps' import { todayAtTtlOffset, ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' import { getTimezoneEditBlockedMessage } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/timezone-editing' +import type { RowInsertTarget } from '@/app/workspace/[workspaceId]/tables/[tableId]/types' import { type TimezoneState, useTimezoneState } from '@/hooks/queries/general-settings' -import { useDeleteTableRow, useDeleteTableRows, useUpdateTableRow } from '@/hooks/queries/tables' +import { + useCreateTableRow, + useDeleteTableRow, + useDeleteTableRows, + useUpdateTableRow, +} from '@/hooks/queries/tables' import { cleanCellValue, dateValueToLocalParts, formatValueForInput, localPartsToDateValue, + storageToDisplay, todayLocalCalendarDate, } from '../../utils' import { SelectValueEditor } from '../select-field' @@ -38,47 +46,89 @@ import { SelectValueEditor } from '../select-field' const logger = createLogger('RowModal') export interface RowModalProps { - mode: 'edit' | 'delete' + mode: 'add' | 'edit' | 'delete' isOpen: boolean onClose: () => void table: TableInfo row?: TableRow rowIds?: string[] + /** Where add mode inserts the row; appends when omitted. */ + insertAt?: RowInsertTarget onSuccess: () => void } +/** Structural equality for a cleaned cell value vs what the row already holds. */ +function cellValueUnchanged(next: unknown, previous: unknown): boolean { + if (next === previous) return true + const nextEmpty = next === null || next === undefined + const previousEmpty = previous === null || previous === undefined + if (nextEmpty || previousEmpty) return nextEmpty && previousEmpty + if (typeof next === 'object' || typeof previous === 'object') { + return JSON.stringify(next) === JSON.stringify(previous) + } + return false +} + +/** + * Builds the write payload. Only fields the user actually touched are sent, so + * an untouched empty column is left absent instead of being written as `null` — + * and in edit mode a field whose value is unchanged is dropped too, leaving a + * no-op save with nothing to write. Toggles are the exception on insert: they + * always carry a concrete boolean, so a required checkbox the user never + * clicked still has to reach the server as `false`. + */ function cleanRowData( columns: ColumnDefinition[], rowData: Record, timeZone: string, - dateEditorsReady: boolean + dateEditorsReady: boolean, + options: { mode: 'add' | 'edit'; baseline?: Record } ): Record { const cleanData: Record = {} columns.forEach((col) => { - const value = rowData[col.name] - if (columnTypeOf(col).editor === 'date' && !dateEditorsReady) { + const columnId = getColumnId(col) + const definition = columnTypeOf(col) + if (definition.editor === 'date' && !dateEditorsReady) { return } + const touched = columnId in rowData + const alwaysSend = options.mode === 'add' && definition.editor === 'toggle' + if (!touched && !alwaysSend) return + const value = rowData[columnId] + let cleaned: unknown try { - cleanData[col.name] = cleanCellValue(value, col, timeZone) + cleaned = cleanCellValue(value, col, timeZone) } catch { throw new Error(`Invalid JSON for field: ${col.name}`) } + if (options.baseline && cellValueUnchanged(cleaned, options.baseline[columnId])) return + cleanData[columnId] = cleaned }) return cleanData } /** - * Modal for editing a row's values or confirming row deletion. + * Modal for adding a complete row, editing a row's values, or confirming row + * deletion. Adding inserts every value in one request, so it works on a table + * whose update lock blocks filling in a blank row from the grid. * - * `rowData` is initialized from the `row` prop at mount time only. Both call-sites - * conditionally mount this component per open, so each open gets fresh state. If a + * `rowData` is initialized from the `row` prop at mount time only. Every call-site + * conditionally mounts this component per open, so each open gets fresh state. If a * call-site ever keeps it mounted across target-row changes, it must supply a `key` * prop (e.g. the row id) so React remounts with the new row's values. */ -export function RowModal({ mode, isOpen, onClose, table, row, rowIds, onSuccess }: RowModalProps) { +export function RowModal({ + mode, + isOpen, + onClose, + table, + row, + rowIds, + insertAt, + onSuccess, +}: RowModalProps) { const params = useParams() const workspaceId = params.workspaceId as string const tableId = table.id @@ -97,33 +147,58 @@ export function RowModal({ mode, isOpen, onClose, table, row, rowIds, onSuccess mode === 'edit' && row ? row.data : {} ) const [error, setError] = useState(null) - const updateRowMutation = useUpdateTableRow({ workspaceId, tableId }) - const deleteRowMutation = useDeleteTableRow({ workspaceId, tableId }) - const deleteRowsMutation = useDeleteTableRows({ workspaceId, tableId }) + // This modal renders its own failure in ``; without the flag + // every rejection would also arrive as a toast saying the same sentence. + const rowMutationContext = { workspaceId, tableId, suppressErrorToast: true } + const createRowMutation = useCreateTableRow(rowMutationContext) + const updateRowMutation = useUpdateTableRow(rowMutationContext) + const deleteRowMutation = useDeleteTableRow(rowMutationContext) + const deleteRowsMutation = useDeleteTableRows(rowMutationContext) const isSubmitting = - updateRowMutation.isPending || deleteRowMutation.isPending || deleteRowsMutation.isPending + createRowMutation.isPending || + updateRowMutation.isPending || + deleteRowMutation.isPending || + deleteRowsMutation.isPending + const isAddMode = mode === 'add' const timezoneBlockedMessage = getTimezoneEditBlockedMessage(timezoneState) const hasEditableColumn = columns.some( (column) => columnTypeOf(column).editor !== 'date' || dateEditorsReady ) + /** Toggles always save a boolean, so only other required columns can be left empty. */ + const missingRequiredValue = columns.some( + (column) => + column.required && + columnTypeOf(column).editor !== 'toggle' && + isEmptyCellValue(rowData[getColumnId(column)]) + ) + const canSubmit = hasEditableColumn && !missingRequiredValue const handleFormSubmit = async (e?: React.FormEvent) => { e?.preventDefault() setError(null) - if (!hasEditableColumn) return + if (!canSubmit) return try { - const cleanData = cleanRowData(columns, rowData, timeZone, dateEditorsReady) - - if (row) { - await updateRowMutation.mutateAsync({ rowId: row.id, data: cleanData }) + const cleanData = cleanRowData(columns, rowData, timeZone, dateEditorsReady, { + mode: isAddMode ? 'add' : 'edit', + baseline: isAddMode ? undefined : row?.data, + }) + + if (isAddMode) { + await createRowMutation.mutateAsync({ data: cleanData, ...insertAt }) + } else if (row) { + // Nothing changed — close instead of writing an empty patch. + if (Object.keys(cleanData).length > 0) { + await updateRowMutation.mutateAsync({ rowId: row.id, data: cleanData }) + } } onSuccess() } catch (err) { - logger.error('Failed to edit row:', err) - setError(getErrorMessage(err, 'Failed to edit row')) + const action = isAddMode ? 'add' : 'edit' + logger.error(`Failed to ${action} row:`, err) + setError(getErrorMessage(err, `Failed to ${action} row`)) } } @@ -182,20 +257,25 @@ export function RowModal({ mode, isOpen, onClose, table, row, rowIds, onSuccess } return ( - - Edit Row + + {isAddMode ? 'Add Row' : 'Edit Row'}

- Update values for {table?.name ?? 'table'} + {isAddMode ? 'Fill in values for' : 'Update values for'} {table?.name ?? 'table'}

- - + {saveBlockedReason ? ( + + + + + + + {saveBlockedReason} + + ) : ( + + )} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts index d08d7cb5a9e..66900798c6d 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts @@ -396,3 +396,66 @@ describe('dateEditorRawValue', () => { container.remove() }) }) + +describe('read-only InlineEditor', () => { + beforeEach(() => { + vi.clearAllMocks() + mockUseTimezoneState.mockReturnValue({ + timezone: 'America/Los_Angeles', + status: 'ready', + }) + }) + + it('shows a text value that can be selected but not changed', () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onSave = vi.fn() + + act(() => + root.render( + createElement(InlineEditor, { + value: 'Original text', + column: column('string'), + readOnly: true, + onSave, + onCancel: vi.fn(), + }) + ) + ) + + const input = container.querySelector('input') as HTMLInputElement + expect(input.value).toBe('Original text') + expect(input.readOnly).toBe(true) + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) + + expect(onSave).toHaveBeenCalledWith('Original text', 'enter') + act(() => root.unmount()) + container.remove() + }) + + it('opens a date read-only without the calendar picker', () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + + act(() => + root.render( + createElement(InlineEditor, { + value: '2026-06-15T06:00:30-07:00', + column: column('ttl'), + readOnly: true, + onSave: vi.fn(), + onCancel: vi.fn(), + }) + ) + ) + + const input = container.querySelector('input') as HTMLInputElement + expect(input.value).toBe('2026-06-15T06:00:30-07:00') + expect(input.readOnly).toBe(true) + expect(mockCalendar).not.toHaveBeenCalled() + act(() => root.unmount()) + container.remove() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx index d6e754beb4a..dff9efd7ba3 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx @@ -35,6 +35,8 @@ interface InlineEditorProps { value: unknown column: ColumnDefinition initialCharacter?: string + /** Shows the value without allowing changes; text stays selectable and copyable. */ + readOnly?: boolean onSave: (value: unknown, reason: SaveReason) => void onCancel: () => void } @@ -105,6 +107,7 @@ function ReadyInlineDateEditor({ value, column, initialCharacter, + readOnly, onSave, onCancel, initialTimeZone, @@ -274,33 +277,38 @@ function ReadyInlineDateEditor({ }} onKeyDown={handleKeyDown} onBlur={scheduleBlurSave} + readOnly={readOnly} placeholder={isOffsetDate ? 'YYYY-MM-DDTHH:mm:ss±HH:mm' : 'mm/dd/yyyy'} className={cn( 'w-full min-w-0 select-text border-none bg-transparent p-0 text-[var(--text-primary)] text-small outline-hidden', invalid && 'text-[var(--text-error)]' )} /> - - - - - - + {!readOnly && ( + + + + + + + )} ) } @@ -310,6 +318,7 @@ function InlineTextEditor({ value, column, initialCharacter, + readOnly, onSave, onCancel, }: InlineEditorProps) { @@ -394,6 +403,7 @@ function InlineTextEditor({ onKeyDown={handleKeyDown} onWheel={handleEditorWheel} onBlur={() => doSave('blur')} + readOnly={readOnly} className={cn( 'w-full min-w-0 select-text border-none bg-transparent p-0 text-[var(--text-primary)] text-small outline-hidden', invalid && 'text-[var(--text-error)]' @@ -409,7 +419,7 @@ function InlineTextEditor({ * toggles and commits when the menu closes. Escape discards the draft, matching * the text/date inline editors. */ -function InlineSelectEditor({ value, column, onSave, onCancel }: InlineEditorProps) { +function InlineSelectEditor({ value, column, readOnly, onSave, onCancel }: InlineEditorProps) { const isMulti = !!column.multiple const allOptions = column.options ?? [] const [draft, setDraft] = useState(() => selectedOptionIds(column, value)) @@ -475,13 +485,17 @@ function InlineSelectEditor({ value, column, onSave, onCancel }: InlineEditorPro {!isMulti && !column.required && ( - setDraftAnd([])}> + setDraftAnd([])}> None {draft.length === 0 && } )} {allOptions.map((option) => ( - handleSelectOption(e, option.id)}> + handleSelectOption(e, option.id)} + > {draft.includes(option.id) && } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx index 86963aa768b..0c603a6e1dc 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx @@ -33,6 +33,8 @@ export interface DataRowProps { isFirstRow: boolean editingColumnName: string | null initialCharacter: string | null + /** Opens cell editors read-only, e.g. on an update-locked table. */ + editorsReadOnly: boolean pendingCellValue: Record | null normalizedSelection: NormalizedSelection | null onClick: (rowId: string, columnName: string, options?: { toggleBoolean?: boolean }) => void @@ -121,6 +123,7 @@ function dataRowPropsAreEqual(prev: DataRowProps, next: DataRowProps): boolean { prev.rowIndex !== next.rowIndex || prev.isFirstRow !== next.isFirstRow || prev.editingColumnName !== next.editingColumnName || + prev.editorsReadOnly !== next.editorsReadOnly || prev.pendingCellValue !== next.pendingCellValue || prev.onClick !== next.onClick || prev.onDoubleClick !== next.onDoubleClick || @@ -170,6 +173,7 @@ export const DataRow = React.memo(function DataRow({ isFirstRow, editingColumnName, initialCharacter, + editorsReadOnly, pendingCellValue, normalizedSelection, isRowChecked, @@ -417,6 +421,7 @@ export const DataRow = React.memo(function DataRow({ column={column} isEditing={isEditing} initialCharacter={isEditing ? initialCharacter : undefined} + readOnly={editorsReadOnly} onSave={(value, reason) => onSave(row.id, column.key, value, reason)} onCancel={onCancel} waitingOnLabels={ diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/column-header-menu.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/column-header-menu.tsx index e2667e8434b..6ba6ebfa820 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/column-header-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/column-header-menu.tsx @@ -15,6 +15,10 @@ interface ColumnHeaderMenuProps { column: DisplayColumn colIndex: number readOnly?: boolean + /** Why column changes are unavailable; disables the schema rows and explains them. */ + schemaLockedReason?: string + /** Why deleting is unavailable; disables the destructive column row. */ + deleteLockedReason?: string isRenaming: boolean isColumnSelected: boolean renameValue: string @@ -65,6 +69,8 @@ export const ColumnHeaderMenu = React.memo(function ColumnHeaderMenu({ column, colIndex, readOnly, + schemaLockedReason, + deleteLockedReason, isRenaming, isColumnSelected, renameValue, @@ -346,6 +352,8 @@ export const ColumnHeaderMenu = React.memo(function ColumnHeaderMenu({ column={column} deleteLabel={deleteLabel} onOpenConfig={onOpenConfig} + schemaLockedReason={schemaLockedReason} + deleteLockedReason={deleteLockedReason} onInsertLeft={onInsertLeft} onInsertRight={onInsertRight} onDeleteColumn={onDeleteColumn} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/workflow-group-meta-cell.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/workflow-group-meta-cell.tsx index e9f4e435e11..69c159e585b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/workflow-group-meta-cell.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/workflow-group-meta-cell.tsx @@ -12,6 +12,7 @@ import { DropdownMenuSubContent, DropdownMenuSubTrigger, DropdownMenuTrigger, + Tooltip, } from '@sim/emcn' import { ArrowDown, @@ -70,6 +71,10 @@ interface ColumnOptionsMenuProps { * it leaves the group with siblings). */ deleteLabel?: string onOpenConfig: (columnName: string) => void + /** Why column changes are unavailable; disables the schema rows and explains them. */ + schemaLockedReason?: string + /** Why deleting is unavailable; disables the destructive column row. */ + deleteLockedReason?: string onInsertLeft: (columnName: string) => void onInsertRight: (columnName: string) => void onDeleteColumn: (columnName: string) => void @@ -108,6 +113,24 @@ interface ColumnOptionsMenuProps { onPinToggle?: (columnName: string) => void } +/** + * A menu row a lock disables. A disabled `DropdownMenuItem` sets + * `pointer-events: none`, so it can never receive the hover its own tooltip + * would need — the trigger wraps it instead (same shape as the folder menu). + * Renders the row untouched when nothing blocks it. + */ +function MenuRow({ reason, children }: { reason?: string; children: React.ReactElement }) { + if (!reason) return children + return ( + + +
{children}
+ + {reason} + + ) +} + /** * Shared column-options dropdown rendered next to the column header chevron * AND on right-click of the workflow group meta cell. Anchors to a fixed @@ -122,6 +145,8 @@ export function ColumnOptionsMenu({ column, deleteLabel, onOpenConfig, + schemaLockedReason, + deleteLockedReason, onInsertLeft, onInsertRight, onDeleteColumn, @@ -139,6 +164,9 @@ export function ColumnOptionsMenu({ isPinned, onPinToggle, }: ColumnOptionsMenuProps) { + // Hiding a workflow output leaves the data alone, so no lock covers it. + const destructiveReason = + deleteLabel === 'Hide column' ? undefined : (schemaLockedReason ?? deleteLockedReason) const showRunActions = Boolean(onRunColumnAll && onRunColumnIncomplete) const showRunSelected = Boolean(onRunColumnSelected) && selectedRowCount > 0 const runLabels = runMenuLabels(hasActiveFilter) @@ -228,10 +256,15 @@ export function ColumnOptionsMenu({ View workflow )} - onOpenConfig(column.key)}> - - Edit column - + + onOpenConfig(column.key)} + > + + Edit column + + {onPinToggle && ( onPinToggle(column.key)}> {isPinned ? : } @@ -239,23 +272,36 @@ export function ColumnOptionsMenu({ )} {/* Stops acting on this column and starts creating siblings — `Edit column` - above is unconditional, so the rule is always backed. */} + above always renders (disabled or not), so the rule is always backed. */} - onInsertLeft(column.key)}> - - Insert column left - - onInsertRight(column.key)}> - - Insert column right - + + onInsertLeft(column.key)} + > + + Insert column left + + + + onInsertRight(column.key)} + > + + Insert column right + + - (onDeleteGroup ? onDeleteGroup() : onDeleteColumn(column.key))} - > - {deleteLabel === 'Hide column' ? : } - {deleteLabel ?? 'Delete column'} - + + (onDeleteGroup ? onDeleteGroup() : onDeleteColumn(column.key))} + > + {deleteLabel === 'Hide column' ? : } + {deleteLabel ?? 'Delete column'} + + ) @@ -281,6 +327,10 @@ interface WorkflowGroupMetaCellProps { isGroupSelected: boolean onSelectGroup: (startColIndex: number, size: number) => void onOpenConfig: (columnName: string) => void + /** Why column changes are unavailable; disables the schema rows and explains them. */ + schemaLockedReason?: string + /** Why deleting is unavailable; disables the destructive column row. */ + deleteLockedReason?: string onRunColumn?: (groupId: string, mode?: RunMode, rowIds?: string[], limit?: RunLimit) => void onInsertLeft?: (columnName: string) => void onInsertRight?: (columnName: string) => void @@ -334,6 +384,8 @@ export function WorkflowGroupMetaCell({ isGroupSelected, onSelectGroup, onOpenConfig, + schemaLockedReason, + deleteLockedReason, onRunColumn, onInsertLeft, onInsertRight, @@ -539,6 +591,8 @@ export function WorkflowGroupMetaCell({ position={optionsMenuPosition} column={column} onOpenConfig={onOpenConfig} + schemaLockedReason={schemaLockedReason} + deleteLockedReason={deleteLockedReason} onInsertLeft={onInsertLeft} onInsertRight={onInsertRight} onDeleteColumn={onDeleteColumn} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index f800493d046..f040e922466 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -27,6 +27,7 @@ import type { import { getColumnId } from '@/lib/table/column-keys' import { columnTypeOf } from '@/lib/table/column-types' import { TABLE_LIMITS } from '@/lib/table/constants' +import { isEmptyCellValue } from '@/lib/table/deps' import { cellValueFilterConditions } from '@/lib/table/query-builder/cell-filter' import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { FindBar } from '@/app/workspace/[workspaceId]/components' @@ -34,6 +35,7 @@ import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/provide import { getTimezoneEditBlockedMessage } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/timezone-editing' import type { RemoteTableSelection } from '@/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room' import type { BlockedTableAction } from '@/app/workspace/[workspaceId]/tables/[tableId]/lock-copy' +import { LOCK_TOOLTIPS } from '@/app/workspace/[workspaceId]/tables/[tableId]/lock-copy' import { useTimezoneState } from '@/hooks/queries/general-settings' import { useAddTableColumn, @@ -55,7 +57,7 @@ import { extractCreatedRowId, useTableUndo } from '@/hooks/use-table-undo' import type { ChatContext } from '@/stores/panel' import type { DeletedRowSnapshot } from '@/stores/table/types' import { useContextMenu, useTable } from '../../hooks' -import type { EditingCell, QueryOptions, SaveReason } from '../../types' +import type { EditingCell, QueryOptions, RowInsertTarget, SaveReason } from '../../types' import { cleanCellValue, generateColumnName as sharedGenerateColumnName } from '../../utils' import type { ColumnConfig } from '../column-config-sidebar' import { ColumnDropdown } from '../column-dropdown' @@ -208,6 +210,8 @@ interface TableGridProps { onOpenEnrichmentDetails: (rowId: string, groupId: string) => void /** Open the row-edit modal for `row`. Wrapper renders the modal. */ onOpenRowModal: (row: TableRowType) => void + /** Opens the add-row form, which inserts a complete row at `insertAt` (appends when omitted). */ + onOpenAddRowModal: (insertAt?: RowInsertTarget) => void /** Open the row-delete modal for `snapshots`. Wrapper renders the modal. */ onRequestDeleteRows: (snapshots: DeletedRowSnapshot[]) => void /** @@ -370,6 +374,15 @@ function writeLoadedRowsWithChip(opts: { return true } +/** + * Whether new rows must go through the add-row form instead of a blank grid row. + * A blank row only works when the grid can fill it in afterwards: typing into it + * is an update, and the server rejects an empty row when any column is required. + */ +function needsAddRowForm(updateLocked: boolean | undefined, columns: ColumnDefinition[]): boolean { + return Boolean(updateLocked) || columns.some((column) => column.required) +} + /** * Value-equality for a cell's stored value vs a pending edit. Primitives compare * with `===`; arrays/objects (multiselect id arrays, json) compare structurally @@ -450,6 +463,7 @@ export function TableGrid({ onOpenExecutionDetails, onOpenEnrichmentDetails, onOpenRowModal, + onOpenAddRowModal, onRequestDeleteRows, onRequestDeleteAllByFilter, onRequestDeleteColumns, @@ -699,15 +713,14 @@ export function TableGrid({ // requires the delete lock clear too — mirror that here or the affordance // stays live on an append-only table and only fails on click. const canDestroyColumn = canMutateSchema && !locks?.deleteLocked - // Duplicate inserts a full copied row in one shot, so unlike the blank-row - // paths it needs the insert lock only — it is valid on an append-only table. + /** + * Inserts that carry the whole row in one request (Duplicate, paste-append, the + * add-row form) need only the insert lock, so they stay valid on an append-only + * table. New row, Shift+Enter, and Insert row fall back to that form whenever + * `needsAddRowForm` says a blank row can't work. + */ const canInsertFullRow = userPermissions.canEdit && !locks?.insertLocked - // Manual grid entry is "add an empty row, then type into its cells" — the - // typing is an update. So a *useful* manual add needs BOTH insert and update - // unlocked; on an append-only table (update locked) it would leave a blank - // row the user can't fill. The control stays visible and explains itself via - // `onBlockedAction`. Full-row inserts still flow through CSV import / API / - // blocks / Mothership, which the insert lock alone governs server-side. + /** A blank grid row is filled in by typing, which is an update, so it needs both locks off. */ const canManualAddRow = userPermissions.canEdit && !locks?.insertLocked && !locks?.updateLocked const canEditCellRef = useRef(canEditCell) canEditCellRef.current = canEditCell @@ -715,8 +728,8 @@ export function TableGrid({ canManualAddRowRef.current = canManualAddRow const canInsertFullRowRef = useRef(canInsertFullRow) canInsertFullRowRef.current = canInsertFullRow - // Read by the closure-free double-click handler to tell "locked" apart from - // "no write permission" — only the former gets the explanation modal. + // Read by the closure-free save and keyboard handlers to tell "locked" apart + // from "no write permission" — only the former gets the explanation toast. const updateLockedRef = useRef(locks?.updateLocked) updateLockedRef.current = locks?.updateLocked const onBlockedActionRef = useRef(onBlockedAction) @@ -727,6 +740,8 @@ export function TableGrid({ // Refs for callback props read inside effects with stable empty deps. const onOpenRowModalRef = useRef(onOpenRowModal) onOpenRowModalRef.current = onOpenRowModal + const onOpenAddRowModalRef = useRef(onOpenAddRowModal) + onOpenAddRowModalRef.current = onOpenAddRowModal const { contextMenu, @@ -1617,6 +1632,11 @@ export function TableGrid({ const anchorId = contextMenu.row.id // Fractional ordering: express intent by neighbor id, not integer position. const intent = offset === 0 ? { beforeRowId: anchorId } : { afterRowId: anchorId } + if (needsAddRowForm(updateLockedRef.current, schemaColumnsRef.current)) { + closeContextMenu() + onOpenAddRowModalRef.current(intent) + return + } createRef.current( { data: {}, ...intent }, { @@ -1756,6 +1776,13 @@ export function TableGrid({ // Stable identity so 's React.memo still bails out; lock state // is read from refs instead of being closed over. const handleAddRowClick = useCallback(() => { + if ( + canInsertFullRowRef.current && + needsAddRowForm(updateLockedRef.current, schemaColumnsRef.current) + ) { + onOpenAddRowModalRef.current() + return + } if (!canManualAddRowRef.current) { onBlockedActionRef.current('add-row') return @@ -2721,7 +2748,16 @@ export function TableGrid({ (rowId: string, columnName: string, options?: { toggleBoolean?: boolean }) => { const column = columnsRef.current.find((c) => c.key === columnName) if (column && columnTypeOf(column).editor === 'toggle') { - if (!options?.toggleBoolean || !canEditCellRef.current) return + if (!options?.toggleBoolean) return + // A toggle writes on the click itself, so there is no read-only editor to + // fall back to — an update-locked table has to explain the refusal here, + // the same way the Enter/Space keyboard paths do. + if (!canEditCellRef.current) { + if (canEditRef.current && updateLockedRef.current) { + onBlockedActionRef.current('edit-cell') + } + return + } const row = rowsRef.current.find((r) => r.id === rowId) if (row) { toggleBooleanCell(rowId, columnName, row.data[columnName]) @@ -2741,23 +2777,24 @@ export function TableGrid({ (rowId: string, columnName: string, columnKey: string) => { const column = columnsRef.current.find((c) => c.key === columnKey) if (column && columnTypeOf(column).editor === 'toggle') return - - // Double-click means "edit this cell". On an update-locked table, say so - // rather than opening the expanded viewer — which looks like an editor - // that silently refuses to save. Only for users who could otherwise edit: - // without write access the lock isn't why they can't, and they still get - // the read-only expanded viewer below. - if (canEditRef.current && updateLockedRef.current) { - onBlockedActionRef.current('edit-cell') + // A read-only view of an empty cell has nothing to show or copy. + if ( + !canEditCellRef.current && + isEmptyCellValue(rowsRef.current.find((r) => r.id === rowId)?.data[columnName]) + ) { return } setSelectionFocus(null) setIsColumnSelection(false) - // Types with a bounded value edit in place (calendar picker, numeric - // input); only free-form prose opens the big expanded popover. - if (column && !columnTypeOf(column).expandable && canEditCellRef.current) { + // Editors open for anyone with write access. On an update-locked table + // they open read-only, so the value can still be selected and copied; + // `handleInlineSave` stays as a backstop that refuses any change with the + // lock explanation. Types with a bounded value edit in place (calendar + // picker, numeric input); only free-form prose opens the big expanded + // popover. + if (column && !columnTypeOf(column).expandable && canEditRef.current) { setEditingCell({ rowId, columnName }) setInitialCharacter(null) return @@ -2966,14 +3003,22 @@ export function TableGrid({ if (e.shiftKey && e.key === 'Enter') { if (!canEditRef.current) return - // Same manual-add path as the Add row button, so it owes the same - // explanation rather than silently doing nothing on a locked table. + const row = currentRows[anchor.rowIndex] + // Mirrors handleAddRowClick; keep the two new-row paths in sync. + if ( + row && + canInsertFullRowRef.current && + needsAddRowForm(updateLockedRef.current, schemaColumnsRef.current) + ) { + e.preventDefault() + onOpenAddRowModalRef.current({ afterRowId: row.id }) + return + } if (!canManualAddRowRef.current) { e.preventDefault() onBlockedActionRef.current('add-row') return } - const row = currentRows[anchor.rowIndex] if (!row) return e.preventDefault() const position = row.position + 1 @@ -2997,23 +3042,24 @@ export function TableGrid({ if (e.key === 'Enter' || e.key === 'F2') { if (!canEditRef.current) return e.preventDefault() - // The primary keyboard edit path — same lock notice as double-click and - // Space, rather than a keypress that silently does nothing. - if (updateLockedRef.current) { - onBlockedActionRef.current('edit-cell') - return - } - if (!canEditCellRef.current) return const col = cols[anchor.colIndex] if (!col) return const row = currentRows[anchor.rowIndex] if (!row) return + // The keyboard twin of double-click: the editor opens read-only on an + // update-locked table. A toggle writes on the keypress itself, so it + // explains the lock here instead. if (columnTypeOf(col).editor === 'toggle') { + if (updateLockedRef.current) { + onBlockedActionRef.current('edit-cell') + return + } toggleBooleanCellRef.current(row.id, col.key, row.data[col.key]) return } + if (!canEditCellRef.current && isEmptyCellValue(row.data[col.key])) return setEditingCell({ rowId: row.id, columnName: col.key }) setInitialCharacter(null) return @@ -3022,8 +3068,8 @@ export function TableGrid({ if (e.key === ' ' && !e.shiftKey) { if (!canEditRef.current) return e.preventDefault() - // Space opens the same row editor as double-click, so it follows the - // update lock too — otherwise the form fills in and only 423s on save. + // Space opens the whole-row editor, which explains the update lock up + // front — otherwise the form fills in and only 423s on save. if (updateLockedRef.current) { onBlockedActionRef.current('edit-cell') return @@ -3846,6 +3892,14 @@ export function TableGrid({ } const changed = !cellValuesEqual(oldValue, normalizedValue, column) + if (changed && updateLockedRef.current) { + onBlockedActionRef.current('edit-cell') + setEditingCell(null) + setInitialCharacter(null) + scrollRef.current?.focus({ preventScroll: true }) + return + } + if (changed) { pushUndoRef.current({ type: 'update-cell', @@ -4746,6 +4800,12 @@ export function TableGrid({ groupName={workflowGroupById.get(g.groupId)?.name} onSelectGroup={handleGroupSelect} onOpenConfig={() => handleConfigureWorkflowGroup(g.groupId)} + schemaLockedReason={ + locks?.schemaLocked ? LOCK_TOOLTIPS.schema : undefined + } + deleteLockedReason={ + locks?.deleteLocked ? LOCK_TOOLTIPS.delete : undefined + } onRunColumn={userPermissions.canEdit ? handleRunColumn : undefined} hasActiveFilter={Boolean(effectiveFilter)} selectedRowIds={selectedRowIds} @@ -4887,6 +4947,12 @@ export function TableGrid({ workflowGroups={tableWorkflowGroups} sourceInfo={columnSourceInfo.get(column.key)} onOpenConfig={handleConfigureColumn} + schemaLockedReason={ + locks?.schemaLocked ? LOCK_TOOLTIPS.schema : undefined + } + deleteLockedReason={ + locks?.deleteLocked ? LOCK_TOOLTIPS.delete : undefined + } onViewWorkflow={handleViewWorkflow} onSortColumn={onSortColumn} onClearSort={onClearSort} @@ -4907,7 +4973,6 @@ export function TableGrid({ trigger='inline-header' disabled={addColumnMutation.isPending} blocked={!canMutateSchema} - onBlocked={() => onBlockedAction('add-column')} onPickType={handleAddColumnOfType} onPickWorkflow={handleAddWorkflowColumn} onPickEnrichment={onOpenEnrichments} @@ -4962,6 +5027,7 @@ export function TableGrid({ initialCharacter={ editingCell?.rowId === row.id ? initialCharacter : null } + editorsReadOnly={Boolean(locks?.updateLocked)} pendingCellValue={ pendingUpdate && pendingUpdate.rowId === row.id ? pendingUpdate.data @@ -5050,7 +5116,10 @@ export function TableGrid({ )} {!isLoadingTable && !isLoadingRows && userPermissions.canEdit && ( - + )} @@ -5092,7 +5161,7 @@ export function TableGrid({ hasWorkflowColumns={hasWorkflowColumns} workflowCellScoped={Boolean(contextMenuGroupId)} disableEdit={!canEditCell} - disableInsert={!canManualAddRow} + disableInsert={!canInsertFullRow} disableDuplicate={!canInsertFullRow} disableDelete={!canDeleteRow} onAddToChat={addToChatRowIds.length > 0 ? handleAddSelectionToChat : undefined} @@ -5108,7 +5177,8 @@ export function TableGrid({ rows={rows} columns={displayColumns} onSave={handleInlineSave} - canEdit={canEditCell} + canEdit={userPermissions.canEdit} + saveBlockedReason={locks?.updateLocked ? LOCK_TOOLTIPS.update : undefined} scrollContainer={scrollRef.current} /> diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-primitives.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-primitives.tsx index f05d7332524..fab52045ad8 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-primitives.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-primitives.tsx @@ -1,8 +1,8 @@ 'use client' import React from 'react' -import { Button, Checkbox, cn } from '@sim/emcn' -import { Plus } from '@sim/emcn/icons' +import { Button, Checkbox, cn, Tooltip } from '@sim/emcn' +import { Lock, Plus } from '@sim/emcn/icons' import { ADD_COL_WIDTH, CELL_HEADER_CHECKBOX, COL_WIDTH } from './constants' import type { DisplayColumn } from './types' @@ -58,19 +58,42 @@ export const SelectAllCheckbox = React.memo(function SelectAllCheckbox({ ) }) -export const AddRowButton = React.memo(function AddRowButton({ onClick }: { onClick: () => void }) { +interface AddRowButtonProps { + onClick: () => void + blockedReason?: string +} + +export const AddRowButton = React.memo(function AddRowButton({ + onClick, + blockedReason, +}: AddRowButtonProps) { + const Icon = blockedReason ? Lock : Plus + const button = ( + + ) return (
- + {blockedReason ? ( + + {button} + {blockedReason} + + ) : ( + button + )}
) }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/lock-copy.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/lock-copy.ts index 6f559998503..8ff9790506d 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/lock-copy.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/lock-copy.ts @@ -1,18 +1,22 @@ /** - * Single source of truth for lock vocabulary shared by the lock settings modal + * Single source of truth for lock vocabulary shared by the Table Security modal * and the lock toasts (the on-open announcement and blocked actions). Kept out of * `lib/table/mutation-locks.ts` — that module is server-tainted (importing it * from a client component pulls `next/headers` into the browser bundle). + * + * The modal speaks Allow/Deny, so this copy does too: a set lock reads as its + * action being "disabled", never as a separate "locked" state. */ import type { TableLockKind, TableLocks } from '@/lib/table/types' export interface LockField { - /** The `TableLocks` flag this row toggles. */ + /** The `TableLocks` flag this row controls. */ key: keyof TableLocks kind: TableLockKind - /** The action being locked, phrased to read after "Lock " and inside a list. */ + /** The action being denied, phrased to read inside a list. */ noun: string + label: string hint: string } @@ -20,71 +24,61 @@ export const LOCK_FIELDS: LockField[] = [ { key: 'insertLocked', kind: 'insert', - noun: 'adding rows', - hint: 'On: no new rows can be added — by anyone, including CSV import, the API, workflow blocks, and Sim.', + noun: 'inserting rows', + label: 'Inserting Rows', + hint: 'Allow new rows to be added, including through CSV imports, the API, workflows, and Sim. Deny blocks new rows from every surface.', }, { key: 'updateLocked', kind: 'update', - noun: 'editing rows', - hint: 'On: existing cell values cannot be changed. Workflow and enrichment columns still populate.', + noun: 'updating rows', + label: 'Updating Rows', + hint: 'Allow existing cell values to be changed. Deny blocks edits from every surface. Workflow and enrichment columns still populate.', }, { key: 'deleteLocked', kind: 'delete', noun: 'deleting rows', - hint: 'On: rows cannot be deleted, and the table cannot be archived.', + label: 'Deleting Rows', + hint: 'Allow rows to be deleted and the table to be archived. Deny blocks those actions and destructive column changes.', }, { key: 'schemaLocked', kind: 'schema', - noun: 'changing columns', - hint: 'On: columns cannot be added, renamed, retyped, or removed.', + noun: 'changing the table schema', + label: 'Changing Table Schema', + hint: 'Allow columns to be added, renamed, retyped, or removed. Deny blocks schema changes. Removing or retyping columns also requires Deleting Rows set to Allow.', }, ] -/** The locked verbs' nouns, in display order. Empty when nothing is locked. */ -export function lockedNouns(locks: TableLocks): string[] { - return LOCK_FIELDS.filter((f) => locks[f.key]).map((f) => f.noun) -} - /** - * Plain-language summary of a lock set — the named mode when the combination - * matches one, otherwise a list of what is locked. + * Tooltip for a control a denied action disables. One sentence per lock kind so + * the grid chrome (New row, New column, the column menu, the expanded editor's + * Save) all name the same Table Security row. */ -export function describeLocks(locks: TableLocks): { name: string; detail: string } { - const locked = lockedNouns(locks) - if (locked.length === 0) { - return { name: 'Unlocked', detail: 'anyone with edit access can change this table.' } - } - if (locked.length === LOCK_FIELDS.length) { - return { name: 'Read-only', detail: 'no one can change this table’s rows or columns.' } - } - // Append-only describes the row semantics — adding is the only thing left. - // A schema lock on top doesn't change that, so it keeps the name and is - // called out in the detail rather than demoted to the generic case. - if (!locks.insertLocked && locks.updateLocked && locks.deleteLocked) { - return { - name: 'Append-only', - detail: locks.schemaLocked - ? 'rows can be added, but not edited or deleted, and columns are locked.' - : 'rows can be added, but not edited or deleted.', - } - } - return { name: 'Locked', detail: `${locked.join(', ')} locked.` } +export const LOCK_TOOLTIPS: Record = { + insert: 'Inserting rows is disabled in Table Security.', + update: 'Updating rows is disabled in Table Security.', + delete: 'Deleting rows is disabled in Table Security.', + schema: 'Changing the table schema is disabled in Table Security.', +} + +/** The denied actions' nouns, in display order. Empty when everything is allowed. */ +export function lockedNouns(locks: TableLocks): string[] { + return LOCK_FIELDS.filter((f) => locks[f.key]).map((f) => f.noun) } /** * Why a locked-table notice was raised. `'status'` is the informational case - * (the announcement shown once when a locked table is opened); the rest are + * (the announcement shown once when a restricted table is opened); the rest are * actions the user just tried and couldn't do. */ export type BlockedTableAction = 'add-row' | 'add-column' | 'delete-column' | 'edit-cell' | 'status' /** - * Copy for the action the user attempted. Explains what is blocked and — for - * the append-only manual-entry case — what to do instead, since that one is - * blocked by the *update* lock rather than the insert lock. + * Copy for the action the user attempted, in the modal's vocabulary: each + * notice names the Table Security row that denies it, so the reader knows which + * setting an admin has to flip. */ export function describeBlockedAction( action: BlockedTableAction, @@ -92,46 +86,40 @@ export function describeBlockedAction( ): { title: string; text: string } { switch (action) { case 'add-row': - if (locks.insertLocked) { - return { - title: 'Adding rows is locked', - text: 'No new rows can be added until an admin unlocks this table.', - } - } return { - title: 'This table is append-only', - text: 'Rows can’t be edited once added, so typing one into the grid is unavailable. Import a CSV, or add rows from the API, a workflow, or Sim.', + title: 'Inserting rows is disabled', + text: 'An admin has set Inserting Rows to Deny in Table Security.', } case 'add-column': return { - title: 'Changing columns is locked', - text: 'Columns can’t be added, renamed, retyped, or removed until an admin unlocks this table.', + title: 'Changing the table schema is disabled', + text: 'An admin has set Changing Table Schema to Deny in Table Security, so columns can’t be added, renamed, retyped, or removed.', } case 'delete-column': - // Reachable with the schema lock off but the delete lock on — removing a - // column clears its value from every row, so it needs both. + // Reachable with Changing Table Schema on Allow but Deleting Rows on Deny — + // removing a column clears its value from every row, so it needs both. return locks.schemaLocked ? { - title: 'Changing columns is locked', - text: 'Columns can’t be added, renamed, retyped, or removed until an admin unlocks this table.', + title: 'Changing the table schema is disabled', + text: 'An admin has set Changing Table Schema to Deny in Table Security, so columns can’t be added, renamed, retyped, or removed.', } : { - title: 'Deleting columns is locked', - text: 'Removing a column deletes its value from every row, so it’s blocked while deleting is locked.', + title: 'Deleting rows is disabled', + text: 'Removing a column clears its value from every row, so it needs Deleting Rows set to Allow in Table Security.', } case 'edit-cell': return { - title: 'Editing rows is locked', - text: 'Existing cell values can’t be changed until an admin unlocks this table.', + title: 'Updating rows is disabled', + text: 'An admin has set Updating Rows to Deny in Table Security.', } case 'status': { const nouns = lockedNouns(locks) return { - title: 'Table locks', + title: 'Table Security', text: nouns.length > 0 - ? `An admin has locked ${nouns.join(', ')} on this table.` - : 'Nothing is locked on this table.', + ? `An admin has set ${nouns.join(', ')} to Deny on this table.` + : 'Every action is allowed on this table.', } } } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index a6fbe933f0c..2f3a338ff5b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -93,14 +93,19 @@ import { import { COLUMN_SIDEBAR_WIDTH } from './components/table-grid/constants' import { columnTypeIcon } from './components/table-grid/headers' import { useTable, useTableEventStream, useTableRoom } from './hooks' -import { type BlockedTableAction, describeBlockedAction, lockedNouns } from './lock-copy' +import { + type BlockedTableAction, + describeBlockedAction, + LOCK_TOOLTIPS, + lockedNouns, +} from './lock-copy' import { ALL_VIEW_PARAM, DEFAULT_TABLE_DETAIL_SORT_DIRECTION, tableDetailParsers, tableDetailUrlKeys, } from './search-params' -import type { QueryOptions } from './types' +import type { QueryOptions, RowInsertTarget } from './types' import { generateColumnName } from './utils' const logger = createLogger('Table') @@ -226,6 +231,7 @@ export function Table({ const blockedToastIdRef = useRef(null) const [isImportCsvOpen, setIsImportCsvOpen] = useState(false) const [editingRow, setEditingRow] = useState(null) + const [addRowTarget, setAddRowTarget] = useState(null) const [deletingRows, setDeletingRows] = useState([]) const [deletingAll, setDeletingAll] = useState<{ excludeRowIds: string[] @@ -295,6 +301,7 @@ export function Table({ }, []) const onCloseSlideout = () => dispatch({ type: 'CLOSE' }) const onOpenRowModal = (row: TableRowType) => setEditingRow(row) + const onOpenAddRowModal = (insertAt: RowInsertTarget = {}) => setAddRowTarget(insertAt) // useCallback because is memo-wrapped — these flow into // the breadcrumbs / headerActions memos, whose identity drives that re-render. const onRequestDeleteTable = useCallback(() => setShowDeleteTableConfirm(true), []) @@ -1303,7 +1310,7 @@ export function Table({ ...(userPermissions.canAdmin ? [ { - label: 'Lock settings', + label: 'Table Security', icon: Lock, onClick: () => setShowLockSettings(true), }, @@ -1355,7 +1362,7 @@ export function Table({ description: text, ...(canOpenLockSettings ? { - action: { label: 'Lock settings', onClick: () => setShowLockSettings(true) }, + action: { label: 'Table Security', onClick: () => setShowLockSettings(true) }, // An action would otherwise pin the toast open until dismissed. duration: BLOCKED_TOAST_MS, } @@ -1392,7 +1399,7 @@ export function Table({ ) // A toast's action is captured when it is created, so a viewer who loses - // admin access mid-toast would keep a Lock settings button that opens + // admin access mid-toast would keep a Table Security button that opens // nothing. Dismiss on that transition only — a viewer who never had access // has a legitimate action-less notice that must survive. const couldOpenLockSettingsRef = useRef(canOpenLockSettings) @@ -1434,7 +1441,6 @@ export function Table({ trigger='header' disabled={false} blocked={!canMutateSchema} - onBlocked={() => showBlockedToast('add-column')} onPickType={handleAddColumnOfType} onPickWorkflow={handleAddWorkflowColumn} onPickEnrichment={onOpenEnrichments} @@ -1610,6 +1616,7 @@ export function Table({ onOpenExecutionDetails={onOpenExecutionDetails} onOpenEnrichmentDetails={onOpenEnrichmentDetails} onOpenRowModal={onOpenRowModal} + onOpenAddRowModal={onOpenAddRowModal} onRequestDeleteRows={onRequestDeleteRows} onRequestDeleteAllByFilter={onRequestDeleteAllByFilter} onRequestDeleteColumns={onRequestDeleteColumns} @@ -1714,6 +1721,12 @@ export function Table({ workspaceId={workspaceId} tableId={tableId} onColumnRename={onColumnRename} + readOnly={!canMutateSchema} + readOnlyReason={ + tableData?.locks.schemaLocked + ? LOCK_TOOLTIPS.schema + : 'You don’t have permission to change columns.' + } /> )} + {addRowTarget && tableData && ( + setAddRowTarget(null)} + table={tableData} + insertAt={addRowTarget} + onSuccess={() => setAddRowTarget(null)} + /> + )} {editingRow && tableData && ( setShowLockSettings(false)} workspaceId={workspaceId} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts index 34618913acf..fa026c6ef4d 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts @@ -1,3 +1,4 @@ +import type { InsertTableRowBodyInput } from '@/lib/api/contracts/tables' import type { SortSpec, TablePredicate, TableRow } from '@/lib/table' /** @@ -36,3 +37,9 @@ export interface EditingCell { columnName: string columnKey?: string } + +/** Where a new row goes; an empty target appends it to the end of the table. */ +export type RowInsertTarget = Pick< + InsertTableRowBodyInput, + 'position' | 'afterRowId' | 'beforeRowId' +> diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index 22b4359b9f8..25a9fcb56ad 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -159,6 +159,12 @@ export type TableRowsResponse = Pick< interface RowMutationContext { workspaceId: string tableId: string + /** + * Suppresses the error toast for callers that render the failure themselves — + * the row modal shows it inline, and two copies of the same sentence read as + * two separate failures. The cache self-heal on a 423 still runs. + */ + suppressErrorToast?: boolean } type UpdateTableRowParams = Pick & @@ -809,16 +815,22 @@ function notifyRowWriteError(error: Error, onUpgrade: () => void): void { function handleTableLockRejection( error: unknown, queryClient: ReturnType, - tableId: string + tableId: string, + options?: { silent?: boolean } ): boolean { if (!isApiClientError(error) || error.status !== 423) return false void queryClient.invalidateQueries({ queryKey: tableKeys.detail(tableId), exact: true }) void queryClient.invalidateQueries({ queryKey: tableKeys.lists() }) - toast.error(error.message, { duration: 5000 }) + // `silent` only drops the toast; the refetches above are what un-stale the grid. + if (!options?.silent) toast.error(error.message, { duration: 5000 }) return true } -export function useCreateTableRow({ workspaceId, tableId }: RowMutationContext) { +export function useCreateTableRow({ + workspaceId, + tableId, + suppressErrorToast, +}: RowMutationContext) { const queryClient = useQueryClient() const router = useRouter() @@ -866,7 +878,9 @@ export function useCreateTableRow({ workspaceId, tableId }: RowMutationContext) }) }, onError: (error) => { - if (handleTableLockRejection(error, queryClient, tableId)) return + if (handleTableLockRejection(error, queryClient, tableId, { silent: suppressErrorToast })) + return + if (suppressErrorToast) return notifyRowWriteError(error, () => router.push(buildUpgradeHref(workspaceId, 'tables'))) }, onSettled: () => { @@ -1065,7 +1079,11 @@ export function useBatchCreateTableRows({ workspaceId, tableId }: RowMutationCon * Update a single row in a table. * Uses optimistic updates for instant UI feedback on inline cell edits. */ -export function useUpdateTableRow({ workspaceId, tableId }: RowMutationContext) { +export function useUpdateTableRow({ + workspaceId, + tableId, + suppressErrorToast, +}: RowMutationContext) { const queryClient = useQueryClient() return useMutation({ @@ -1150,8 +1168,10 @@ export function useUpdateTableRow({ workspaceId, tableId }: RowMutationContext) if (context?.didBumpRunState) { queryClient.setQueryData(tableKeys.activeDispatches(tableId), context.runStateSnapshot) } - if (handleTableLockRejection(error, queryClient, tableId)) return + if (handleTableLockRejection(error, queryClient, tableId, { silent: suppressErrorToast })) + return if (isValidationError(error)) return + if (suppressErrorToast) return toast.error(error.message, { duration: 5000 }) }, }) @@ -1234,7 +1254,11 @@ export function useBatchUpdateTableRows({ workspaceId, tableId }: RowMutationCon /** * Delete a single row from a table. */ -export function useDeleteTableRow({ workspaceId, tableId }: RowMutationContext) { +export function useDeleteTableRow({ + workspaceId, + tableId, + suppressErrorToast, +}: RowMutationContext) { const queryClient = useQueryClient() return useMutation({ @@ -1245,8 +1269,10 @@ export function useDeleteTableRow({ workspaceId, tableId }: RowMutationContext) }) }, onError: (error) => { - if (handleTableLockRejection(error, queryClient, tableId)) return + if (handleTableLockRejection(error, queryClient, tableId, { silent: suppressErrorToast })) + return if (isValidationError(error)) return + if (suppressErrorToast) return toast.error(error.message, { duration: 5000 }) }, onSettled: () => { @@ -1259,7 +1285,11 @@ export function useDeleteTableRow({ workspaceId, tableId }: RowMutationContext) * Delete multiple rows from a table. * Returns both deleted ids and failure details for partial-failure UI. */ -export function useDeleteTableRows({ workspaceId, tableId }: RowMutationContext) { +export function useDeleteTableRows({ + workspaceId, + tableId, + suppressErrorToast, +}: RowMutationContext) { const queryClient = useQueryClient() return useMutation({ @@ -1294,8 +1324,10 @@ export function useDeleteTableRows({ workspaceId, tableId }: RowMutationContext) return { deletedRowIds } }, onError: (error) => { - if (handleTableLockRejection(error, queryClient, tableId)) return + if (handleTableLockRejection(error, queryClient, tableId, { silent: suppressErrorToast })) + return if (isValidationError(error)) return + if (suppressErrorToast) return toast.error(error.message, { duration: 5000 }) }, onSettled: () => { diff --git a/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx b/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx index 9370d2fa6ea..d6bec05ca7d 100644 --- a/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx +++ b/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx @@ -1,12 +1,13 @@ 'use client' -import { forwardRef, useState } from 'react' +import { forwardRef, useContext, useState } from 'react' import * as PopoverPrimitive from '@radix-ui/react-popover' import { ChevronDown } from '../../icons' import { cn } from '../../lib/cn' import { Calendar, formatDateLabel, formatDateRangeLabel } from '../calendar/calendar' import { chipVariants, TRIGGER_BORDER_CLASS } from '../chip/chip' import { chipContentLabelClass, chipIconSlotClass } from '../chip/chip-chrome' +import { InsideModalContext } from '../modal/modal' import { OverflowText } from '../overflow-text/overflow-text' import { POPOVER_ANIMATION_CLASSES } from '../popover/popover-animation' @@ -45,6 +46,10 @@ interface ChipDatePickerSingleProps extends ChipDatePickerBaseProps { * defaults to the runtime's local day (mirrors `Calendar`'s `today`). */ today?: string + /** Adds a time-of-day field, emitting `YYYY-MM-DDTHH:mm`; the popover stays open while it is set. */ + showTime?: boolean + /** Label beside the time field when `showTime` is set. Defaults to `Time`. */ + timeLabel?: string } interface ChipDatePickerRangeProps extends ChipDatePickerBaseProps { @@ -67,7 +72,8 @@ export type ChipDatePickerProps = ChipDatePickerSingleProps | ChipDatePickerRang * `chipVariants` (filled + border) and the owned chevron for visual parity with * the other chip field controls; `ghost` renders the bare toolbar pill instead. * - * `mode='single'` (default) commits on day click. `mode='range'` opens the + * `mode='single'` (default) commits on day click; with `showTime` it also emits the + * time of day and stays open while it is set. `mode='range'` opens the * range calendar — start/end staged behind Clear/Cancel/Apply, with optional * time-of-day inputs — and commits via `onRangeChange`. * @@ -89,6 +95,12 @@ const ChipDatePicker = forwardRef( className, } = props + /** + * Inside a modal dialog the calendar must be modal too: a non-modal popover + * portaled to `body` inherits the dialog's `pointer-events: none` body lock + * and cannot be clicked. Outside dialogs it stays non-modal. + */ + const insideModal = useContext(InsideModalContext) const [open, setOpen] = useState(false) const triggerText = @@ -98,7 +110,7 @@ const ChipDatePicker = forwardRef( : formatDateLabel(props.value)) return ( - +