fix(workspaces): gate workspace creation before the transaction, not inside it

`lockWorkspaceCreationContext` called `isOrganizationCapabilityWithheld` with
no executor, so it ran on the global `db` pool while the creation transaction
already held a pooled connection and three advisory locks. That is the pool
deadlock `packages/db/tx-tripwire.ts` exists to detect — it is firing in
staging right now, four times per request, 144 lines across 36 requests in six
hours, all on POST /api/workspaces.

It also added up to four SEQUENTIAL reads to a lock hold that serializes every
organization mutation, which is what pushed concurrent creates past the 5s
lock_timeout and answered them as a generic 500.

Nothing is given up by moving the gate out. The re-read was never serialized
against permission-group writes: those take `permission_group:<org>` while
creation takes `organization-mutation:<org>`, which are different advisory-lock
ids and never contend. What the check actually provides is recency against the
caller's earlier preflight, and that holds identically microseconds earlier.
The governing organization is `organizationId ?? observedOrganizationId`, both
known before the transaction, and `lockWorkspaceCreationContext` still refuses
to commit unless live membership still equals `observedOrganizationId` — so a
verdict computed outside can never be applied to a different organization.

The comment justifying the old placement was also wrong: for POST
/api/workspaces the preflight and the insert are the same request, not separate
ones.

Rejected alternatives: threading an executor through the resolver would touch
seven functions across four files, requires exporting the uncached
`resolveOrganizationEnterprisePlan`, and bypassing the React cache is a
fail-OPEN semantic change. `runOutsideTransactionContext` only exits the
ambient context — it would silence the tripwire while leaving the second pooled
connection checkout in place.

Adds create.test.ts pinning the ordering, because no existing test can catch a
regression here: vitest.setup.ts mocks @sim/db globally, so the real pool
instrumentation never runs and the tripwire cannot fire in any apps/sim test.

Author: waleedlatif1

apps/sim/lib/workspaces/create.test.ts

@@ -0,0 +1,99 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockTransaction,
+  mockAssertWorkspaceCreationCapability,
+  mockLockWorkspaceCreationContext,
+  mockGetWorkspaceInvitePolicy,
+} = vi.hoisted(() => ({
+  mockTransaction: vi.fn(),
+  mockAssertWorkspaceCreationCapability: vi.fn(),
+  mockLockWorkspaceCreationContext: vi.fn(),
+  mockGetWorkspaceInvitePolicy: vi.fn(),
+}))
+
+vi.mock('@sim/db', () => ({
+  db: { transaction: mockTransaction },
+}))
+
+vi.mock('@/lib/workspaces/policy', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@/lib/workspaces/policy')>()
+  return {
+    ...actual,
+    assertWorkspaceCreationCapability: mockAssertWorkspaceCreationCapability,
+    lockWorkspaceCreationContext: mockLockWorkspaceCreationContext,
+    getWorkspaceInvitePolicy: mockGetWorkspaceInvitePolicy,
+  }
+})
+
+import { createWorkspace } from '@/lib/workspaces/create'
+import { WORKSPACE_MODE, WorkspaceCreationCapabilityWithheldError } from '@/lib/workspaces/policy'
+
+const params = {
+  userId: 'creator-1',
+  observedOrganizationId: 'org-1',
+  name: 'Test Workspace',
+  organizationId: 'org-1',
+  workspaceMode: WORKSPACE_MODE.ORGANIZATION,
+  billedAccountUserId: 'creator-1',
+}
+
+describe('createWorkspace capability gate placement', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockGetWorkspaceInvitePolicy.mockResolvedValue({})
+  })
+
+  /**
+   * The gate resolves the organization's entitlement and default group — up to
+   * four sequential reads. Run inside the transaction it checked out a SECOND
+   * pooled connection while three advisory locks were held, which is what
+   * `packages/db/tx-tripwire.ts` fires on and what pushed concurrent creates
+   * past the 5s `lock_timeout` into a generic 500.
+   *
+   * Ordering is the whole fix, so it is asserted directly rather than inferred
+   * from the absence of a tripwire warning: nothing else in the unit suite can
+   * catch a regression here, because `vitest.setup.ts` mocks `@sim/db` globally
+   * and the real pool instrumentation never runs.
+   */
+  it('gates before opening the transaction, not inside it', async () => {
+    /**
+     * The callback is deliberately NOT invoked. This pins the ORDER of the gate
+     * against `db.transaction`, so the transaction's own internals stay out of
+     * the assertion and cannot make it fail for an unrelated reason.
+     */
+    mockTransaction.mockResolvedValue({
+      id: 'ws-1',
+      name: params.name,
+      organizationId: 'org-1',
+      workspaceMode: WORKSPACE_MODE.ORGANIZATION,
+      billedAccountUserId: 'creator-1',
+      ownerId: 'creator-1',
+    })
+
+    await createWorkspace(params)
+
+    expect(mockAssertWorkspaceCreationCapability).toHaveBeenCalledWith({
+      organizationId: 'org-1',
+      observedOrganizationId: 'org-1',
+    })
+    expect(mockAssertWorkspaceCreationCapability.mock.invocationCallOrder[0]).toBeLessThan(
+      mockTransaction.mock.invocationCallOrder[0]
+    )
+  })
+
+  /** A withheld capability must refuse before any transaction is opened at all. */
+  it('never opens a transaction when the capability is withheld', async () => {
+    mockAssertWorkspaceCreationCapability.mockRejectedValue(
+      new WorkspaceCreationCapabilityWithheldError()
+    )
+
+    await expect(createWorkspace(params)).rejects.toBeInstanceOf(
+      WorkspaceCreationCapabilityWithheldError
+    )
+    expect(mockTransaction).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/workspaces/create.ts

@@ -8,6 +8,7 @@ import { buildDefaultWorkflowArtifacts } from '@/lib/workflows/defaults'
 import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils'
 import { getRandomWorkspaceColor } from '@/lib/workspaces/colors'
 import {
+  assertWorkspaceCreationCapability,
   getWorkspaceInvitePolicy,
   lockWorkspaceCreationContext,
   resolveInviteFlags,
@@ -59,6 +60,15 @@ export function emitWorkspaceCreatedPlatformEvent(params: {
  * that snapshot under the shared organization/user locks before inserting the
  * workspace, owner permission, and optional starter workflow atomically.
  */
+/**
+ * The workspace.create capability gate is the CALLER's responsibility, because
+ * it must not run inside this transaction — see
+ * {@link assertWorkspaceCreationCapability}. `createWorkspace` calls it before
+ * opening its transaction; the only other entry point,
+ * {@link createDefaultPersonalWorkspaceInTransaction}, passes both
+ * `organizationId` and `observedOrganizationId` as `null` by construction, so
+ * the gate is a no-op on that path rather than a skipped check.
+ */
 export async function createWorkspaceInTransaction(
   tx: DbOrTx,
   {
@@ -167,6 +177,18 @@ export async function createWorkspaceInTransaction(
 
 /** Creates a workspace through the canonical lock-and-insert transaction. */
 export async function createWorkspace(params: CreateWorkspaceParams) {
+  /**
+   * Gate before opening the transaction, never inside it — see
+   * {@link assertWorkspaceCreationCapability} for why the lock never protected
+   * this read. Its `WorkspaceCreationCapabilityWithheldError` is thrown from the
+   * same call stack as before, so `app/api/workspaces/route.ts` still projects
+   * the identical capability refusal.
+   */
+  await assertWorkspaceCreationCapability({
+    organizationId: params.organizationId,
+    observedOrganizationId: params.observedOrganizationId,
+  })
+
   let created: CreatedWorkspace
   try {
     created = await db.transaction((tx) => createWorkspaceInTransaction(tx, params))

apps/sim/lib/workspaces/policy.test.ts

@@ -47,6 +47,7 @@ vi.mock('@/lib/billing/core/plan', () => ({
 }))
 
 import {
+  assertWorkspaceCreationCapability,
   getOrganizationOwnerId,
   getWorkspaceCreationPolicy,
   getWorkspaceInvitePolicy,
@@ -77,6 +78,67 @@ describe('getOrganizationOwnerId', () => {
   })
 })
 
+/**
+ * The capability gate moved OUT of `lockWorkspaceCreationContext` so it no
+ * longer runs on a second pooled connection inside the lock-holding
+ * transaction. These three cases are the ones that lived there, retargeted
+ * rather than rewritten — the gate's behavior is unchanged, only where it runs.
+ */
+describe('assertWorkspaceCreationCapability', () => {
+  it('rejects when the group withheld workspace creation after the preflight', async () => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    setEnvFlags({ isBillingEnabled: false })
+    mockGetUserPermissionConfigForOrganization.mockResolvedValue({
+      disableWorkspaceCreation: true,
+    })
+
+    await expect(
+      assertWorkspaceCreationCapability({
+        organizationId: 'org-1',
+        observedOrganizationId: 'org-1',
+      })
+    ).rejects.toBeInstanceOf(WorkspaceCreationCapabilityWithheldError)
+    expect(mockGetUserPermissionConfigForOrganization).toHaveBeenCalledWith('org-1')
+  })
+
+  /**
+   * A personal workspace is precisely the escape from a scoped group, so the
+   * gate reads the caller's membership organization even when the workspace
+   * being inserted carries none — the same organization the preflight used.
+   * `observedOrganizationId` carries that membership, and
+   * `lockWorkspaceCreationContext` still refuses to commit if live membership
+   * has since diverged from it.
+   */
+  it('rejects a personal workspace when the membership organization withheld creation', async () => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    mockGetUserPermissionConfigForOrganization.mockResolvedValue({
+      disableWorkspaceCreation: true,
+    })
+
+    await expect(
+      assertWorkspaceCreationCapability({
+        organizationId: null,
+        observedOrganizationId: 'org-1',
+      })
+    ).rejects.toBeInstanceOf(WorkspaceCreationCapabilityWithheldError)
+  })
+
+  it('leaves an unaffiliated creator alone, with no group to read', async () => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+
+    await expect(
+      assertWorkspaceCreationCapability({
+        organizationId: null,
+        observedOrganizationId: null,
+      })
+    ).resolves.toBeUndefined()
+    expect(mockGetUserPermissionConfigForOrganization).not.toHaveBeenCalled()
+  })
+})
+
 describe('lockWorkspaceCreationContext', () => {
   it('locks the destination organization and user before rejecting a stale org-mode policy', async () => {
     vi.clearAllMocks()
@@ -138,75 +200,6 @@ describe('lockWorkspaceCreationContext', () => {
     )
   })
 
-  /**
-   * The preflight in `getWorkspaceCreationPolicy` and the insert are separate
-   * requests. A group that withheld creation in between has to be caught under
-   * the lock, or the in-flight create lands a workspace that carries no
-   * `permissionGroupWorkspace` row to bring it back under the regime.
-   */
-  it('rejects when the group withheld workspace creation after the preflight', async () => {
-    vi.clearAllMocks()
-    resetDbChainMock()
-    setEnvFlags({ isBillingEnabled: false })
-    mockAcquireOrganizationUserMutationLocks.mockResolvedValue(undefined)
-    mockGetUserOrganization.mockResolvedValue({ organizationId: 'org-1', role: 'admin' })
-    mockGetUserPermissionConfigForOrganization.mockResolvedValue({
-      disableWorkspaceCreation: true,
-    })
-    queueTableRows(member, [{ userId: 'owner-1' }])
-    const tx = dbChainMock.db as unknown as DbOrTx
-
-    await expect(
-      lockWorkspaceCreationContext(tx, {
-        userId: 'creator-1',
-        organizationId: 'org-1',
-        observedOrganizationId: 'org-1',
-      })
-    ).rejects.toBeInstanceOf(WorkspaceCreationCapabilityWithheldError)
-    expect(mockGetUserPermissionConfigForOrganization).toHaveBeenCalledWith('org-1')
-  })
-
-  /**
-   * A personal workspace is precisely the escape from a scoped group, so the
-   * re-check reads the caller's membership organization even when the workspace
-   * being inserted carries none — the same organization the preflight used.
-   */
-  it('rejects a personal workspace when the membership organization withheld creation', async () => {
-    vi.clearAllMocks()
-    resetDbChainMock()
-    mockAcquireOrganizationUserMutationLocks.mockResolvedValue(undefined)
-    mockGetUserOrganization.mockResolvedValue({ organizationId: 'org-1', role: 'member' })
-    mockGetUserPermissionConfigForOrganization.mockResolvedValue({
-      disableWorkspaceCreation: true,
-    })
-    const tx = dbChainMock.db as unknown as DbOrTx
-
-    await expect(
-      lockWorkspaceCreationContext(tx, {
-        userId: 'creator-1',
-        organizationId: null,
-        observedOrganizationId: 'org-1',
-      })
-    ).rejects.toBeInstanceOf(WorkspaceCreationCapabilityWithheldError)
-  })
-
-  it('leaves an unaffiliated creator alone, with no group to read', async () => {
-    vi.clearAllMocks()
-    resetDbChainMock()
-    mockAcquireOrganizationUserMutationLocks.mockResolvedValue(undefined)
-    mockGetUserOrganization.mockResolvedValue(null)
-    const tx = dbChainMock.db as unknown as DbOrTx
-
-    await expect(
-      lockWorkspaceCreationContext(tx, {
-        userId: 'creator-1',
-        organizationId: null,
-        observedOrganizationId: null,
-      })
-    ).resolves.toEqual({ billedAccountUserId: 'creator-1' })
-    expect(mockGetUserPermissionConfigForOrganization).not.toHaveBeenCalled()
-  })
-
   it('rejects when the paid org entitlement disappeared before insertion', async () => {
     vi.clearAllMocks()
     resetDbChainMock()

apps/sim/lib/workspaces/policy.ts

@@ -127,6 +127,47 @@ export class WorkspaceCreationCapabilityWithheldError extends WorkspaceCreationC
  * Returns the live billing owner. The caller must invoke this in the same
  * transaction as the workspace insert.
  */
+/**
+ * permission-group-enforced: workspace.create — the creation gate, deliberately
+ * OUTSIDE the creation transaction.
+ *
+ * This resolves the organization's entitlement and default group, which is up to
+ * four sequential reads. Running it inside {@link lockWorkspaceCreationContext}
+ * checked out a SECOND pooled connection while that transaction already held one
+ * plus three advisory locks — the pool deadlock `packages/db/tx-tripwire.ts`
+ * exists to detect — and added those round trips to a lock hold that serializes
+ * every organization mutation, which is what pushed concurrent creates past the
+ * 5s `lock_timeout` and answered them as a generic 500.
+ *
+ * Nothing is given up by moving it out. The re-read was never serialized against
+ * permission-group writes: those take `permission_group:<org>`
+ * (`organizations/[id]/permission-groups/utils.ts`) while creation takes
+ * `organization-mutation:<org>` (`billing/organizations/membership.ts`). Those
+ * are different advisory-lock ids and never contend, so holding the lock while
+ * reading bought no exclusion. What the check actually provides is RECENCY
+ * against the caller's earlier preflight, and that holds identically here.
+ *
+ * The governing organization is `organizationId ?? observedOrganizationId`,
+ * both known before the transaction. That is equivalent to the membership read
+ * this previously used, because {@link lockWorkspaceCreationContext} throws
+ * `WorkspaceCreationContextChangedError` unless live membership still equals
+ * `observedOrganizationId` — so a verdict computed here can never be applied to
+ * an organization other than the one that commits.
+ */
+export async function assertWorkspaceCreationCapability({
+  organizationId,
+  observedOrganizationId,
+}: {
+  organizationId: string | null
+  observedOrganizationId: string | null
+}): Promise<void> {
+  const governingOrganizationId = organizationId ?? observedOrganizationId
+  if (!governingOrganizationId) return
+  if (await isOrganizationCapabilityWithheld(governingOrganizationId, 'workspace.create')) {
+    throw new WorkspaceCreationCapabilityWithheldError()
+  }
+}
+
 export async function lockWorkspaceCreationContext(
   tx: DbOrTx,
   {
@@ -151,24 +192,6 @@ export async function lockWorkspaceCreationContext(
     throw new WorkspaceCreationContextChangedError()
   }
 
-  /**
-   * permission-group-enforced: workspace.create — re-read under the lock because
-   * the preflight in `getWorkspaceCreationPolicy` and the insert are separate
-   * requests: a group that withheld creation in between would otherwise still
-   * let the in-flight create land, and a new workspace carries no
-   * `permissionGroupWorkspace` row to bring it back under the regime afterwards.
-   * Governed by the same organization the preflight used — the explicit one, or
-   * the caller's membership when the workspace would be personal — so a personal
-   * workspace stays as governed here as it is there.
-   */
-  const governingOrganizationId = organizationId ?? currentMembership?.organizationId ?? null
-  if (
-    governingOrganizationId &&
-    (await isOrganizationCapabilityWithheld(governingOrganizationId, 'workspace.create'))
-  ) {
-    throw new WorkspaceCreationCapabilityWithheldError()
-  }
-
   if (!organizationId) return { billedAccountUserId: userId }
 
   if (isBillingEnabled) {