Skip to content

Commit caf82ac

Browse files
committed
feat(search): unify source access and scale indexing
1 parent 6b9ecfb commit caf82ac

334 files changed

Lines changed: 74482 additions & 9283 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/docs/content/docs/cli/credentials.mdx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ Update Credential (personal API key required)
103103
| `--service-account-json <value>` | No | Write-only Google service-account JSON key. |
104104
| `--api-token <value>` | No | Write-only provider API token. |
105105
| `--domain <value>` | No | Provider account domain. |
106+
| `--atlassian-product <value>` | No | Atlassian product to verify; defaults to Jira on create and preserves the saved product on reconnect. Accepted values: `jira`, `confluence`. |
106107
| `--signing-secret <value>` | No | Write-only webhook signing secret. |
107108
| `--bot-token <value>` | No | Write-only bot token. |
108109
| `--client-id <value>` | No | OAuth client identifier. |

‎apps/docs/content/docs/cli/reference.mdx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,7 @@ sim credentials update <credentialId> [options]
463463
| `--service-account-json <value>` | No | Write-only Google service-account JSON key. |
464464
| `--api-token <value>` | No | Write-only provider API token. |
465465
| `--domain <value>` | No | Provider account domain. |
466+
| `--atlassian-product <value>` | No | Atlassian product to verify; defaults to Jira on create and preserves the saved product on reconnect. Accepted values: `jira`, `confluence`. |
466467
| `--signing-secret <value>` | No | Write-only webhook signing secret. |
467468
| `--bot-token <value>` | No | Write-only bot token. |
468469
| `--client-id <value>` | No | OAuth client identifier. |

‎apps/docs/openapi-v2-resources.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9330,6 +9330,11 @@
93309330
"minLength": 1,
93319331
"maxLength": 2048
93329332
},
9333+
"atlassianProduct": {
9334+
"description": "Atlassian product to verify; defaults to Jira on create and preserves the saved product on reconnect.",
9335+
"type": "string",
9336+
"enum": ["jira", "confluence"]
9337+
},
93339338
"signingSecret": {
93349339
"description": "Write-only webhook signing secret.",
93359340
"writeOnly": true,

‎apps/sim/app/api/auth/oauth/utils.test.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ describe('OAuth Utils', () => {
177177
).rejects.toThrow('Failed to refresh token')
178178
})
179179

180-
it('should not attempt refresh if no refresh token', async () => {
180+
it('requires reconnection for an expired token without attempting an unavailable refresh', async () => {
181181
const mockCredential = {
182182
id: 'credential-id',
183183
accessToken: 'token',
@@ -186,10 +186,11 @@ describe('OAuth Utils', () => {
186186
providerId: 'google',
187187
}
188188

189-
const result = await refreshTokenIfNeeded('request-id', mockCredential, 'credential-id')
189+
await expect(
190+
refreshTokenIfNeeded('request-id', mockCredential, 'credential-id')
191+
).rejects.toThrow('OAuth access token expired and cannot be refreshed; reconnect the account')
190192

191193
expect(mockRefreshOAuthToken).not.toHaveBeenCalled()
192-
expect(result).toEqual({ accessToken: 'token', refreshed: false })
193194
})
194195

195196
it('keeps a legacy non-expiring Monday credential usable without refreshing it', async () => {

‎apps/sim/app/api/files/authorization.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { db } from '@sim/db'
22
import { document, knowledgeBase, workspaceFile } from '@sim/db/schema'
33
import { createLogger } from '@sim/logger'
44
import { permissionSatisfies } from '@sim/platform-authz/workspace'
5-
import { and, eq, isNull } from 'drizzle-orm'
5+
import { and, eq, isNotNull, isNull, or } from 'drizzle-orm'
66
import { NextResponse } from 'next/server'
77
import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate'
88
import {
@@ -511,6 +511,9 @@ async function hasActiveKbDocumentForKey(
511511
isNull(document.archivedAt),
512512
isNull(document.deletedAt),
513513
isNull(knowledgeBase.deletedAt),
514+
access.kind === 'system'
515+
? undefined
516+
: or(isNull(document.connectorId), isNotNull(document.contentHash)),
514517
knowledgeAccessCondition(access)
515518
)
516519
)

‎apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/access/route.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ export const PATCH = defineInternalJsonRoute({
2424
connectorId: params.connectorId,
2525
knowledgeBaseId: params.id,
2626
accessMode: body.accessMode,
27-
credentialGroupId: body.credentialGroupId,
28-
credentialGroupOptionId: body.credentialGroupOptionId,
2927
credentialId: body.credentialId,
3028
resolveBillingAttribution: (workspaceId: string) =>
3129
resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId),

‎apps/sim/app/api/knowledge/[id]/connectors/route.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,6 @@ export const POST = defineInternalJsonRoute({
5050
sourceConfig: body.sourceConfig,
5151
syncIntervalMinutes: body.syncIntervalMinutes,
5252
accessMode: body.accessMode,
53-
credentialGroupId: body.credentialGroupId,
54-
credentialGroupOptionId: body.credentialGroupOptionId,
5553
resolveBillingAttribution: (workspaceId: string) =>
5654
resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId),
5755
source: 'ui' as const,
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { authMockFns } from '@sim/testing'
5+
import { NextRequest } from 'next/server'
6+
import { beforeEach, describe, expect, it, vi } from 'vitest'
7+
8+
const mocks = vi.hoisted(() => ({ search: vi.fn() }))
9+
vi.mock('@/lib/knowledge/application/search', () => ({
10+
searchKnowledge: { operation: { id: 'knowledge.search' }, execute: mocks.search },
11+
}))
12+
13+
import { POST } from '@/app/api/knowledge/search/route'
14+
15+
describe('workspace search route', () => {
16+
beforeEach(() => {
17+
vi.clearAllMocks()
18+
authMockFns.mockGetSession.mockResolvedValue({
19+
user: { id: 'user-1', email: 'reader@fixture.test', name: 'Reader' },
20+
session: { id: 'session-1' },
21+
})
22+
mocks.search.mockResolvedValue({ results: [], knowledgeBases: [] })
23+
})
24+
25+
it('passes the authenticated request cancellation signal through the existing operation', async () => {
26+
const controller = new AbortController()
27+
const request = new NextRequest('http://localhost/api/knowledge/search', {
28+
method: 'POST',
29+
headers: { 'content-type': 'application/json' },
30+
body: JSON.stringify({
31+
workspaceId: 'workspace-1',
32+
knowledgeBaseIds: ['kb-1'],
33+
query: 'Orion',
34+
}),
35+
signal: controller.signal,
36+
})
37+
const response = await POST(request)
38+
expect(response.status).toBe(200)
39+
const call = mocks.search.mock.calls[0][0]
40+
expect(call.principal).toEqual({ kind: 'session', userId: 'user-1', sessionId: 'session-1' })
41+
expect(call.input.signal).toBe(request.signal)
42+
controller.abort()
43+
expect(call.input.signal.aborted).toBe(true)
44+
await expect(response.json()).resolves.toEqual({
45+
success: true,
46+
data: { query: 'Orion', results: [] },
47+
})
48+
})
49+
50+
it('authenticates before parsing and never enters search for an anonymous request', async () => {
51+
authMockFns.mockGetSession.mockResolvedValueOnce(null)
52+
const response = await POST(
53+
new NextRequest('http://localhost/api/knowledge/search', {
54+
method: 'POST',
55+
body: '{',
56+
headers: { 'content-type': 'application/json' },
57+
})
58+
)
59+
expect(response.status).toBe(401)
60+
expect(mocks.search).not.toHaveBeenCalled()
61+
})
62+
})

‎apps/sim/app/api/knowledge/search/route.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,13 @@ export const POST = defineInternalJsonRoute({
1717
reason: 'A person typing queries; the embedding call is metered against their workspace',
1818
}),
1919
errorPolicy: internalKnowledgeErrorPolicies.search,
20-
mapInput: ({ body }) => ({
20+
mapInput: ({ body }, { request }) => ({
2121
workspaceId: body.workspaceId,
2222
knowledgeBaseIds: body.knowledgeBaseIds,
2323
query: body.query,
2424
topK: body.topK,
25+
surface: 'dashboard' as const,
26+
signal: request.signal,
2527
}),
2628
useCase: searchKnowledge,
2729
present: ({ results, knowledgeBases }, { input }) => {

‎apps/sim/app/api/knowledge/search/utils.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,14 @@ import * as documentsUtilsModule from '@/lib/knowledge/documents/utils'
1818
import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance'
1919
import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
2020

21+
vi.mock('@/lib/core/rate-limiter/provider-admission', () => ({
22+
PROVIDER_QUOTA_COOLDOWN_MS: 300_000,
23+
ProviderQuotaExhaustedError: class ProviderQuotaExhaustedError extends Error {},
24+
isProviderQuotaExhausted: vi.fn().mockResolvedValue(false),
25+
recordProviderCooldown: vi.fn().mockResolvedValue(undefined),
26+
waitForProviderAdmission: vi.fn().mockResolvedValue(undefined),
27+
}))
28+
2129
/**
2230
* Spy on the real documents/utils namespace instead of vi.mock: the shared
2331
* `@/lib/knowledge/embeddings` module may be cached bound to the real module,
@@ -196,6 +204,27 @@ describe('Knowledge Search Utils', () => {
196204
})
197205

198206
describe('handleTagAndVectorSearch', () => {
207+
it('returns only bounded ranked rows without first materializing every matching tag ID', async () => {
208+
resetDbChainMock()
209+
queueTableRows(schemaMock.embedding, [makeResult('second', 0.2), makeResult('first', 0.1)])
210+
211+
const results = await handleTagAndVectorSearch({
212+
knowledgeBaseIds: ['kb-1', 'kb-2'],
213+
access: WORKSPACE_ACCESS_SCOPE,
214+
topK: 2,
215+
structuredFilters: [
216+
{ tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'common' },
217+
],
218+
queryVector: { vector: JSON.stringify(TEST_EMBEDDING), dimensions: 1536 },
219+
distanceThreshold: 0.8,
220+
})
221+
222+
expect(results.map((row) => row.id)).toEqual(['first', 'second'])
223+
expect(dbChainMockFns.select).toHaveBeenCalledTimes(1)
224+
expect(dbChainMockFns.select.mock.calls[0][0]).toHaveProperty('distance')
225+
expect(dbChainMockFns.limit).toHaveBeenCalledWith(2)
226+
})
227+
199228
it('should throw error when no filters provided', async () => {
200229
const params = {
201230
knowledgeBaseIds: ['kb-123'],

0 commit comments

Comments
 (0)