Skip to content

Commit 5815964

Browse files
authored
fix(files): stream served files with byte ranges instead of buffering them under 100 MB (#9542)
* fix(files): stream served files with byte ranges instead of buffering them under 100 MB * fix(files): stream finished document uploads, diagnose playback errors, prove ranges on real bytes * fix(files): probe served bytes before handing off, keep generated documents in-page, version local objects by inode
1 parent 30d6f67 commit 5815964

24 files changed

Lines changed: 1155 additions & 259 deletions

File tree

‎apps/sim/app/api/files/serve/[...path]/route.test.ts‎

Lines changed: 242 additions & 42 deletions
Large diffs are not rendered by default.

‎apps/sim/app/api/files/serve/[...path]/route.ts‎

Lines changed: 156 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -11,35 +11,43 @@ import {
1111
} from '@/lib/api/server/routes'
1212
import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
1313
import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types'
14+
import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream'
1415
import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
1516
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
1617
import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads'
1718
import type { StorageContext } from '@/lib/uploads/config'
1819
import { readOrganizationChatAttachment } from '@/lib/uploads/contexts/organization-assistant/application'
1920
import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace'
2021
import { parseWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/file-manager'
21-
import { downloadFile } from '@/lib/uploads/core/storage-service'
22+
import { downloadFile, downloadFileStream, headObject } from '@/lib/uploads/core/storage-service'
2223
import { resolveServableDocBytes } from '@/lib/uploads/documents/compile'
2324
import { DocCompileUserError } from '@/lib/uploads/documents/compile-error'
24-
import { FILE_CACHE_CONTROL, workspaceFileCacheControl } from '@/lib/uploads/server/delivery'
25+
import {
26+
FILE_CACHE_CONTROL,
27+
storedObjectEtag,
28+
workspaceFileCacheControl,
29+
} from '@/lib/uploads/server/delivery'
2530
import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative'
2631
import { resolveStoredFileContext } from '@/lib/uploads/server/metadata'
2732
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
28-
import { inferContextFromKey, resolveEffectiveMimeType } from '@/lib/uploads/utils/file'
33+
import {
34+
inferContextFromKey,
35+
needsRenderedArtifact,
36+
resolveEffectiveMimeType,
37+
} from '@/lib/uploads/utils/file'
2938
import { internalWorkspaceFileServeAuth } from '@/lib/workspace-files/api'
3039
import { fetchAuthorizedServableWorkspaceFileBuffer } from '@/lib/workspace-files/application/fetch-servable-workspace-file-buffer'
3140
import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery'
32-
import {
33-
readWorkspaceFileContentByKey,
34-
readWorkspaceFileRecordByKey,
35-
} from '@/lib/workspace-files/application/read-workspace-file-content-by-key'
41+
import { readWorkspaceFileRecordByKey } from '@/lib/workspace-files/application/read-workspace-file-record-by-key'
42+
import { streamAuthorizedWorkspaceFile } from '@/lib/workspace-files/application/stream-authorized-workspace-file'
3643
import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile'
3744
import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server'
3845
import { type KnowledgeFileAccess, verifyFileAccess } from '@/app/api/files/authorization'
3946
import {
4047
createConditionalFileResponse,
4148
createFileErrorResponse,
4249
createFileResponse,
50+
createStoredFileStreamResponse,
4351
FileNotFoundError,
4452
type FileResponse,
4553
findLocalFile,
@@ -191,6 +199,75 @@ async function resolveTransformedBytes(params: {
191199
}
192200
}
193201

202+
/**
203+
* Whether a request is answered with the stored object byte-for-byte, so its bytes can stream from
204+
* storage instead of being read whole and capped at the transfer ceiling.
205+
*
206+
* Every other answer is produced from the bytes and stays buffered: a Sim page renders, a generated
207+
* document compiles, and a `preview` may be a HEIC transcode — sniffed from the content, so no name
208+
* or type rules it out. What streams is exactly what {@link resolveServableBytes} and the
209+
* workspace resolver would have passed through unchanged.
210+
*/
211+
function servesStoredObject(
212+
filename: string,
213+
recordedType: string | undefined,
214+
options: ServeOptions
215+
): boolean {
216+
if (options.raw) return true
217+
if (options.preview) return false
218+
if (recordedType === SIM_PAGE_CONTENT_TYPE || filename.toLowerCase().endsWith('.html')) {
219+
return false
220+
}
221+
// A recorded type tells a finished upload from a generation source; without one, every
222+
// document name may be a source.
223+
return !needsRenderedArtifact(recordedType, filename)
224+
}
225+
226+
/**
227+
* The type a stored object streams under: what the buffered path names the same bytes — the
228+
* extension's type for a served file, the route's own table for `raw`.
229+
*/
230+
function storedObjectContentType(filename: string, options: ServeOptions): string {
231+
return options.raw ? getContentType(filename) : resolveEffectiveMimeType(undefined, filename)
232+
}
233+
234+
/**
235+
* Streams a stored object outside the workspace-file model (execution outputs, chat and copilot
236+
* attachments, knowledge documents) when the request is answered with it as-is, or returns null
237+
* for the buffered path. The object is its only record, so storage is asked for its length and
238+
* version, and the version is what proves a revalidation still holds.
239+
*/
240+
async function streamStoredObject(params: {
241+
key: string
242+
context: StorageContext
243+
filename: string
244+
options: ServeOptions
245+
request: NextRequest
246+
}): Promise<NextResponse | null> {
247+
const { key, context, filename, options } = params
248+
if (!servesStoredObject(filename, undefined, options)) return null
249+
const stored = await headObject(key, context)
250+
if (!stored) throw new FileNotFoundError(`File not found: ${key}`)
251+
252+
logger.info('Streaming stored file', { key, context, size: stored.size })
253+
return createStoredFileStreamResponse(
254+
{
255+
filename,
256+
contentType: storedObjectContentType(filename, options),
257+
cacheControl:
258+
resolveServeCacheControl(options.versioned, context, false) ?? FILE_CACHE_CONTROL.private,
259+
size: stored.size,
260+
etag:
261+
stored.version === undefined
262+
? undefined
263+
: storedObjectEtag([context, key, stored.size, stored.version]),
264+
open: async (range) =>
265+
nodeReadableToWebStream(await downloadFileStream({ key, context, range })),
266+
},
267+
params.request.headers
268+
)
269+
}
270+
194271
const STORAGE_KEY_PREFIX_RE = /^\d{13}-[a-z0-9]{7}-/
195272

196273
function stripStorageKeyPrefix(segment: string): string {
@@ -372,21 +449,19 @@ export const GET = withRouteHandler(
372449
cloudKey,
373450
userId,
374451
options,
375-
request.signal,
452+
request,
376453
storageContext,
377-
knowledgeAccess,
378-
request.method === 'HEAD'
454+
knowledgeAccess
379455
)
380456
}
381457

382458
return await handleLocalFile(
383459
cloudKey,
384460
userId,
385461
options,
386-
request.signal,
462+
request,
387463
storageContext,
388-
knowledgeAccess,
389-
request.method === 'HEAD'
464+
knowledgeAccess
390465
)
391466
} catch (error) {
392467
if (request.signal.aborted) throw error
@@ -471,40 +546,47 @@ async function handleWorkspaceFile(
471546
headers: { 'Cache-Control': FILE_CACHE_CONTROL.noStore, 'X-Content-Type-Options': 'nosniff' },
472547
})
473548
}
474-
let file: WorkspaceFileRecord
475-
let resolved: ServableBytes
476-
if (options.raw) {
477-
const read = await readWorkspaceFileContentByKey.execute({ principal, input, request })
478-
file = read.file
479-
resolved = {
480-
buffer: read.content,
481-
contentType: getContentType(file.name),
482-
dependsOnReferencedFiles: false,
483-
}
484-
} else {
485-
file = (await readWorkspaceFileRecordByKey.execute({ principal, input, request })).file
486-
const rendered = await fetchAuthorizedServableWorkspaceFileBuffer(file, principal, {
487-
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
488-
signal: request.signal,
489-
})
490-
const preview = options.preview ? await resolveServableImageBytes(rendered.buffer, key) : null
491-
resolved = {
492-
buffer: preview?.buffer ?? rendered.buffer,
493-
contentType: preview?.contentType ?? renderedWorkspaceContentType(file, rendered.contentType),
494-
dependsOnReferencedFiles: rendered.dependsOnReferencedFiles,
495-
}
496-
assertKnownSizeWithinLimit(
497-
resolved.buffer.length,
498-
MAX_BUFFERED_TRANSFER_BYTES,
499-
'served file response'
549+
const { file } = await readWorkspaceFileRecordByKey.execute({ principal, input, request })
550+
const authorize = () => readWorkspaceFileRecordByKey.authorize({ principal, input, request })
551+
552+
if (servesStoredObject(file.name, file.type, options)) {
553+
logger.info('Streaming workspace file', { fileId: file.id, workspaceId, size: file.size })
554+
return createStoredFileStreamResponse(
555+
{
556+
filename: file.name,
557+
contentType: storedObjectContentType(file.name, options),
558+
cacheControl: workspaceFileCacheControl(options.versioned),
559+
size: file.size,
560+
// A content write uploads under a fresh key and stamps `contentUpdatedAt`, so the pair
561+
// changes whenever the bytes do.
562+
etag: storedObjectEtag([
563+
file.key,
564+
file.size,
565+
(file.contentUpdatedAt ?? file.updatedAt).getTime(),
566+
]),
567+
open: (range) => streamAuthorizedWorkspaceFile({ file, principal, authorize, range }),
568+
},
569+
request.headers
500570
)
501-
await finishFileDelivery({
502-
principal,
503-
authorize: () => readWorkspaceFileRecordByKey.authorize({ principal, input, request }),
504-
receipt: rendered.receipt,
505-
})
506571
}
507572

573+
const rendered = await fetchAuthorizedServableWorkspaceFileBuffer(file, principal, {
574+
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
575+
signal: request.signal,
576+
})
577+
const preview = options.preview ? await resolveServableImageBytes(rendered.buffer, key) : null
578+
const resolved: ServableBytes = {
579+
buffer: preview?.buffer ?? rendered.buffer,
580+
contentType: preview?.contentType ?? renderedWorkspaceContentType(file, rendered.contentType),
581+
dependsOnReferencedFiles: rendered.dependsOnReferencedFiles,
582+
}
583+
assertKnownSizeWithinLimit(
584+
resolved.buffer.length,
585+
MAX_BUFFERED_TRANSFER_BYTES,
586+
'served file response'
587+
)
588+
await finishFileDelivery({ principal, authorize, receipt: rendered.receipt })
589+
508590
logger.info('Workspace file served', {
509591
fileId: file.id,
510592
workspaceId,
@@ -529,12 +611,12 @@ async function handleLocalFile(
529611
filename: string,
530612
userId: string,
531613
options: ServeOptions,
532-
signal: AbortSignal | undefined,
614+
request: NextRequest,
533615
context: StorageContext,
534-
knowledgeAccess: KnowledgeFileAccess | undefined,
535-
head: boolean
616+
knowledgeAccess: KnowledgeFileAccess | undefined
536617
): Promise<NextResponse> {
537618
const ownerKey = `user:${userId}`
619+
const { signal } = request
538620
try {
539621
const hasAccess = await verifyFileAccess(
540622
filename,
@@ -549,7 +631,7 @@ async function handleLocalFile(
549631
throw new FileNotFoundError(`File not found: ${filename}`)
550632
}
551633

552-
if (head)
634+
if (request.method === 'HEAD')
553635
return new NextResponse(null, {
554636
headers: {
555637
'Cache-Control': FILE_CACHE_CONTROL.noStore,
@@ -562,13 +644,22 @@ async function handleLocalFile(
562644
throw new FileNotFoundError(`File not found: ${filename}`)
563645
}
564646

647+
const segment = filename.split('/').pop() || filename
648+
const displayName = stripStorageKeyPrefix(segment)
649+
const streamed = await streamStoredObject({
650+
key: filename,
651+
context,
652+
filename: displayName,
653+
options,
654+
request,
655+
})
656+
if (streamed) return streamed
657+
565658
const rawBuffer = await readLocalFileWithinLimit(
566659
filePath,
567660
MAX_BUFFERED_TRANSFER_BYTES,
568661
'served file'
569662
)
570-
const segment = filename.split('/').pop() || filename
571-
const displayName = stripStorageKeyPrefix(segment)
572663
const workspaceId = getWorkspaceIdForCompile(filename)
573664
const {
574665
buffer: fileBuffer,
@@ -610,12 +701,12 @@ async function handleCloudProxy(
610701
cloudKey: string,
611702
userId: string,
612703
options: ServeOptions,
613-
signal: AbortSignal | undefined,
704+
request: NextRequest,
614705
context: StorageContext,
615-
knowledgeAccess: KnowledgeFileAccess | undefined,
616-
head: boolean
706+
knowledgeAccess: KnowledgeFileAccess | undefined
617707
): Promise<NextResponse> {
618708
const ownerKey = `user:${userId}`
709+
const { signal } = request
619710
try {
620711
logger.info(`Resolved context: ${context} for key: ${cloudKey}`)
621712

@@ -632,13 +723,24 @@ async function handleCloudProxy(
632723
throw new FileNotFoundError(`File not found: ${cloudKey}`)
633724
}
634725

635-
if (head)
726+
if (request.method === 'HEAD')
636727
return new NextResponse(null, {
637728
headers: {
638729
'Cache-Control': FILE_CACHE_CONTROL.noStore,
639730
'X-Content-Type-Options': 'nosniff',
640731
},
641732
})
733+
const segment = cloudKey.split('/').pop() || 'download'
734+
const displayName = stripStorageKeyPrefix(segment)
735+
const streamed = await streamStoredObject({
736+
key: cloudKey,
737+
context,
738+
filename: displayName,
739+
options,
740+
request,
741+
})
742+
if (streamed) return streamed
743+
642744
let rawBuffer: Buffer
643745

644746
if (context === 'copilot') {
@@ -652,9 +754,6 @@ async function handleCloudProxy(
652754
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
653755
})
654756
}
655-
656-
const segment = cloudKey.split('/').pop() || 'download'
657-
const displayName = stripStorageKeyPrefix(segment)
658757
const workspaceId = getWorkspaceIdForCompile(cloudKey)
659758
const {
660759
buffer: fileBuffer,

0 commit comments

Comments
 (0)