From b668d32e08e38ed04d0f5a07d57122b9f882154a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:27:32 -0700 Subject: [PATCH 01/18] fix(files): serve rendered documents instead of generator source Generated docs (docx/pptx/pdf/xlsx) store their generation source as the primary file; the rendered binary lives in a separate content-addressed artifact store. resolveServableDocBytes is the chokepoint that swaps one for the other, and the serve route, single-file download and ~50 tool routes all go through it. Archive compression and the public v1 download read raw bytes instead, so a generated document arrived as source text under a .docx name and Word reported it as corrupt. Both now resolve through the servable reader. Adds fetchServableWorkspaceFileBuffer beside the raw reader so the record to UserFile mapping lives in one place, preserving storageContext; the raw reader's doc comment now says it returns generation source, so the next call site has to choose deliberately. v1 also has to send the resolved content type rather than the record's source MIME, and returns a retryable 409 rather than a 500 when an artifact is still compiling. docNotReadyMessage centralizes the 409 copy, and isRenderableDocumentName is shared so the read path and its callers agree on which extensions can expand. --- apps/sim/app/api/tools/file/manage/route.ts | 12 +++++- apps/sim/app/api/v1/files/[fileId]/route.ts | 17 +++++++-- .../workspace/workspace-file-manager.ts | 38 ++++++++++++++++++- .../lib/uploads/utils/file-utils.server.ts | 5 ++- apps/sim/lib/uploads/utils/file-utils.ts | 12 ++++++ .../uploads/utils/servable-file-response.ts | 30 +++++++++++---- 6 files changed, 99 insertions(+), 15 deletions(-) diff --git a/apps/sim/app/api/tools/file/manage/route.ts b/apps/sim/app/api/tools/file/manage/route.ts index 248002656ee..8f1cca412f6 100644 --- a/apps/sim/app/api/tools/file/manage/route.ts +++ b/apps/sim/app/api/tools/file/manage/route.ts @@ -11,6 +11,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { splitWorkspaceFilePath } from '@/lib/copilot/tools/server/files/workspace-file' import { acquireLock, releaseLock } from '@/lib/core/config/redis' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { ensureAbsoluteUrl } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers' @@ -685,7 +686,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger, { + // Generated docs store their generation source, not the rendered binary, so + // the archive must carry the servable bytes instead of the raw source text. + // A still-compiling artifact throws, and the handler's catch turns that into + // the shared 409 via `docNotReadyResponse`. + const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { maxBytes: MAX_COMPRESS_FILE_BYTES, }) totalBytes += buffer.length @@ -864,6 +869,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const notReady = docNotReadyResponse(error) if (notReady) return notReady + // A file over its per-file cap is a size rejection, not a fault. Rendered + // documents can cross it even when the stored source was well under. + if (isPayloadSizeLimitError(error)) { + return NextResponse.json({ success: false, error: error.message }, { status: 413 }) + } if (error instanceof ShareValidationError) { return NextResponse.json({ success: false, error: error.message }, { status: 400 }) } diff --git a/apps/sim/app/api/v1/files/[fileId]/route.ts b/apps/sim/app/api/v1/files/[fileId]/route.ts index 258e36b9ffb..40ec025932b 100644 --- a/apps/sim/app/api/v1/files/[fileId]/route.ts +++ b/apps/sim/app/api/v1/files/[fileId]/route.ts @@ -6,7 +6,11 @@ import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' -import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace' +import { + fetchServableWorkspaceFileBuffer, + getWorkspaceFile, +} from '@/lib/uploads/contexts/workspace' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/servable-file-response' import { performDeleteWorkspaceFileItems } from '@/lib/workspace-files/orchestration' import { checkRateLimit, @@ -48,7 +52,9 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo return NextResponse.json({ error: 'File not found' }, { status: 404 }) } - const buffer = await fetchWorkspaceFileBuffer(fileRecord) + // Generated docs store their generation source; serve the rendered artifact. + // Its content type is the rendered one, not the source MIME on the record. + const { buffer, contentType } = await fetchServableWorkspaceFileBuffer(fileRecord) recordAudit({ workspaceId, @@ -76,7 +82,7 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo return new Response(new Uint8Array(buffer), { status: 200, headers: { - 'Content-Type': fileRecord.type || 'application/octet-stream', + 'Content-Type': contentType || fileRecord.type || 'application/octet-stream', 'Content-Disposition': `attachment; filename="${fileRecord.name.replace(/[^\w.-]/g, '_')}"; filename*=UTF-8''${encodeURIComponent(fileRecord.name)}`, 'Content-Length': String(buffer.length), 'X-File-Id': fileRecord.id, @@ -88,6 +94,11 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo }, }) } catch (error) { + // A generated doc whose artifact is still compiling is retryable, not a fault: + // without this the caller sees a 500 and has no reason to try again. + if (isDocNotReadyError(error)) { + return NextResponse.json({ error: docNotReadyMessage() }, { status: 409 }) + } logger.error(`[${requestId}] Error downloading file:`, error) return NextResponse.json({ error: 'Failed to download file' }, { status: 500 }) } diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 6967a7d313f..5b490c8168e 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -21,6 +21,7 @@ import { normalizeVfsSegment } from '@/lib/copilot/vfs/normalize-segment' import { canonicalWorkspaceFilePath, decodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils' import { resolveWorkflowAliasForWorkspace } from '@/lib/copilot/vfs/workflow-alias-resolver' import { isReservedWorkflowAliasBackingDisplayPath } from '@/lib/copilot/vfs/workflow-aliases' +import { generateRequestId } from '@/lib/core/utils/request' import { generateRestoreName } from '@/lib/core/utils/restore-name' import type { DbOrTx } from '@/lib/db/types' import { getServePathPrefix } from '@/lib/uploads' @@ -968,7 +969,42 @@ export async function getWorkspaceFile( } /** - * Download workspace file content + * Download the bytes a user should actually receive for a workspace file. + * + * Generated docs (docx/pptx/pdf/xlsx) store their GENERATION SOURCE as the primary + * file, so {@link fetchWorkspaceFileBuffer} hands back JavaScript/Python text under + * a `.docx` name. This resolves the rendered artifact instead, and is what every + * download/attachment surface should call. Reach for the raw reader only when the + * source itself is wanted (style extraction, compile checks, the copilot VFS). + * + * Throws `DocCompileUserError` when a generated doc's artifact is still compiling — + * callers surface a retryable 409 via `docNotReadyResponse` rather than shipping source. + */ +export async function fetchServableWorkspaceFileBuffer( + fileRecord: WorkspaceFileRecord, + options: { maxBytes?: number; signal?: AbortSignal } = {} +): Promise<{ buffer: Buffer; contentType: string }> { + const { downloadServableFileFromStorage } = await import('@/lib/uploads/utils/file-utils.server') + + return downloadServableFileFromStorage( + { + id: fileRecord.id, + name: fileRecord.name, + url: fileRecord.url ?? fileRecord.path, + size: fileRecord.size, + type: fileRecord.type, + key: fileRecord.key, + context: fileRecord.storageContext ?? 'workspace', + }, + generateRequestId(), + logger, + options + ) +} + +/** + * Download raw workspace file content. For generated docs this is the GENERATION + * SOURCE, not the rendered document — see {@link fetchServableWorkspaceFileBuffer}. */ export async function fetchWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index cc230df8108..97013439e77 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -20,6 +20,7 @@ import { getMimeTypeFromExtension, inferContextFromKey, isInternalFileUrl, + isRenderableDocumentName, processSingleFileToUserFile, type RawFileInput, resolveTrustedFileContext, @@ -375,8 +376,8 @@ export async function downloadServableFileFromStorage( // Cheap pre-filter so only generated-doc candidates pay for the heavier resolver // import below. - const ext = getFileExtension(userFile.name) - if (ext !== 'pdf' && ext !== 'docx' && ext !== 'pptx' && ext !== 'xlsx') { + if (!isRenderableDocumentName(userFile.name)) { + const ext = getFileExtension(userFile.name) return { buffer, contentType: userFile.type || getMimeTypeFromExtension(ext) } } diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 7837617a0cc..442d52d324f 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -210,6 +210,18 @@ export function getFileExtension(filename: string): string { return lastDot !== -1 ? filename.slice(lastDot + 1).toLowerCase() : '' } +/** + * Extensions whose stored bytes may be a generation source that renders to a larger + * binary. Everything else stores exactly what it serves, so its declared size is + * an accurate byte budget. + */ +const RENDERABLE_DOCUMENT_EXTENSIONS = new Set(['pdf', 'docx', 'pptx', 'xlsx']) + +/** True when `fileName` may be backed by a generation source rather than final bytes. */ +export function isRenderableDocumentName(fileName: string): boolean { + return RENDERABLE_DOCUMENT_EXTENSIONS.has(getFileExtension(fileName)) +} + const ARCHIVE_EXTENSIONS = new Set(SUPPORTED_ARCHIVE_EXTENSIONS) /** diff --git a/apps/sim/lib/uploads/utils/servable-file-response.ts b/apps/sim/lib/uploads/utils/servable-file-response.ts index 1e7d1a6124b..63ffb0223f0 100644 --- a/apps/sim/lib/uploads/utils/servable-file-response.ts +++ b/apps/sim/lib/uploads/utils/servable-file-response.ts @@ -1,6 +1,23 @@ import { NextResponse } from 'next/server' import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile' +/** True when `error` means a generated document's artifact is still compiling. */ +export function isDocNotReadyError(error: unknown): error is DocCompileUserError { + return error instanceof DocCompileUserError +} + +/** + * Message for a still-compiling generated document. Batch callers pass the names + * they resolved so the copy says which documents to wait on. + */ +export function docNotReadyMessage(fileNames?: string[]): string { + if (!fileNames || fileNames.length === 0) { + return 'A document is still being generated. Wait for it to finish, then try again.' + } + const subject = fileNames.length === 1 ? 'A document is' : `${fileNames.length} documents are` + return `${subject} still being generated: ${fileNames.join(', ')}. Wait for them to finish, then try again.` +} + /** * Canonical retryable response for an attachment/upload whose generated-document * artifact is still compiling. Returns the 409 when `error` is a @@ -8,16 +25,13 @@ import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compil * otherwise `null` so the caller falls through to its own error handling. Shared * by every tool route that downloads workspace files so the status, body shape, * and user-facing copy stay identical instead of being re-typed per route. + * + * Routes whose error envelope differs, or that resolved a batch and want the pending + * files named, build the 409 themselves from {@link docNotReadyMessage}. */ export function docNotReadyResponse(error: unknown): NextResponse | null { - if (error instanceof DocCompileUserError) { - return NextResponse.json( - { - success: false, - error: 'A document is still being generated. Wait for it to finish, then try again.', - }, - { status: 409 } - ) + if (isDocNotReadyError(error)) { + return NextResponse.json({ success: false, error: docNotReadyMessage() }, { status: 409 }) } return null } From 8a020d1444860d77587a2dd40cd6e15404200a3a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:27:46 -0700 Subject: [PATCH 02/18] perf(files): stream workspace archives instead of buffering them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bulk download route materialized every selected file before writing a byte, so peak memory tracked the size of the selection. Ordinary files are now appended as lazy streams: each opens its storage read only when the archiver reaches it, so one entry is resident at a time rather than the whole archive. Generated documents still resolve to buffers first. They are the only entries whose bytes decide anything, and every status this route returns comes from them — once the first byte is written the status code is committed, so those decisions have to happen before the archive starts. The per-entry allowance and byte budget therefore govern documents only. archiver processes appended entries through a sequential queue; lazystream defers each storage read until that entry's turn, since handing the archiver an open stream per entry would hold more connections than the storage client pools. nodeReadableToWebStream moves out of input-validation.server.ts into a shared util rather than being written twice: Readable.toWeb throws ERR_INVALID_STATE when a consumer cancels while the source is still flowing, which is exactly what a cancelled download does. Trade: a storage read failing mid-archive truncates the response rather than returning 500, since the status is already sent. Documents cannot hit this. The table export route already behaves this way. --- .../[id]/files/download/route.test.ts | 272 ++++++++++++++++++ .../workspaces/[id]/files/download/route.ts | 194 ++++++++++++- .../core/security/input-validation.server.ts | 62 +--- apps/sim/lib/core/utils/node-stream.ts | 63 ++++ apps/sim/package.json | 4 + 5 files changed, 520 insertions(+), 75 deletions(-) create mode 100644 apps/sim/app/api/workspaces/[id]/files/download/route.test.ts create mode 100644 apps/sim/lib/core/utils/node-stream.ts diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts new file mode 100644 index 00000000000..828f6c4e2e9 --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -0,0 +1,272 @@ +/** + * @vitest-environment node + */ +import { Readable } from 'stream' +import { createMockRequest } from '@sim/testing' +import JSZip from 'jszip' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockGetSession, + mockVerifyWorkspaceMembership, + mockListWorkspaceFiles, + mockListWorkspaceFileFolders, + mockFetchServableWorkspaceFileBuffer, + mockDownloadFileStream, +} = vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockVerifyWorkspaceMembership: vi.fn(), + mockListWorkspaceFiles: vi.fn(), + mockListWorkspaceFileFolders: vi.fn(), + mockFetchServableWorkspaceFileBuffer: vi.fn(), + mockDownloadFileStream: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ + auth: { api: { getSession: vi.fn() } }, + getSession: mockGetSession, +})) + +vi.mock('@/app/api/workflows/utils', () => ({ + verifyWorkspaceMembership: mockVerifyWorkspaceMembership, +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + listWorkspaceFiles: mockListWorkspaceFiles, + listWorkspaceFileFolders: mockListWorkspaceFileFolders, + buildWorkspaceFileFolderPathMap: (folders: Array<{ id: string; name: string }>) => + new Map(folders.map((folder) => [folder.id, folder.name])), + fetchServableWorkspaceFileBuffer: mockFetchServableWorkspaceFileBuffer, +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFileStream: mockDownloadFileStream, +})) + +vi.mock('@sim/audit', () => ({ + recordAudit: vi.fn(), + AuditAction: { FILE_DOWNLOADED: 'file.downloaded' }, + AuditResourceType: { FILE: 'file' }, +})) + +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() })) + +import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { GET } from '@/app/api/workspaces/[id]/files/download/route' + +const WORKSPACE_ID = 'ws-1' +const context = { params: Promise.resolve({ id: WORKSPACE_ID }) } +const MB = 1024 * 1024 + +function workspaceFile(id: string, name: string, folderId: string | null = 'folder-1') { + return { + id, + name, + key: `workspace/${WORKSPACE_ID}/${id}`, + path: `/serve/${id}`, + size: 100, + type: 'application/octet-stream', + folderId, + } +} + +function requestFor(query: string) { + return createMockRequest( + 'GET', + undefined, + {}, + `http://localhost:3000/api/workspaces/${WORKSPACE_ID}/files/download?${query}` + ) +} + +async function zipFrom(response: Response) { + return JSZip.loadAsync(Buffer.from(await response.arrayBuffer())) +} + +describe('workspace files download route', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) + mockVerifyWorkspaceMembership.mockResolvedValue({ role: 'member' }) + mockListWorkspaceFileFolders.mockResolvedValue([ + { id: 'folder-1', name: 'Reports', parentId: null }, + ]) + mockDownloadFileStream.mockImplementation(async () => Readable.from([Buffer.from('plain')])) + }) + + it('zips the rendered bytes for a generated doc, not its stored source', async () => { + mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'overview.docx')]) + // A real .docx is a ZIP; the stored source would be plain JS text. + const rendered = Buffer.from('PKrendered-docx') + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: rendered, + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(200) + const entry = (await zipFrom(response)).file('Reports/overview.docx') + expect(entry).not.toBeNull() + expect(Buffer.from(await entry!.async('uint8array'))).toEqual(rendered) + }) + + it('streams ordinary files instead of materializing them', async () => { + mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'clip.mp4')]) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(200) + // Nothing has been read yet: the entry opens its storage read only once the + // consumer pulls the archive, which is what keeps peak memory to one entry. + expect(mockDownloadFileStream).not.toHaveBeenCalled() + + const zip = await zipFrom(response) + + expect(mockDownloadFileStream).toHaveBeenCalledTimes(1) + // Never routed through the buffering document reader. + expect(mockFetchServableWorkspaceFileBuffer).not.toHaveBeenCalled() + + const entry = zip.file('Reports/clip.mp4') + expect(entry).not.toBeNull() + expect(await entry!.async('string')).toBe('plain') + }) + + it('preserves nested folder paths across both entry kinds', async () => { + mockListWorkspaceFileFolders.mockResolvedValue([ + { id: 'folder-1', name: 'Reports', parentId: null }, + { id: 'folder-2', name: 'visuals', parentId: 'folder-1' }, + ]) + mockListWorkspaceFiles.mockResolvedValue([ + workspaceFile('f1', 'summary.docx', 'folder-1'), + workspaceFile('f2', 'hero.png', 'folder-2'), + ]) + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: Buffer.from('PKdoc'), + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }) + + const zip = await zipFrom(await GET(requestFor('fileIds=f1&fileIds=f2'), context)) + + expect(zip.file('Reports/summary.docx')).not.toBeNull() + expect(zip.file('visuals/hero.png')).not.toBeNull() + }) + + it('returns 409 naming the documents whose artifacts are still compiling', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + workspaceFile('f1', 'ready.docx'), + workspaceFile('f2', 'pending.docx'), + ]) + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { + if (file.name === 'pending.docx') + throw new DocCompileUserError('Document is still being generated') + return { buffer: Buffer.from('PKok'), contentType: 'application/octet-stream' } + }) + + const response = await GET(requestFor('fileIds=f1&fileIds=f2'), context) + + expect(response.status).toBe(409) + const body = await response.json() + expect(body.error).toContain('pending.docx') + expect(body.error).not.toContain('ready.docx') + }) + + it('rejects with 400, not 500, when a document blows its own allowance', async () => { + mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'huge.docx')]) + mockFetchServableWorkspaceFileBuffer.mockRejectedValue( + new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) + ) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(400) + const body = await response.json() + expect(body.error).toContain('huge.docx') + expect(body.error).not.toContain('Selected files total') + }) + + it('blames the shared budget once earlier documents have consumed it', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + workspaceFile('f1', 'first.docx'), + workspaceFile('f2', 'second.docx'), + ]) + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { + // The first document eats the whole budget, so the second's cap is the remainder. + if (file.name === 'first.docx') { + return { buffer: Buffer.alloc(240 * MB), contentType: 'application/octet-stream' } + } + throw new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) + }) + + const response = await GET(requestFor('fileIds=f1&fileIds=f2'), context) + + expect(response.status).toBe(400) + const body = await response.json() + expect(body.error).toContain('Selected files total') + expect(body.error).not.toContain('second.docx') + }) + + it('lets an uploaded office file larger than the render headroom through', async () => { + const big = { ...workspaceFile('f1', 'deck.pptx'), size: 80 * MB } + mockListWorkspaceFiles.mockResolvedValue([big]) + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: Buffer.from('PKdeck'), + contentType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + }) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(200) + // Capped at the declared size, not the smaller render headroom. + expect(mockFetchServableWorkspaceFileBuffer.mock.calls[0][1].maxBytes).toBe(80 * MB) + }) + + it('surfaces a storage failure as a 500 even when another document is pending', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + workspaceFile('f1', 'pending.docx'), + workspaceFile('f2', 'broken.docx'), + ]) + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { + if (file.name === 'pending.docx') + throw new DocCompileUserError('Document is still being generated') + throw new Error('storage down') + }) + + const response = await GET(requestFor('fileIds=f1&fileIds=f2'), context) + + // A 409 would tell the client to retry something that can never succeed. + expect(response.status).toBe(500) + }) + + it('stops resolving documents once one hard-fails', async () => { + const files = Array.from({ length: 20 }, (_, index) => + workspaceFile(`f${index}`, `doc${index}.docx`) + ) + mockListWorkspaceFiles.mockResolvedValue(files) + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { + if (file.name === 'doc0.docx') throw new Error('storage down') + return { buffer: Buffer.from('PKok'), contentType: 'application/octet-stream' } + }) + + const response = await GET( + requestFor(files.map((file) => `fileIds=${file.id}`).join('&')), + context + ) + + expect(response.status).toBe(500) + expect(mockFetchServableWorkspaceFileBuffer.mock.calls.length).toBeLessThan(files.length) + }) + + it('rejects a selection whose declared sizes already exceed the limit', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + { ...workspaceFile('f1', 'a.mp4'), size: 200 * MB }, + { ...workspaceFile('f2', 'b.mp4'), size: 200 * MB }, + ]) + + const response = await GET(requestFor('fileIds=f1&fileIds=f2'), context) + + expect(response.status).toBe(400) + expect(mockDownloadFileStream).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 577f0e7b2dc..9b0b14a3ac7 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -1,25 +1,76 @@ +import { PassThrough, type Readable } from 'stream' import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' -import JSZip from 'jszip' +import { toError } from '@sim/utils/errors' +import { ZipArchive } from 'archiver' +import lazystream from 'lazystream' import { type NextRequest, NextResponse } from 'next/server' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { buildWorkspaceFileFolderPathMap, - fetchWorkspaceFileBuffer, + fetchServableWorkspaceFileBuffer, listWorkspaceFileFolders, listWorkspaceFiles, } from '@/lib/uploads/contexts/workspace' -import { formatFileSize } from '@/lib/uploads/utils/file-utils' +import { downloadFileStream } from '@/lib/uploads/core/storage-service' +import { formatFileSize, isRenderableDocumentName } from '@/lib/uploads/utils/file-utils' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/servable-file-response' import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' const logger = createLogger('WorkspaceFilesDownloadAPI') const MAX_ZIP_DOWNLOAD_FILES = 100 const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 +/** + * Headroom a document may render to beyond what it declared. Office extensions + * cover both ordinary uploads — which serve exactly their declared size — and + * source-backed generated docs, and the two are indistinguishable before the bytes + * are read. So the allowance is the larger of the declared size and this ceiling: + * an uploaded 80 MiB deck still downloads, while a 6 KiB generator source cannot + * quietly render into hundreds of megabytes. + */ +const RENDERED_DOCUMENT_HEADROOM_BYTES = 50 * 1024 * 1024 +/** + * Fan-out for files that serve exactly their declared size. Their total is already + * bounded by the pre-download check, so concurrency cannot push residency past it. + * Renderable documents are read one at a time instead — see below. + */ +const ZIP_MATERIALIZE_CONCURRENCY = 5 + +function overLimitResponse(bytes: number, qualifier = ''): NextResponse { + return NextResponse.json( + { + error: `Selected files total ${formatFileSize(bytes)}${qualifier}, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`, + }, + { status: 400 } + ) +} + +/** + * A `Readable` that opens its storage read on first pull rather than up front. Handing + * the archiver one open stream per entry would hold a connection per selected file — + * more than the storage client pools — with most sitting idle until their turn. + */ +function lazyWorkspaceFileStream(file: WorkspaceFileRecord): Readable { + return new lazystream.Readable(() => { + const relay = new PassThrough() + downloadFileStream({ key: file.key, context: file.storageContext ?? 'workspace' }) + .then((source) => { + source.on('error', (error) => relay.destroy(toError(error))) + source.pipe(relay) + }) + .catch((error) => relay.destroy(toError(error))) + return relay + }) +} function collectDescendantFolderIds( selectedFolderIds: string[], @@ -82,17 +133,123 @@ export const GET = withRouteHandler( ) } - const totalBytes = filesToZip.reduce((sum, file) => sum + file.size, 0) - if (totalBytes > MAX_ZIP_DOWNLOAD_BYTES) { + const declaredBytes = filesToZip.reduce((sum, file) => sum + file.size, 0) + if (declaredBytes > MAX_ZIP_DOWNLOAD_BYTES) { + return overLimitResponse(declaredBytes) + } + + // Generated docs (docx/pptx/pdf/xlsx) store their generation source, not the + // rendered binary, so bytes are resolved through the servable reader — a raw + // read ships source text under a `.docx` name. + // + // Only those documents can exceed the size they declared, so only they can push + // residency past the budget the pre-download check already enforced. They are + // therefore read one at a time, which keeps the overshoot to a single entry; + // everything else stays parallel because its bytes are exactly its declared size. + // Once the budget is blown or a read hard-fails, the abort flag stops reads that + // have not started yet. + const controller = new AbortController() + let renderedBytes = 0 + + interface DownloadOutcome { + buffer: Buffer | null + pendingName: string | null + /** Set only when an entry's own allowance bound it, never the shared budget. */ + overLimitEntry: { name: string; allowance: number } | null + overLimit: boolean + error: unknown + } + const skipped: DownloadOutcome = { + buffer: null, + pendingName: null, + overLimitEntry: null, + overLimit: false, + error: null, + } + + const readEntry = async (file: WorkspaceFileRecord): Promise => { + if (controller.signal.aborted) return skipped + const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes) + // Uploads and source-backed documents are indistinguishable before the bytes + // are read, so the allowance is the larger of the declared size and the render + // headroom: a real 80 MiB deck still downloads, a small generator source cannot + // render unbounded. + const allowance = Math.max(file.size, RENDERED_DOCUMENT_HEADROOM_BYTES) + try { + const { buffer } = await fetchServableWorkspaceFileBuffer(file, { + maxBytes: Math.min(remaining, allowance), + signal: controller.signal, + }) + renderedBytes += buffer.length + const overLimit = renderedBytes > MAX_ZIP_DOWNLOAD_BYTES + if (overLimit) controller.abort() + return { ...skipped, buffer, overLimit } + } catch (error) { + // Recorded even when another worker already aborted: a size rejection + // describes this file, so losing it to someone else's cancellation would + // downgrade an actionable 400 into an opaque 500. + if (error instanceof PayloadSizeLimitError) { + controller.abort() + // Attributed to this entry when its own cap was the binding one — ties + // included, since the entry's ceiling still made it unshippable and + // downloading it alone is the way through. Otherwise the budget ran out. + return { + ...skipped, + overLimit: true, + overLimitEntry: allowance <= remaining ? { name: file.name, allowance } : null, + } + } + // Any other error from an already-aborted read is a consequence of the + // cancellation, not a cause. Checked before this worker aborts anything so + // the worker that actually failed still records its own error. + if (controller.signal.aborted) return skipped + // A pending artifact is worth reporting in full, so keep resolving the + // rest of the selection; anything else dooms the request. + const pending = isDocNotReadyError(error) + if (!pending) controller.abort() + return { ...skipped, pendingName: pending ? file.name : null, error } + } + } + + // Documents are resolved up front, one at a time: they are the only entries that + // need their bytes in hand, and every status this route can return is decided + // from them. Nothing may fail after the first byte is written — the status code + // is committed by then — so this has to happen before the archive starts. + const documentBuffers = new Map() + const documents = filesToZip.filter((file) => isRenderableDocumentName(file.name)) + const downloads = await mapWithConcurrency(documents, 1, (file) => readEntry(file)) + downloads.forEach((outcome, index) => { + if (outcome.buffer) documentBuffers.set(documents[index].id, outcome.buffer) + }) + + // Size first: the request cannot succeed at any size-adjacent retry, and a + // descriptive 400 beats an opaque 500 raised by whatever the abort cancelled. + const overLimitEntry = downloads.find((result) => result.overLimitEntry)?.overLimitEntry + if (overLimitEntry) { + // Naming the entry that blew its own allowance, and quoting the allowance that + // actually applied: an aggregate message here would tell the user to select + // fewer files when the selection was fine. return NextResponse.json( { - error: `Selected files total ${formatFileSize(totalBytes)}, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`, + error: `"${overLimitEntry.name}" is too large to include in a zip. Entries are capped at ${formatFileSize(overLimitEntry.allowance)}; download it on its own instead.`, }, { status: 400 } ) } - const buffers = await Promise.all(filesToZip.map((file) => fetchWorkspaceFileBuffer(file))) + if (downloads.some((result) => result.overLimit)) { + return overLimitResponse(renderedBytes, ' once documents are rendered') + } + + // A hard failure outranks a pending artifact: waiting cannot fix it, so a 409 + // would send the client into a retry loop that never succeeds. + const failure = downloads.find((result) => result.error && !result.pendingName) + if (failure?.error) throw failure.error + + const pendingNames = downloads.flatMap((result) => result.pendingName ?? []) + if (pendingNames.length > 0) { + return NextResponse.json({ error: docNotReadyMessage(pendingNames) }, { status: 409 }) + } // Entry paths stay workspace-root-relative so a mixed selection of folders and // loose files keeps the layout the user sees in the files list. @@ -103,12 +260,20 @@ export const GET = withRouteHandler( })) ) - const zip = new JSZip() - for (const [index, buffer] of buffers.entries()) { - zip.file(entryPaths[index], buffer) - } + // Ordinary files are never materialized: each entry opens its storage read only + // when the archiver reaches it (`lazystream` defers the factory until first read), + // so peak memory is one entry rather than the whole selection. Resolved documents + // are appended from the buffers above. + const archive = new ZipArchive({ store: true }) + archive.on('warning', (error: Error) => { + logger.warn('Archive warning while streaming workspace files', { error }) + }) - const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' }) + filesToZip.forEach((file, index) => { + const buffer = documentBuffers.get(file.id) + archive.append(buffer ?? lazyWorkspaceFileStream(file), { name: entryPaths[index] }) + }) + void archive.finalize() recordAudit({ workspaceId, @@ -116,7 +281,7 @@ export const GET = withRouteHandler( action: AuditAction.FILE_DOWNLOADED, resourceType: AuditResourceType.FILE, description: `Downloaded ${filesToZip.length} file${filesToZip.length === 1 ? '' : 's'} as zip`, - metadata: { fileCount: filesToZip.length, totalBytes }, + metadata: { fileCount: filesToZip.length, totalBytes: declaredBytes }, request, }) captureServerEvent( @@ -126,7 +291,8 @@ export const GET = withRouteHandler( { groups: { workspace: workspaceId } } ) - return new NextResponse(new Uint8Array(zipBuffer), { + // No Content-Length: the archive size is not known until it has been produced. + return new NextResponse(nodeReadableToWebStream(archive), { headers: { 'Content-Type': 'application/zip', 'Content-Disposition': 'attachment; filename="workspace-files.zip"', diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index e8ea4071577..02c44a1ed97 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -18,6 +18,7 @@ import { } from 'undici' import { isHosted, isPrivateDatabaseHostsAllowed } from '@/lib/core/config/env-flags' import { type ValidationResult, validateExternalUrl } from '@/lib/core/security/input-validation' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' const logger = createLogger('InputValidation') @@ -717,67 +718,6 @@ function contentEncodingDecoder( } } -/** - * Bridges an undici `request()` Node `Readable` into a WHATWG `ReadableStream` for a - * `Response` body. Node's built-in `Readable.toWeb` is NOT used: its adapter throws an - * unhandled `ERR_INVALID_STATE` ("Controller is already closed") when the web stream is - * cancelled while the Node stream is still flowing — which `followRedirectsGuarded` does - * on every redirect hop (`response.body.cancel()`). This bridge instead swallows a late - * enqueue after close and destroys the source on cancel, so cancelling a live body frees - * its socket cleanly. `maxResponseSize` overruns surface as the source's `error` event and - * reject the read, preserving the DoS backstop. - */ -function nodeReadableToWebStream(nodeStream: Readable): ReadableStream { - let settled = false - return new ReadableStream({ - start(controller) { - nodeStream.on('data', (chunk: Buffer) => { - try { - // Copy, not a view: undici may recycle the pooled buffer backing `chunk` after - // this handler returns, which would corrupt a chunk still queued for a slow - // consumer. `new Uint8Array(chunk)` allocates a fresh backing buffer. - controller.enqueue(new Uint8Array(chunk)) - } catch { - // Controller already closed (consumer cancelled) — stop the source, drop the chunk. - nodeStream.destroy() - return - } - if ((controller.desiredSize ?? 1) <= 0) nodeStream.pause() - }) - nodeStream.once('end', () => { - settled = true - try { - controller.close() - } catch {} - }) - nodeStream.once('error', (err) => { - settled = true - try { - controller.error(err) - } catch {} - }) - // An abort (signal) or upstream reset can `destroy()` the source with no `error` - // event; without this the reader would hang forever. `close` fires after every - // terminal path, so only act when `end`/`error` didn't already settle the stream. - nodeStream.once('close', () => { - if (settled) return - settled = true - try { - controller.error(new Error('MCP transport stream closed before completing')) - } catch {} - }) - // Start paused so nothing buffers before the consumer pulls (backpressure). - nodeStream.pause() - }, - pull() { - nodeStream.resume() - }, - cancel(reason) { - nodeStream.destroy(reason instanceof Error ? reason : undefined) - }, - }) -} - /** * Streaming-safe replacement for `undiciFetch(url, { ...init, dispatcher })`. * diff --git a/apps/sim/lib/core/utils/node-stream.ts b/apps/sim/lib/core/utils/node-stream.ts new file mode 100644 index 00000000000..4918c5d77d3 --- /dev/null +++ b/apps/sim/lib/core/utils/node-stream.ts @@ -0,0 +1,63 @@ +import type { Readable } from 'stream' + +/** + * Bridges a Node `Readable` into a WHATWG `ReadableStream` suitable for a `Response` + * body. Node's built-in `Readable.toWeb` is NOT used: its adapter throws an unhandled + * `ERR_INVALID_STATE` ("Controller is already closed") when the web stream is cancelled + * while the Node stream is still flowing — which happens whenever a consumer aborts a + * live body (a redirect hop cancelling the previous response, a browser cancelling a + * download mid-transfer). This bridge instead swallows a late enqueue after close and + * destroys the source on cancel, so cancelling a live body frees its socket cleanly. + * Source errors — a size-limit overrun, a storage read failing mid-archive — surface as + * the source's `error` event and reject the read. + */ +export function nodeReadableToWebStream(nodeStream: Readable): ReadableStream { + let settled = false + return new ReadableStream({ + start(controller) { + nodeStream.on('data', (chunk: Buffer) => { + try { + // Copy, not a view: the producer may recycle the pooled buffer backing `chunk` + // after this handler returns, which would corrupt a chunk still queued for a + // slow consumer. `new Uint8Array(chunk)` allocates a fresh backing buffer. + controller.enqueue(new Uint8Array(chunk)) + } catch { + // Controller already closed (consumer cancelled) — stop the source, drop the chunk. + nodeStream.destroy() + return + } + if ((controller.desiredSize ?? 1) <= 0) nodeStream.pause() + }) + nodeStream.once('end', () => { + settled = true + try { + controller.close() + } catch {} + }) + nodeStream.once('error', (err) => { + settled = true + try { + controller.error(err) + } catch {} + }) + // An abort or upstream reset can `destroy()` the source with no `error` event; + // without this the reader would hang forever. `close` fires after every terminal + // path, so only act when `end`/`error` didn't already settle the stream. + nodeStream.once('close', () => { + if (settled) return + settled = true + try { + controller.error(new Error('Stream closed before completing')) + } catch {} + }) + // Start paused so nothing buffers before the consumer pulls (backpressure). + nodeStream.pause() + }, + pull() { + nodeStream.resume() + }, + cancel(reason) { + nodeStream.destroy(reason instanceof Error ? reason : undefined) + }, + }) +} diff --git a/apps/sim/package.json b/apps/sim/package.json index b0205c85f7f..16ecf6329e4 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -132,6 +132,7 @@ "@trigger.dev/sdk": "4.4.3", "@typescript/typescript6": "^6.0.2", "ajv": "8.18.0", + "archiver": "8.0.0", "better-auth": "1.6.23", "binary-extensions": "3.1.0", "browser-image-compression": "^2.0.2", @@ -170,6 +171,7 @@ "js-tiktoken": "1.0.21", "js-yaml": "4.3.0", "jszip": "3.10.1", + "lazystream": "1.0.1", "lru-cache": "11.3.6", "lucide-react": "^0.511.0", "mammoth": "^1.9.0", @@ -231,11 +233,13 @@ "@tailwindcss/typography": "0.5.19", "@testing-library/jest-dom": "^6.6.3", "@trigger.dev/build": "4.4.3", + "@types/archiver": "8.0.0", "@types/busboy": "1.5.4", "@types/fluent-ffmpeg": "2.1.28", "@types/html-to-text": "9.0.4", "@types/js-yaml": "4.0.9", "@types/jsdom": "21.1.7", + "@types/lazystream": "1.0.0", "@types/micromatch": "4.0.10", "@types/node": "24.2.1", "@types/nodemailer": "8.0.1", From 6d76dde3eefd9d995ffe70ed774065babc9783f4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:27:53 -0700 Subject: [PATCH 03/18] improvement(files): surface archive download errors in place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bulk and folder downloads navigated to the API route, so any rejection replaced the Files page with the raw JSON error body. Single-file download already fetched and saved the blob, so the two paths had diverged. That was survivable when the only failures were "too many files" and "too large"; resolving rendered documents adds a 409 for a still-compiling artifact, which is reachable in normal use. Both archive paths now fetch the zip and show the server's message as a toast — the route writes that copy for a person, so it is worth surfacing rather than discarding. Single-file download shows its error too instead of only logging it. --- .../workspace/[workspaceId]/files/files.tsx | 22 +++++-- apps/sim/lib/uploads/client/download.ts | 61 +++++++++++++++---- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index c6bcecde52d..2f85a03f24b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -27,7 +27,7 @@ import { usePostHog } from 'posthog-js/react' import { getDocumentIcon } from '@/components/icons/document-icons' import { useLimitUpgradeToast } from '@/lib/billing/client' import { captureEvent } from '@/lib/posthog/client' -import { triggerFileDownload } from '@/lib/uploads/client/download' +import { triggerArchiveDownload, triggerFileDownload } from '@/lib/uploads/client/download' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' import { @@ -951,6 +951,7 @@ export function Files() { }) } catch (err) { logger.error('Failed to download file:', err) + toast.error(toError(err).message) } }, [workspaceId] @@ -1071,7 +1072,7 @@ export function Files() { setShowDeleteConfirm(true) }, [selectedFileIds, selectedFolderIds, files, folders]) - const handleBulkDownload = useCallback(() => { + const handleBulkDownload = useCallback(async () => { const selectedFiles = files.filter((file) => selectedFileIds.includes(file.id)) if (selectedFiles.length === 1 && selectedFolderIds.length === 0) { handleDownload(selectedFiles[0]) @@ -1088,7 +1089,14 @@ export function Files() { is_bulk: true, file_count: selectedFileIds.length + selectedFolderIds.length, }) - window.location.href = `/api/workspaces/${workspaceId}/files/download?${query.toString()}` + try { + await triggerArchiveDownload( + `/api/workspaces/${workspaceId}/files/download?${query.toString()}` + ) + } catch (err) { + logger.error('Failed to download selection:', err) + toast.error(toError(err).message) + } }, [selectedFileIds, selectedFolderIds, files, handleDownload, workspaceId]) const fileDetailBreadcrumbs = useMemo(() => { @@ -1285,8 +1293,14 @@ export function Files() { return } if (item.kind === 'folder') { - window.location.href = `/api/workspaces/${workspaceId}/files/download?folderIds=${encodeURIComponent(item.folder.id)}` + const folderId = item.folder.id closeContextMenu() + triggerArchiveDownload( + `/api/workspaces/${workspaceId}/files/download?folderIds=${encodeURIComponent(folderId)}` + ).catch((err) => { + logger.error('Failed to download folder:', err) + toast.error(toError(err).message) + }) return } handleDownload(item.file) diff --git a/apps/sim/lib/uploads/client/download.ts b/apps/sim/lib/uploads/client/download.ts index 9cbdd88d263..ed69a2ffdb2 100644 --- a/apps/sim/lib/uploads/client/download.ts +++ b/apps/sim/lib/uploads/client/download.ts @@ -1,5 +1,35 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +/** Hand a fetched blob to the browser as a file save, then release the object URL. */ +function saveBlob(blob: Blob, fileName: string): void { + const objectUrl = URL.createObjectURL(blob) + const anchor = document.createElement('a') + anchor.href = objectUrl + anchor.download = fileName + document.body.appendChild(anchor) + anchor.click() + document.body.removeChild(anchor) + URL.revokeObjectURL(objectUrl) +} + +function fileNameFromDisposition(response: Response, fallback: string): string { + return response.headers.get('Content-Disposition')?.match(/filename="([^"]+)"/)?.[1] ?? fallback +} + +/** + * Read the server's error copy off a failed download so the caller can surface it. + * These routes answer with `{ error }` and the message is written for the user — + * which document is still compiling, which entry is too large. + */ +async function downloadErrorMessage(response: Response, fallback: string): Promise { + try { + const body = await response.json() + return typeof body?.error === 'string' && body.error ? body.error : fallback + } catch { + return fallback + } +} + export async function triggerFileDownload(record: WorkspaceFileRecord): Promise { const isMarkdown = record.type === 'text/markdown' || @@ -10,17 +40,26 @@ export async function triggerFileDownload(record: WorkspaceFileRecord): Promise< ? `/api/files/export/${encodeURIComponent(record.id)}` : `/api/files/serve/${encodeURIComponent(record.key)}?context=workspace&t=${Date.now()}` + // boundary-raw-fetch: binary download read as a blob, not a JSON contract response const response = await fetch(url, { cache: 'no-store' }) - if (!response.ok) throw new Error(`Failed to download file: ${response.statusText}`) + if (!response.ok) { + throw new Error(await downloadErrorMessage(response, `Failed to download "${record.name}"`)) + } - const blob = await response.blob() - const objectUrl = URL.createObjectURL(blob) - const a = document.createElement('a') - a.href = objectUrl - a.download = - response.headers.get('Content-Disposition')?.match(/filename="([^"]+)"/)?.[1] ?? record.name - document.body.appendChild(a) - a.click() - document.body.removeChild(a) - URL.revokeObjectURL(objectUrl) + saveBlob(await response.blob(), fileNameFromDisposition(response, record.name)) +} + +/** + * Download a multi-file selection as a zip. Fetched rather than navigated to, so a + * rejection — a document still compiling, an entry too large — surfaces as an error + * the caller can show in place instead of replacing the page with raw JSON. + */ +export async function triggerArchiveDownload(url: string): Promise { + // boundary-raw-fetch: binary zip download read as a blob, not a JSON contract response + const response = await fetch(url, { cache: 'no-store' }) + if (!response.ok) { + throw new Error(await downloadErrorMessage(response, 'Failed to download the selected files')) + } + + saveBlob(await response.blob(), fileNameFromDisposition(response, 'workspace-files.zip')) } From b0f43c7001df3dabddbdbabf36e3b57239b3ea53 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:38:45 -0700 Subject: [PATCH 04/18] improvement(files): simplify the archive route and stop buffering real uploads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review passes over the branch converged on the same points. Routing every office extension through the buffered path held an entire selection of genuinely uploaded documents in memory — for a check the resolver settles on the first few magic bytes. Only files stored under a generator-source content type need resolving; a real .docx serves what is stored and now streams like anything else, which is what the change was for. With resolution sequential, the AbortController cancelled nothing (its signal never reached the storage read), the success-path over-limit branch was unreachable, and the cancellation guard in the catch could not fire. A plain loop that returns at the point of failure replaces the outcome record, the sentinel, the fan-out helper at limit 1, and four post-hoc scans. ZIP_MATERIALIZE_CONCURRENCY was dead, and the aggregate over-limit message reported a byte count that was by construction under the limit. lazystream and its types are gone: Readable.from over an async generator defers the open the same way and propagates source errors natively, so the PassThrough relay went with them. The client uses requestRaw with the existing binary contract instead of a hand-built query string and a re-typed error extractor, and both archive call sites share one callback. The render headroom moves beside isRenderableDocumentName, the servable reader stops minting a request id per file, and the v1 download returns a view rather than a second full copy. --- apps/sim/app/api/v1/files/[fileId]/route.ts | 32 +-- .../[id]/files/download/route.test.ts | 58 +++-- .../workspaces/[id]/files/download/route.ts | 222 +++++++----------- .../workspace/[workspaceId]/files/files.tsx | 38 ++- apps/sim/lib/uploads/client/download.ts | 64 ++--- .../workspace/workspace-file-manager.ts | 4 +- apps/sim/lib/uploads/utils/file-utils.ts | 7 + apps/sim/package.json | 2 - 8 files changed, 198 insertions(+), 229 deletions(-) diff --git a/apps/sim/app/api/v1/files/[fileId]/route.ts b/apps/sim/app/api/v1/files/[fileId]/route.ts index 40ec025932b..ee5c13c3509 100644 --- a/apps/sim/app/api/v1/files/[fileId]/route.ts +++ b/apps/sim/app/api/v1/files/[fileId]/route.ts @@ -79,20 +79,24 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo { groups: { workspace: workspaceId } } ) - return new Response(new Uint8Array(buffer), { - status: 200, - headers: { - 'Content-Type': contentType || fileRecord.type || 'application/octet-stream', - 'Content-Disposition': `attachment; filename="${fileRecord.name.replace(/[^\w.-]/g, '_')}"; filename*=UTF-8''${encodeURIComponent(fileRecord.name)}`, - 'Content-Length': String(buffer.length), - 'X-File-Id': fileRecord.id, - 'X-File-Name': encodeURIComponent(fileRecord.name), - 'X-Uploaded-At': - fileRecord.uploadedAt instanceof Date - ? fileRecord.uploadedAt.toISOString() - : String(fileRecord.uploadedAt), - }, - }) + // View, not copy — a second full copy would double peak memory for a large file. + return new Response( + new Uint8Array(buffer.buffer as ArrayBuffer, buffer.byteOffset, buffer.byteLength), + { + status: 200, + headers: { + 'Content-Type': contentType || fileRecord.type || 'application/octet-stream', + 'Content-Disposition': `attachment; filename="${fileRecord.name.replace(/[^\w.-]/g, '_')}"; filename*=UTF-8''${encodeURIComponent(fileRecord.name)}`, + 'Content-Length': String(buffer.length), + 'X-File-Id': fileRecord.id, + 'X-File-Name': encodeURIComponent(fileRecord.name), + 'X-Uploaded-At': + fileRecord.uploadedAt instanceof Date + ? fileRecord.uploadedAt.toISOString() + : String(fileRecord.uploadedAt), + }, + } + ) } catch (error) { // A generated doc whose artifact is still compiling is retryable, not a fault: // without this the caller sees a 500 and has no reason to try again. diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index 828f6c4e2e9..dd72250593f 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -71,6 +71,11 @@ function workspaceFile(id: string, name: string, folderId: string | null = 'fold } } +/** A file whose stored bytes are a generator source, so it must be resolved. */ +function generatedDocument(id: string, name: string, folderId: string | null = 'folder-1') { + return { ...workspaceFile(id, name, folderId), type: 'text/x-docxjs' } +} + function requestFor(query: string) { return createMockRequest( 'GET', @@ -96,7 +101,7 @@ describe('workspace files download route', () => { }) it('zips the rendered bytes for a generated doc, not its stored source', async () => { - mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'overview.docx')]) + mockListWorkspaceFiles.mockResolvedValue([generatedDocument('f1', 'overview.docx')]) // A real .docx is a ZIP; the stored source would be plain JS text. const rendered = Buffer.from('PKrendered-docx') mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ @@ -139,7 +144,7 @@ describe('workspace files download route', () => { { id: 'folder-2', name: 'visuals', parentId: 'folder-1' }, ]) mockListWorkspaceFiles.mockResolvedValue([ - workspaceFile('f1', 'summary.docx', 'folder-1'), + generatedDocument('f1', 'summary.docx', 'folder-1'), workspaceFile('f2', 'hero.png', 'folder-2'), ]) mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ @@ -155,8 +160,8 @@ describe('workspace files download route', () => { it('returns 409 naming the documents whose artifacts are still compiling', async () => { mockListWorkspaceFiles.mockResolvedValue([ - workspaceFile('f1', 'ready.docx'), - workspaceFile('f2', 'pending.docx'), + generatedDocument('f1', 'ready.docx'), + generatedDocument('f2', 'pending.docx'), ]) mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { if (file.name === 'pending.docx') @@ -173,7 +178,7 @@ describe('workspace files download route', () => { }) it('rejects with 400, not 500, when a document blows its own allowance', async () => { - mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'huge.docx')]) + mockListWorkspaceFiles.mockResolvedValue([generatedDocument('f1', 'huge.docx')]) mockFetchServableWorkspaceFileBuffer.mockRejectedValue( new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) ) @@ -188,8 +193,8 @@ describe('workspace files download route', () => { it('blames the shared budget once earlier documents have consumed it', async () => { mockListWorkspaceFiles.mockResolvedValue([ - workspaceFile('f1', 'first.docx'), - workspaceFile('f2', 'second.docx'), + generatedDocument('f1', 'first.docx'), + generatedDocument('f2', 'second.docx'), ]) mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { // The first document eats the whole budget, so the second's cap is the remainder. @@ -207,25 +212,40 @@ describe('workspace files download route', () => { expect(body.error).not.toContain('second.docx') }) - it('lets an uploaded office file larger than the render headroom through', async () => { - const big = { ...workspaceFile('f1', 'deck.pptx'), size: 80 * MB } - mockListWorkspaceFiles.mockResolvedValue([big]) - mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ - buffer: Buffer.from('PKdeck'), - contentType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - }) + it('streams an uploaded office file rather than resolving it', async () => { + // A real upload serves exactly its stored bytes, so it must not take the buffered + // path — otherwise a selection of large decks is held in memory for nothing. + const upload = { + ...workspaceFile('f1', 'deck.pptx'), + size: 80 * MB, + type: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + } + mockListWorkspaceFiles.mockResolvedValue([upload]) const response = await GET(requestFor('fileIds=f1'), context) + await zipFrom(response) expect(response.status).toBe(200) - // Capped at the declared size, not the smaller render headroom. - expect(mockFetchServableWorkspaceFileBuffer.mock.calls[0][1].maxBytes).toBe(80 * MB) + expect(mockFetchServableWorkspaceFileBuffer).not.toHaveBeenCalled() + expect(mockDownloadFileStream).toHaveBeenCalledTimes(1) + }) + + it('caps a generated document at the render headroom', async () => { + mockListWorkspaceFiles.mockResolvedValue([generatedDocument('f1', 'report.docx')]) + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: Buffer.from('PKdoc'), + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }) + + await GET(requestFor('fileIds=f1'), context) + + expect(mockFetchServableWorkspaceFileBuffer.mock.calls[0][1].maxBytes).toBe(50 * MB) }) it('surfaces a storage failure as a 500 even when another document is pending', async () => { mockListWorkspaceFiles.mockResolvedValue([ - workspaceFile('f1', 'pending.docx'), - workspaceFile('f2', 'broken.docx'), + generatedDocument('f1', 'pending.docx'), + generatedDocument('f2', 'broken.docx'), ]) mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { if (file.name === 'pending.docx') @@ -241,7 +261,7 @@ describe('workspace files download route', () => { it('stops resolving documents once one hard-fails', async () => { const files = Array.from({ length: 20 }, (_, index) => - workspaceFile(`f${index}`, `doc${index}.docx`) + generatedDocument(`f${index}`, `doc${index}.docx`) ) mockListWorkspaceFiles.mockResolvedValue(files) mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 9b0b14a3ac7..19623d9c69f 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -1,14 +1,17 @@ -import { PassThrough, type Readable } from 'stream' +import { Readable } from 'node:stream' import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { ZipArchive } from 'archiver' -import lazystream from 'lazystream' import { type NextRequest, NextResponse } from 'next/server' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { + DOCXJS_SOURCE_MIME, + PPTXGENJS_SOURCE_MIME, + PYTHON_PDF_SOURCE_MIME, + PYTHON_XLSX_SOURCE_MIME, +} from '@/lib/copilot/tools/server/files/doc-compile' import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -21,7 +24,11 @@ import { listWorkspaceFiles, } from '@/lib/uploads/contexts/workspace' import { downloadFileStream } from '@/lib/uploads/core/storage-service' -import { formatFileSize, isRenderableDocumentName } from '@/lib/uploads/utils/file-utils' +import { + formatFileSize, + isRenderableDocumentName, + RENDERED_DOCUMENT_HEADROOM_BYTES, +} from '@/lib/uploads/utils/file-utils' import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/servable-file-response' import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' @@ -29,49 +36,55 @@ import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' const logger = createLogger('WorkspaceFilesDownloadAPI') const MAX_ZIP_DOWNLOAD_FILES = 100 const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 + +/** Content types under which a generated document's *source* is stored. */ +const GENERATED_SOURCE_TYPES = new Set([ + DOCXJS_SOURCE_MIME, + PPTXGENJS_SOURCE_MIME, + PYTHON_PDF_SOURCE_MIME, + PYTHON_XLSX_SOURCE_MIME, +]) + /** - * Headroom a document may render to beyond what it declared. Office extensions - * cover both ordinary uploads — which serve exactly their declared size — and - * source-backed generated docs, and the two are indistinguishable before the bytes - * are read. So the allowance is the larger of the declared size and this ceiling: - * an uploaded 80 MiB deck still downloads, while a 6 KiB generator source cannot - * quietly render into hundreds of megabytes. + * Whether this entry's stored bytes are a generation source that has to be resolved + * before it can go in the archive. An ordinary uploaded `.docx` serves exactly what is + * stored, so it streams like anything else — routing every office extension through the + * buffered path would hold a whole selection of real documents in memory for a check + * the resolver settles on the first few magic bytes. Metadata without a type falls back + * to the extension: better to resolve and pass through than to stream source text under + * a document name. */ -const RENDERED_DOCUMENT_HEADROOM_BYTES = 50 * 1024 * 1024 +function needsRendering(file: WorkspaceFileRecord): boolean { + return file.type ? GENERATED_SOURCE_TYPES.has(file.type) : isRenderableDocumentName(file.name) +} + /** - * Fan-out for files that serve exactly their declared size. Their total is already - * bounded by the pre-download check, so concurrency cannot push residency past it. - * Renderable documents are read one at a time instead — see below. + * A `Readable` that opens its storage read on first pull rather than up front. The + * archiver works through entries sequentially, so handing it an open stream per entry + * would hold a connection per selected file — more than the storage client pools — with + * most sitting idle until their turn. The generator body does not run until the first + * read, and a failure to open surfaces as the stream's `error` event. */ -const ZIP_MATERIALIZE_CONCURRENCY = 5 +function lazyWorkspaceFileStream(file: WorkspaceFileRecord): Readable { + return Readable.from( + (async function* () { + yield* await downloadFileStream({ + key: file.key, + context: file.storageContext ?? 'workspace', + }) + })() + ) +} -function overLimitResponse(bytes: number, qualifier = ''): NextResponse { +function selectionTooLargeResponse(bytes: number): NextResponse { return NextResponse.json( { - error: `Selected files total ${formatFileSize(bytes)}${qualifier}, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`, + error: `Selected files total ${formatFileSize(bytes)}, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`, }, { status: 400 } ) } -/** - * A `Readable` that opens its storage read on first pull rather than up front. Handing - * the archiver one open stream per entry would hold a connection per selected file — - * more than the storage client pools — with most sitting idle until their turn. - */ -function lazyWorkspaceFileStream(file: WorkspaceFileRecord): Readable { - return new lazystream.Readable(() => { - const relay = new PassThrough() - downloadFileStream({ key: file.key, context: file.storageContext ?? 'workspace' }) - .then((source) => { - source.on('error', (error) => relay.destroy(toError(error))) - source.pipe(relay) - }) - .catch((error) => relay.destroy(toError(error))) - return relay - }) -} - function collectDescendantFolderIds( selectedFolderIds: string[], folders: Array<{ id: string; parentId: string | null }> @@ -135,118 +148,49 @@ export const GET = withRouteHandler( const declaredBytes = filesToZip.reduce((sum, file) => sum + file.size, 0) if (declaredBytes > MAX_ZIP_DOWNLOAD_BYTES) { - return overLimitResponse(declaredBytes) + return selectionTooLargeResponse(declaredBytes) } - // Generated docs (docx/pptx/pdf/xlsx) store their generation source, not the - // rendered binary, so bytes are resolved through the servable reader — a raw - // read ships source text under a `.docx` name. - // - // Only those documents can exceed the size they declared, so only they can push - // residency past the budget the pre-download check already enforced. They are - // therefore read one at a time, which keeps the overshoot to a single entry; - // everything else stays parallel because its bytes are exactly its declared size. - // Once the budget is blown or a read hard-fails, the abort flag stops reads that - // have not started yet. - const controller = new AbortController() + // Generated documents are resolved before the archive starts. They are the only + // entries whose bytes decide anything, and every status this route returns comes + // from them — once the first byte is written the status code is committed. They + // resolve one at a time so at most one rendered document is resident. + const renderedDocuments = new Map() + const pendingNames: string[] = [] let renderedBytes = 0 - interface DownloadOutcome { - buffer: Buffer | null - pendingName: string | null - /** Set only when an entry's own allowance bound it, never the shared budget. */ - overLimitEntry: { name: string; allowance: number } | null - overLimit: boolean - error: unknown - } - const skipped: DownloadOutcome = { - buffer: null, - pendingName: null, - overLimitEntry: null, - overLimit: false, - error: null, - } + for (const file of filesToZip) { + if (!needsRendering(file)) continue + + const remaining = MAX_ZIP_DOWNLOAD_BYTES - renderedBytes + // A source renders to something larger than it declared, so the allowance is the + // render headroom — bounded by what is left of the request's budget. + const allowance = Math.min(remaining, RENDERED_DOCUMENT_HEADROOM_BYTES) - const readEntry = async (file: WorkspaceFileRecord): Promise => { - if (controller.signal.aborted) return skipped - const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes) - // Uploads and source-backed documents are indistinguishable before the bytes - // are read, so the allowance is the larger of the declared size and the render - // headroom: a real 80 MiB deck still downloads, a small generator source cannot - // render unbounded. - const allowance = Math.max(file.size, RENDERED_DOCUMENT_HEADROOM_BYTES) try { - const { buffer } = await fetchServableWorkspaceFileBuffer(file, { - maxBytes: Math.min(remaining, allowance), - signal: controller.signal, - }) + const { buffer } = await fetchServableWorkspaceFileBuffer(file, { maxBytes: allowance }) renderedBytes += buffer.length - const overLimit = renderedBytes > MAX_ZIP_DOWNLOAD_BYTES - if (overLimit) controller.abort() - return { ...skipped, buffer, overLimit } + renderedDocuments.set(file.id, buffer) } catch (error) { - // Recorded even when another worker already aborted: a size rejection - // describes this file, so losing it to someone else's cancellation would - // downgrade an actionable 400 into an opaque 500. if (error instanceof PayloadSizeLimitError) { - controller.abort() - // Attributed to this entry when its own cap was the binding one — ties - // included, since the entry's ceiling still made it unshippable and - // downloading it alone is the way through. Otherwise the budget ran out. - return { - ...skipped, - overLimit: true, - overLimitEntry: allowance <= remaining ? { name: file.name, allowance } : null, - } + // Blamed on the entry when its own ceiling was the binding cap; otherwise the + // documents ahead of it have consumed the budget. + return allowance === RENDERED_DOCUMENT_HEADROOM_BYTES + ? NextResponse.json( + { + error: `"${file.name}" renders to more than ${formatFileSize(RENDERED_DOCUMENT_HEADROOM_BYTES)} and is too large to include in a zip; download it on its own instead.`, + }, + { status: 400 } + ) + : selectionTooLargeResponse(declaredBytes) } - // Any other error from an already-aborted read is a consequence of the - // cancellation, not a cause. Checked before this worker aborts anything so - // the worker that actually failed still records its own error. - if (controller.signal.aborted) return skipped - // A pending artifact is worth reporting in full, so keep resolving the - // rest of the selection; anything else dooms the request. - const pending = isDocNotReadyError(error) - if (!pending) controller.abort() - return { ...skipped, pendingName: pending ? file.name : null, error } + // Pending artifacts are collected so the 409 can name all of them; anything + // else dooms the request and waiting cannot fix it. + if (!isDocNotReadyError(error)) throw error + pendingNames.push(file.name) } } - // Documents are resolved up front, one at a time: they are the only entries that - // need their bytes in hand, and every status this route can return is decided - // from them. Nothing may fail after the first byte is written — the status code - // is committed by then — so this has to happen before the archive starts. - const documentBuffers = new Map() - const documents = filesToZip.filter((file) => isRenderableDocumentName(file.name)) - const downloads = await mapWithConcurrency(documents, 1, (file) => readEntry(file)) - downloads.forEach((outcome, index) => { - if (outcome.buffer) documentBuffers.set(documents[index].id, outcome.buffer) - }) - - // Size first: the request cannot succeed at any size-adjacent retry, and a - // descriptive 400 beats an opaque 500 raised by whatever the abort cancelled. - const overLimitEntry = downloads.find((result) => result.overLimitEntry)?.overLimitEntry - if (overLimitEntry) { - // Naming the entry that blew its own allowance, and quoting the allowance that - // actually applied: an aggregate message here would tell the user to select - // fewer files when the selection was fine. - return NextResponse.json( - { - error: `"${overLimitEntry.name}" is too large to include in a zip. Entries are capped at ${formatFileSize(overLimitEntry.allowance)}; download it on its own instead.`, - }, - { status: 400 } - ) - } - - if (downloads.some((result) => result.overLimit)) { - return overLimitResponse(renderedBytes, ' once documents are rendered') - } - - // A hard failure outranks a pending artifact: waiting cannot fix it, so a 409 - // would send the client into a retry loop that never succeeds. - const failure = downloads.find((result) => result.error && !result.pendingName) - if (failure?.error) throw failure.error - - const pendingNames = downloads.flatMap((result) => result.pendingName ?? []) if (pendingNames.length > 0) { return NextResponse.json({ error: docNotReadyMessage(pendingNames) }, { status: 409 }) } @@ -261,17 +205,15 @@ export const GET = withRouteHandler( ) // Ordinary files are never materialized: each entry opens its storage read only - // when the archiver reaches it (`lazystream` defers the factory until first read), - // so peak memory is one entry rather than the whole selection. Resolved documents - // are appended from the buffers above. + // when the archiver reaches it, so one entry is resident rather than the archive. const archive = new ZipArchive({ store: true }) archive.on('warning', (error: Error) => { logger.warn('Archive warning while streaming workspace files', { error }) }) filesToZip.forEach((file, index) => { - const buffer = documentBuffers.get(file.id) - archive.append(buffer ?? lazyWorkspaceFileStream(file), { name: entryPaths[index] }) + const rendered = renderedDocuments.get(file.id) + archive.append(rendered ?? lazyWorkspaceFileStream(file), { name: entryPaths[index] }) }) void archive.finalize() diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 2f85a03f24b..94126965733 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -1072,6 +1072,18 @@ export function Files() { setShowDeleteConfirm(true) }, [selectedFileIds, selectedFolderIds, files, folders]) + const downloadArchive = useCallback( + async (selection: { fileIds?: string[]; folderIds?: string[] }) => { + try { + await triggerArchiveDownload({ workspaceId, ...selection }) + } catch (err) { + logger.error('Failed to download selection:', err) + toast.error(toError(err).message) + } + }, + [workspaceId] + ) + const handleBulkDownload = useCallback(async () => { const selectedFiles = files.filter((file) => selectedFileIds.includes(file.id)) if (selectedFiles.length === 1 && selectedFolderIds.length === 0) { @@ -1079,25 +1091,14 @@ export function Files() { return } - const query = new URLSearchParams() - for (const fileId of selectedFileIds) query.append('fileIds', fileId) - for (const folderId of selectedFolderIds) query.append('folderIds', folderId) - - if (query.size === 0) return + if (selectedFileIds.length === 0 && selectedFolderIds.length === 0) return captureEvent(posthogRef.current, 'file_downloaded', { workspace_id: workspaceId, is_bulk: true, file_count: selectedFileIds.length + selectedFolderIds.length, }) - try { - await triggerArchiveDownload( - `/api/workspaces/${workspaceId}/files/download?${query.toString()}` - ) - } catch (err) { - logger.error('Failed to download selection:', err) - toast.error(toError(err).message) - } - }, [selectedFileIds, selectedFolderIds, files, handleDownload, workspaceId]) + await downloadArchive({ fileIds: selectedFileIds, folderIds: selectedFolderIds }) + }, [selectedFileIds, selectedFolderIds, files, handleDownload, downloadArchive, workspaceId]) const fileDetailBreadcrumbs = useMemo(() => { if (!selectedFile) return [] @@ -1295,17 +1296,12 @@ export function Files() { if (item.kind === 'folder') { const folderId = item.folder.id closeContextMenu() - triggerArchiveDownload( - `/api/workspaces/${workspaceId}/files/download?folderIds=${encodeURIComponent(folderId)}` - ).catch((err) => { - logger.error('Failed to download folder:', err) - toast.error(toError(err).message) - }) + void downloadArchive({ folderIds: [folderId] }) return } handleDownload(item.file) closeContextMenu() - }, [selectedRowIds, handleBulkDownload, closeContextMenu, workspaceId, handleDownload]) + }, [selectedRowIds, handleBulkDownload, closeContextMenu, downloadArchive, handleDownload]) const handleContextMenuRename = useCallback(() => { const item = contextMenuItemRef.current diff --git a/apps/sim/lib/uploads/client/download.ts b/apps/sim/lib/uploads/client/download.ts index ed69a2ffdb2..337ce57b8a3 100644 --- a/apps/sim/lib/uploads/client/download.ts +++ b/apps/sim/lib/uploads/client/download.ts @@ -1,33 +1,29 @@ +import { requestRaw } from '@/lib/api/client/request' +import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' /** Hand a fetched blob to the browser as a file save, then release the object URL. */ -function saveBlob(blob: Blob, fileName: string): void { +export function saveBlob(blob: Blob, fileName: string): void { const objectUrl = URL.createObjectURL(blob) const anchor = document.createElement('a') anchor.href = objectUrl anchor.download = fileName - document.body.appendChild(anchor) anchor.click() - document.body.removeChild(anchor) - URL.revokeObjectURL(objectUrl) + // Deferred: revoking synchronously after click() can race the download starting. + setTimeout(() => URL.revokeObjectURL(objectUrl), 0) } function fileNameFromDisposition(response: Response, fallback: string): string { - return response.headers.get('Content-Disposition')?.match(/filename="([^"]+)"/)?.[1] ?? fallback -} - -/** - * Read the server's error copy off a failed download so the caller can surface it. - * These routes answer with `{ error }` and the message is written for the user — - * which document is still compiling, which entry is too large. - */ -async function downloadErrorMessage(response: Response, fallback: string): Promise { - try { - const body = await response.json() - return typeof body?.error === 'string' && body.error ? body.error : fallback - } catch { - return fallback + const disposition = response.headers.get('Content-Disposition') ?? '' + const encoded = disposition.match(/filename\*=UTF-8''([^;]+)/i)?.[1] + if (encoded) { + try { + return decodeURIComponent(encoded) + } catch { + // Fall through to the plain form. + } } + return disposition.match(/filename="([^"]+)"/)?.[1] ?? fallback } export async function triggerFileDownload(record: WorkspaceFileRecord): Promise { @@ -40,26 +36,32 @@ export async function triggerFileDownload(record: WorkspaceFileRecord): Promise< ? `/api/files/export/${encodeURIComponent(record.id)}` : `/api/files/serve/${encodeURIComponent(record.key)}?context=workspace&t=${Date.now()}` - // boundary-raw-fetch: binary download read as a blob, not a JSON contract response + // boundary-raw-fetch: binary download read as a blob; these paths have no contract const response = await fetch(url, { cache: 'no-store' }) - if (!response.ok) { - throw new Error(await downloadErrorMessage(response, `Failed to download "${record.name}"`)) - } + if (!response.ok) throw new Error(`Failed to download "${record.name}"`) saveBlob(await response.blob(), fileNameFromDisposition(response, record.name)) } /** - * Download a multi-file selection as a zip. Fetched rather than navigated to, so a - * rejection — a document still compiling, an entry too large — surfaces as an error - * the caller can show in place instead of replacing the page with raw JSON. + * Download a selection of files as a zip. Fetched rather than navigated to, so a + * rejection — a document still compiling, an entry too large — surfaces as an error the + * caller can show in place instead of replacing the page with raw JSON. `requestRaw` + * throws an `ApiClientError` carrying the route's own message. */ -export async function triggerArchiveDownload(url: string): Promise { - // boundary-raw-fetch: binary zip download read as a blob, not a JSON contract response - const response = await fetch(url, { cache: 'no-store' }) - if (!response.ok) { - throw new Error(await downloadErrorMessage(response, 'Failed to download the selected files')) - } +export async function triggerArchiveDownload(input: { + workspaceId: string + fileIds?: string[] + folderIds?: string[] +}): Promise { + const response = await requestRaw( + downloadWorkspaceFileItemsContract, + { + params: { id: input.workspaceId }, + query: { fileIds: input.fileIds ?? [], folderIds: input.folderIds ?? [] }, + }, + { cache: 'no-store' } + ) saveBlob(await response.blob(), fileNameFromDisposition(response, 'workspace-files.zip')) } diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 5b490c8168e..0c0780fd842 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -982,7 +982,7 @@ export async function getWorkspaceFile( */ export async function fetchServableWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, - options: { maxBytes?: number; signal?: AbortSignal } = {} + options: { maxBytes?: number; signal?: AbortSignal; requestId?: string } = {} ): Promise<{ buffer: Buffer; contentType: string }> { const { downloadServableFileFromStorage } = await import('@/lib/uploads/utils/file-utils.server') @@ -996,7 +996,7 @@ export async function fetchServableWorkspaceFileBuffer( key: fileRecord.key, context: fileRecord.storageContext ?? 'workspace', }, - generateRequestId(), + options.requestId ?? generateRequestId(), logger, options ) diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 442d52d324f..79c3bf77f9c 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -217,6 +217,13 @@ export function getFileExtension(filename: string): string { */ const RENDERABLE_DOCUMENT_EXTENSIONS = new Set(['pdf', 'docx', 'pptx', 'xlsx']) +/** + * How far a generated document may render beyond the source it declared. A generator + * source is text and is orders of magnitude smaller than the document it produces, so + * this bounds the expansion rather than the document. + */ +export const RENDERED_DOCUMENT_HEADROOM_BYTES = 50 * 1024 * 1024 + /** True when `fileName` may be backed by a generation source rather than final bytes. */ export function isRenderableDocumentName(fileName: string): boolean { return RENDERABLE_DOCUMENT_EXTENSIONS.has(getFileExtension(fileName)) diff --git a/apps/sim/package.json b/apps/sim/package.json index 16ecf6329e4..c82474e0c36 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -171,7 +171,6 @@ "js-tiktoken": "1.0.21", "js-yaml": "4.3.0", "jszip": "3.10.1", - "lazystream": "1.0.1", "lru-cache": "11.3.6", "lucide-react": "^0.511.0", "mammoth": "^1.9.0", @@ -239,7 +238,6 @@ "@types/html-to-text": "9.0.4", "@types/js-yaml": "4.0.9", "@types/jsdom": "21.1.7", - "@types/lazystream": "1.0.0", "@types/micromatch": "4.0.10", "@types/node": "24.2.1", "@types/nodemailer": "8.0.1", From 98d6b69f2ca3d8917a39bb6178a8e24b08417633 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:41:12 -0700 Subject: [PATCH 05/18] improvement(files): keep the lazy entry stream in byte mode Readable.from defaults to object mode; these chunks are bytes headed for an archive, so the mode is now explicit. Also makes the fire-and-forget bulk download call consistent with its sibling. --- apps/sim/app/api/workspaces/[id]/files/download/route.ts | 4 +++- apps/sim/app/workspace/[workspaceId]/files/files.tsx | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 19623d9c69f..246598d7a6a 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -72,7 +72,9 @@ function lazyWorkspaceFileStream(file: WorkspaceFileRecord): Readable { key: file.key, context: file.storageContext ?? 'workspace', }) - })() + })(), + // `Readable.from` defaults to object mode; these are bytes headed for an archive. + { objectMode: false } ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 94126965733..b622a030d37 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -1289,7 +1289,7 @@ export function Files() { if (!item) return const rowId = item.kind === 'file' ? fileRowId(item.file.id) : folderRowId(item.folder.id) if (selectedRowIds.has(rowId) && selectedRowIds.size > 1) { - handleBulkDownload() + void handleBulkDownload() closeContextMenu() return } From 7d73714529abc45a1d9f7c6480fa857a6d9ecaad Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:41:54 -0700 Subject: [PATCH 06/18] improvement(files): correct the servable reader's 409 note Batch callers build the response from docNotReadyMessage rather than through docNotReadyResponse, so the doc comment now describes the outcome instead of naming one of the two helpers. --- .../lib/uploads/contexts/workspace/workspace-file-manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 0c0780fd842..b1ae9b45da2 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -978,7 +978,7 @@ export async function getWorkspaceFile( * source itself is wanted (style extraction, compile checks, the copilot VFS). * * Throws `DocCompileUserError` when a generated doc's artifact is still compiling — - * callers surface a retryable 409 via `docNotReadyResponse` rather than shipping source. + * callers turn that into a retryable 409 rather than shipping source. */ export async function fetchServableWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, From d197af1ca96fe896f32538be1260dc6ca1fa73de Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:44:49 -0700 Subject: [PATCH 07/18] improvement(files): correct two comments that described the wrong behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resolution loop's comment claimed at most one rendered document is resident. It is not: every resolved buffer is held until the archive is assembled, bounded by the request's remaining budget. The loop resolves one at a time, it does not release one at a time. RENDERED_DOCUMENT_HEADROOM_BYTES was documented as bounding the expansion beyond the declared size, but it is passed straight through as maxBytes and is an absolute ceiling on the rendered document — renamed to MAX_RENDERED_DOCUMENT_BYTES so the name matches. Download failures also carry a fallback message, so a transport error surfaces something better than a bare "Failed to fetch". --- .../workspaces/[id]/files/download/route.ts | 20 +++++++++---------- .../workspace/[workspaceId]/files/files.tsx | 4 ++-- apps/sim/lib/core/utils/node-stream.ts | 2 +- apps/sim/lib/uploads/client/download.ts | 1 - apps/sim/lib/uploads/utils/file-utils.ts | 8 ++++---- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 246598d7a6a..45876c18a17 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -27,7 +27,7 @@ import { downloadFileStream } from '@/lib/uploads/core/storage-service' import { formatFileSize, isRenderableDocumentName, - RENDERED_DOCUMENT_HEADROOM_BYTES, + MAX_RENDERED_DOCUMENT_BYTES, } from '@/lib/uploads/utils/file-utils' import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/servable-file-response' import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' @@ -153,10 +153,10 @@ export const GET = withRouteHandler( return selectionTooLargeResponse(declaredBytes) } - // Generated documents are resolved before the archive starts. They are the only - // entries whose bytes decide anything, and every status this route returns comes - // from them — once the first byte is written the status code is committed. They - // resolve one at a time so at most one rendered document is resident. + // Generated documents are resolved before the archive starts: once the first byte + // is written the status code is committed, so anything that can still fail the + // request has to fail here. Their buffers are held until the archive is assembled, + // bounded by what is left of the request's byte budget. const renderedDocuments = new Map() const pendingNames: string[] = [] let renderedBytes = 0 @@ -165,9 +165,9 @@ export const GET = withRouteHandler( if (!needsRendering(file)) continue const remaining = MAX_ZIP_DOWNLOAD_BYTES - renderedBytes - // A source renders to something larger than it declared, so the allowance is the - // render headroom — bounded by what is left of the request's budget. - const allowance = Math.min(remaining, RENDERED_DOCUMENT_HEADROOM_BYTES) + // A source's declared size says nothing about what it renders to, so the cap is + // the per-document ceiling, bounded by what is left of the budget. + const allowance = Math.min(remaining, MAX_RENDERED_DOCUMENT_BYTES) try { const { buffer } = await fetchServableWorkspaceFileBuffer(file, { maxBytes: allowance }) @@ -177,10 +177,10 @@ export const GET = withRouteHandler( if (error instanceof PayloadSizeLimitError) { // Blamed on the entry when its own ceiling was the binding cap; otherwise the // documents ahead of it have consumed the budget. - return allowance === RENDERED_DOCUMENT_HEADROOM_BYTES + return allowance === MAX_RENDERED_DOCUMENT_BYTES ? NextResponse.json( { - error: `"${file.name}" renders to more than ${formatFileSize(RENDERED_DOCUMENT_HEADROOM_BYTES)} and is too large to include in a zip; download it on its own instead.`, + error: `"${file.name}" renders to more than ${formatFileSize(MAX_RENDERED_DOCUMENT_BYTES)} and is too large to include in a zip; download it on its own instead.`, }, { status: 400 } ) diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index b622a030d37..3c894f3b8cc 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -951,7 +951,7 @@ export function Files() { }) } catch (err) { logger.error('Failed to download file:', err) - toast.error(toError(err).message) + toast.error(getErrorMessage(err, `Failed to download "${file.name}"`)) } }, [workspaceId] @@ -1078,7 +1078,7 @@ export function Files() { await triggerArchiveDownload({ workspaceId, ...selection }) } catch (err) { logger.error('Failed to download selection:', err) - toast.error(toError(err).message) + toast.error(getErrorMessage(err, 'Failed to download the selected files')) } }, [workspaceId] diff --git a/apps/sim/lib/core/utils/node-stream.ts b/apps/sim/lib/core/utils/node-stream.ts index 4918c5d77d3..1288a2280d7 100644 --- a/apps/sim/lib/core/utils/node-stream.ts +++ b/apps/sim/lib/core/utils/node-stream.ts @@ -1,4 +1,4 @@ -import type { Readable } from 'stream' +import type { Readable } from 'node:stream' /** * Bridges a Node `Readable` into a WHATWG `ReadableStream` suitable for a `Response` diff --git a/apps/sim/lib/uploads/client/download.ts b/apps/sim/lib/uploads/client/download.ts index 337ce57b8a3..9f41f99f24e 100644 --- a/apps/sim/lib/uploads/client/download.ts +++ b/apps/sim/lib/uploads/client/download.ts @@ -2,7 +2,6 @@ import { requestRaw } from '@/lib/api/client/request' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -/** Hand a fetched blob to the browser as a file save, then release the object URL. */ export function saveBlob(blob: Blob, fileName: string): void { const objectUrl = URL.createObjectURL(blob) const anchor = document.createElement('a') diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 79c3bf77f9c..54f0e548117 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -218,11 +218,11 @@ export function getFileExtension(filename: string): string { const RENDERABLE_DOCUMENT_EXTENSIONS = new Set(['pdf', 'docx', 'pptx', 'xlsx']) /** - * How far a generated document may render beyond the source it declared. A generator - * source is text and is orders of magnitude smaller than the document it produces, so - * this bounds the expansion rather than the document. + * Ceiling on a single rendered generated document. A generator source is text and is + * orders of magnitude smaller than the document it produces, so the declared size is no + * bound at all and the rendered bytes need a cap of their own. */ -export const RENDERED_DOCUMENT_HEADROOM_BYTES = 50 * 1024 * 1024 +export const MAX_RENDERED_DOCUMENT_BYTES = 50 * 1024 * 1024 /** True when `fileName` may be backed by a generation source rather than final bytes. */ export function isRenderableDocumentName(fileName: string): boolean { From 912da6fd45ac38d5f63feb2b5638ac06bcb8cbd5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:53:09 -0700 Subject: [PATCH 08/18] fix(files): put streamed files and rendered documents on one byte budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The budget only counted rendered documents, so streamed entries never reserved their declared bytes. A mixed selection could clear the up-front declared-size gate and still ship close to two full limits: ordinary files up to the cap, plus documents drawing a fresh cap of their own. Streamed entries ship exactly what they declared, so their share is known before anything is read and is now reserved up front; documents draw from what is left. The budget-exhausted rejection also quoted declared sizes, which for a selection of small sources reads as a tiny total exceeding the limit. That case has no knowable byte count — the documents have not been rendered — so it now says what happened without inventing a number. --- .../[id]/files/download/route.test.ts | 40 ++++++++++++++++++- .../workspaces/[id]/files/download/route.ts | 25 +++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index dd72250593f..8879f17a813 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -191,6 +191,42 @@ describe('workspace files download route', () => { expect(body.error).not.toContain('Selected files total') }) + it('counts streamed files against the same budget as rendered documents', async () => { + // 200 MB of ordinary files leaves 50 MB of the 250 MB budget for documents. + mockListWorkspaceFiles.mockResolvedValue([ + { ...workspaceFile('f1', 'clip.mp4'), size: 200 * MB }, + generatedDocument('f2', 'report.docx'), + ]) + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: Buffer.from('PKdoc'), + contentType: 'application/octet-stream', + }) + + await zipFrom(await GET(requestFor('fileIds=f1&fileIds=f2'), context)) + + // Without reserving the streamed bytes the document would get the full 50 MB + // ceiling, letting the archive ship 250 MB of documents on top of 200 MB of video. + expect(mockFetchServableWorkspaceFileBuffer.mock.calls[0][1].maxBytes).toBe(50 * MB) + }) + + it('rejects when streamed files leave no budget for a document', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + { ...workspaceFile('f1', 'clip.mp4'), size: 249 * MB }, + generatedDocument('f2', 'report.docx'), + ]) + mockFetchServableWorkspaceFileBuffer.mockRejectedValue( + new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) + ) + + const response = await GET(requestFor('fileIds=f1&fileIds=f2'), context) + + expect(response.status).toBe(400) + const body = await response.json() + expect(body.error).toContain('once documents are rendered') + // No byte count: the rendered total is not knowable, so quoting one would mislead. + expect(body.error).not.toContain('Selected files total') + }) + it('blames the shared budget once earlier documents have consumed it', async () => { mockListWorkspaceFiles.mockResolvedValue([ generatedDocument('f1', 'first.docx'), @@ -208,7 +244,7 @@ describe('workspace files download route', () => { expect(response.status).toBe(400) const body = await response.json() - expect(body.error).toContain('Selected files total') + expect(body.error).toContain('once documents are rendered') expect(body.error).not.toContain('second.docx') }) @@ -237,7 +273,7 @@ describe('workspace files download route', () => { contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', }) - await GET(requestFor('fileIds=f1'), context) + await zipFrom(await GET(requestFor('fileIds=f1'), context)) expect(mockFetchServableWorkspaceFileBuffer.mock.calls[0][1].maxBytes).toBe(50 * MB) }) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 45876c18a17..d236024c60d 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -87,6 +87,20 @@ function selectionTooLargeResponse(bytes: number): NextResponse { ) } +/** + * The rendered archive would exceed the limit even though the declared sizes did not — + * generated documents render to more than the source they declared, so no accurate byte + * count exists to quote here. + */ +function archiveTooLargeResponse(): NextResponse { + return NextResponse.json( + { + error: `The selected files exceed the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit once documents are rendered. Select fewer files.`, + }, + { status: 400 } + ) +} + function collectDescendantFolderIds( selectedFolderIds: string[], folders: Array<{ id: string; parentId: string | null }> @@ -153,6 +167,13 @@ export const GET = withRouteHandler( return selectionTooLargeResponse(declaredBytes) } + // Streamed entries ship exactly what they declared, so their share of the budget is + // known up front and is reserved here. Documents then draw from what is left — + // one budget across both kinds, or the archive could ship two full limits' worth. + const reservedForStreamed = filesToZip + .filter((file) => !needsRendering(file)) + .reduce((sum, file) => sum + file.size, 0) + // Generated documents are resolved before the archive starts: once the first byte // is written the status code is committed, so anything that can still fail the // request has to fail here. Their buffers are held until the archive is assembled, @@ -164,7 +185,7 @@ export const GET = withRouteHandler( for (const file of filesToZip) { if (!needsRendering(file)) continue - const remaining = MAX_ZIP_DOWNLOAD_BYTES - renderedBytes + const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - reservedForStreamed - renderedBytes) // A source's declared size says nothing about what it renders to, so the cap is // the per-document ceiling, bounded by what is left of the budget. const allowance = Math.min(remaining, MAX_RENDERED_DOCUMENT_BYTES) @@ -184,7 +205,7 @@ export const GET = withRouteHandler( }, { status: 400 } ) - : selectionTooLargeResponse(declaredBytes) + : archiveTooLargeResponse() } // Pending artifacts are collected so the 409 can name all of them; anything // else dooms the request and waiting cannot fix it. From 2477737960fd62c066728da83525259d44282dc5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 18:01:04 -0700 Subject: [PATCH 09/18] fix(files): attach the download anchor and handle a finalize rejection A detached anchor's click() works in current browsers, but every other download helper in the app attaches first, and a silent no-op on this path would look exactly like a download that never started. Not worth the ambiguity for two DOM operations. archive.finalize() was fired with void, so a failure after the response had started became an unhandled rejection on top of the stream error event that already fails the response. It is caught and logged now. --- apps/sim/app/api/workspaces/[id]/files/download/route.ts | 6 +++++- apps/sim/lib/uploads/client/download.ts | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index d236024c60d..7c926287750 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -238,7 +238,11 @@ export const GET = withRouteHandler( const rendered = renderedDocuments.get(file.id) archive.append(rendered ?? lazyWorkspaceFileStream(file), { name: entryPaths[index] }) }) - void archive.finalize() + archive.finalize().catch((error) => { + // The archive's `error` event already fails the response stream; this keeps the + // same failure from also surfacing as an unhandled rejection. + logger.error('Failed to finalize workspace file archive', { error }) + }) recordAudit({ workspaceId, diff --git a/apps/sim/lib/uploads/client/download.ts b/apps/sim/lib/uploads/client/download.ts index 9f41f99f24e..ac873f66ae8 100644 --- a/apps/sim/lib/uploads/client/download.ts +++ b/apps/sim/lib/uploads/client/download.ts @@ -7,7 +7,12 @@ export function saveBlob(blob: Blob, fileName: string): void { const anchor = document.createElement('a') anchor.href = objectUrl anchor.download = fileName + // Attached before clicking: a detached anchor works in current browsers, but every + // other download helper in the app attaches, and a silent no-op here would look + // exactly like a download that never started. + document.body.appendChild(anchor) anchor.click() + document.body.removeChild(anchor) // Deferred: revoking synchronously after click() can race the download starting. setTimeout(() => URL.revokeObjectURL(objectUrl), 0) } From 65be47e968356870bd8819066e77f0543f21008d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 18:13:36 -0700 Subject: [PATCH 10/18] fix(files): guard the archive download against concurrent clicks Navigating to the route meant a second click just re-navigated. Fetching the archive instead means each click starts another download that holds the whole zip in tab memory, and the Download button gave no sign anything was happening. The button now reflects the in-flight download alongside the existing move and delete states. The ref guard is there as well as the state because two clicks in the same tick would both pass a state check. --- .../app/workspace/[workspaceId]/files/files.tsx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 3c894f3b8cc..051a807f233 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -1072,13 +1072,24 @@ export function Files() { setShowDeleteConfirm(true) }, [selectedFileIds, selectedFolderIds, files, folders]) + const [isDownloadingArchive, setIsDownloadingArchive] = useState(false) + // Ref as well as state: two clicks in the same tick would both pass a state check, + // and each concurrent archive holds the whole zip in tab memory. + const archiveDownloadInFlightRef = useRef(false) + const downloadArchive = useCallback( async (selection: { fileIds?: string[]; folderIds?: string[] }) => { + if (archiveDownloadInFlightRef.current) return + archiveDownloadInFlightRef.current = true + setIsDownloadingArchive(true) try { await triggerArchiveDownload({ workspaceId, ...selection }) } catch (err) { logger.error('Failed to download selection:', err) toast.error(getErrorMessage(err, 'Failed to download the selected files')) + } finally { + archiveDownloadInFlightRef.current = false + setIsDownloadingArchive(false) } }, [workspaceId] @@ -1991,7 +2002,9 @@ export function Files() { onMove={canEdit ? handleContextMenuMove : undefined} moveOptions={canEdit ? contextMenuMoveOptions : undefined} onDelete={canEdit ? handleBulkDelete : undefined} - isLoading={bulkArchiveItems.isPending || moveItems.isPending} + isLoading={ + bulkArchiveItems.isPending || moveItems.isPending || isDownloadingArchive + } /> {isDraggingOver ? (
From 44de9c796459860b37183b685c97b7a074320c3a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 18:24:44 -0700 Subject: [PATCH 11/18] fix(files): resolve every generated-document source type, not four of five MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The archive keyed off a locally enumerated set of source content types that omitted text/x-pdflibjs — the isolated-vm PDF generator. Those files failed the check and streamed their generator source under a .pdf name, which is the exact corruption this change exists to fix. A canonical five-member set already existed in the file viewer; enumerating a fourth copy was the mistake, not the missing string. The set moves to file-utils beside the other file-type predicates, the viewer imports it, and the route asks isGeneratedDocumentSourceType rather than carrying its own list. A parameterized test pins all five, so adding a generator without extending the set fails. --- .../[id]/files/download/route.test.ts | 21 +++++++++++++++++++ .../workspaces/[id]/files/download/route.ts | 17 ++------------- .../file-viewer/use-editable-file-content.ts | 9 ++------ apps/sim/lib/uploads/utils/file-utils.ts | 19 +++++++++++++++++ 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index 8879f17a813..41dc0680b7b 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -248,6 +248,27 @@ describe('workspace files download route', () => { expect(body.error).not.toContain('second.docx') }) + it.each([ + ['text/x-docxjs', 'report.docx'], + ['text/x-pptxgenjs', 'deck.pptx'], + ['text/x-pdflibjs', 'isolated.pdf'], + ['text/x-python-pdf', 'sandboxed.pdf'], + ['text/x-python-xlsx', 'sheet.xlsx'], + ])('resolves %s rather than streaming its source', async (type, name) => { + // Both PDF generators must be covered: the isolated-vm path stores pdf-lib JS and + // the E2B path stores Python, and either one streamed raw is the corruption bug. + mockListWorkspaceFiles.mockResolvedValue([{ ...workspaceFile('f1', name), type }]) + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: Buffer.from('PKrendered'), + contentType: 'application/octet-stream', + }) + + await zipFrom(await GET(requestFor('fileIds=f1'), context)) + + expect(mockFetchServableWorkspaceFileBuffer).toHaveBeenCalledTimes(1) + expect(mockDownloadFileStream).not.toHaveBeenCalled() + }) + it('streams an uploaded office file rather than resolving it', async () => { // A real upload serves exactly its stored bytes, so it must not take the buffered // path — otherwise a selection of large decks is held in memory for nothing. diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 7c926287750..67b070f303f 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -6,12 +6,6 @@ import { type NextRequest, NextResponse } from 'next/server' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { - DOCXJS_SOURCE_MIME, - PPTXGENJS_SOURCE_MIME, - PYTHON_PDF_SOURCE_MIME, - PYTHON_XLSX_SOURCE_MIME, -} from '@/lib/copilot/tools/server/files/doc-compile' import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -26,6 +20,7 @@ import { import { downloadFileStream } from '@/lib/uploads/core/storage-service' import { formatFileSize, + isGeneratedDocumentSourceType, isRenderableDocumentName, MAX_RENDERED_DOCUMENT_BYTES, } from '@/lib/uploads/utils/file-utils' @@ -37,14 +32,6 @@ const logger = createLogger('WorkspaceFilesDownloadAPI') const MAX_ZIP_DOWNLOAD_FILES = 100 const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 -/** Content types under which a generated document's *source* is stored. */ -const GENERATED_SOURCE_TYPES = new Set([ - DOCXJS_SOURCE_MIME, - PPTXGENJS_SOURCE_MIME, - PYTHON_PDF_SOURCE_MIME, - PYTHON_XLSX_SOURCE_MIME, -]) - /** * Whether this entry's stored bytes are a generation source that has to be resolved * before it can go in the archive. An ordinary uploaded `.docx` serves exactly what is @@ -55,7 +42,7 @@ const GENERATED_SOURCE_TYPES = new Set([ * a document name. */ function needsRendering(file: WorkspaceFileRecord): boolean { - return file.type ? GENERATED_SOURCE_TYPES.has(file.type) : isRenderableDocumentName(file.name) + return file.type ? isGeneratedDocumentSourceType(file.type) : isRenderableDocumentName(file.name) } /** diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts index cb6defd8d95..984d2b2e1de 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts @@ -3,6 +3,7 @@ import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react' import { toast } from '@sim/emcn' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { GENERATED_DOCUMENT_SOURCE_TYPES } from '@/lib/uploads/utils/file-utils' import { useUpdateWorkspaceFileContent, useWorkspaceFileContent, @@ -20,13 +21,7 @@ import { * editable text is the source program, not the compiled artifact. The serve route * returns that source only when asked for the raw representation. */ -const GENERATED_SOURCE_FILE_TYPES = new Set([ - 'text/x-pptxgenjs', - 'text/x-docxjs', - 'text/x-pdflibjs', - 'text/x-python-pdf', - 'text/x-python-xlsx', -]) +const GENERATED_SOURCE_FILE_TYPES = GENERATED_DOCUMENT_SOURCE_TYPES /** * Poll cadence for the content query while the post-stream reconcile waits for a fetch showing the diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 54f0e548117..cd36052fdbe 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -217,6 +217,25 @@ export function getFileExtension(filename: string): string { */ const RENDERABLE_DOCUMENT_EXTENSIONS = new Set(['pdf', 'docx', 'pptx', 'xlsx']) +/** + * Content types under which a generated document's *generation source* is stored. A + * file carrying one of these renders to something other than its stored bytes, so any + * surface that hands out the file itself has to resolve it first. Both PDF generators + * are here: the E2B path stores Python, the isolated-vm path stores pdf-lib JS. + */ +export const GENERATED_DOCUMENT_SOURCE_TYPES = new Set([ + 'text/x-docxjs', + 'text/x-pptxgenjs', + 'text/x-pdflibjs', + 'text/x-python-pdf', + 'text/x-python-xlsx', +]) + +/** True when the stored bytes for `contentType` are a generation source. */ +export function isGeneratedDocumentSourceType(contentType: string | undefined | null): boolean { + return contentType ? GENERATED_DOCUMENT_SOURCE_TYPES.has(contentType) : false +} + /** * Ceiling on a single rendered generated document. A generator source is text and is * orders of magnitude smaller than the document it produces, so the declared size is no From 03882bc22986bb5902e71b178a2177584a94315f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 19:28:16 -0700 Subject: [PATCH 12/18] test(files): validate the produced archive with a real zip reader Every existing test mocks the storage stream and reads the result back with the same JS library that wrote it, which cannot catch the failure this route exists to fix: an archive a real zip reader will not open. This drives 5 MB of non-repeating bytes from an fs stream through archiver, the lazy entry generator and the Node-to-web bridge, then checks the output with the operating system's unzip and compares the extracted bytes. Skips rather than fails where unzip is unavailable. --- .../files/download/route.integration.test.ts | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 apps/sim/app/api/workspaces/[id]/files/download/route.integration.test.ts diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.integration.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.integration.test.ts new file mode 100644 index 00000000000..d7a871fbcbf --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.integration.test.ts @@ -0,0 +1,171 @@ +/** + * @vitest-environment node + * + * Exercises the real archive plumbing — archiver, the lazy entry generator, and the + * Node-to-web bridge — against multi-chunk file streams, and validates the result with + * the operating system's `unzip` rather than the same JS library that produced it. The + * bug this route exists to fix was an archive a real zip reader could not open, so a + * self-consistent JS round-trip is not the assertion that matters. + */ +import { execFile } from 'node:child_process' +import { createReadStream } from 'node:fs' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { createMockRequest } from '@sim/testing' +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const run = promisify(execFile) + +const { + mockGetSession, + mockVerifyWorkspaceMembership, + mockListWorkspaceFiles, + mockListFolders, + mockDownloadFileStream, +} = vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockVerifyWorkspaceMembership: vi.fn(), + mockListWorkspaceFiles: vi.fn(), + mockListFolders: vi.fn(), + mockDownloadFileStream: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ + auth: { api: { getSession: vi.fn() } }, + getSession: mockGetSession, +})) +vi.mock('@/app/api/workflows/utils', () => ({ + verifyWorkspaceMembership: mockVerifyWorkspaceMembership, +})) +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + listWorkspaceFiles: mockListWorkspaceFiles, + listWorkspaceFileFolders: mockListFolders, + buildWorkspaceFileFolderPathMap: (folders: Array<{ id: string; name: string }>) => + new Map(folders.map((folder) => [folder.id, folder.name])), + fetchServableWorkspaceFileBuffer: vi.fn(), +})) +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFileStream: mockDownloadFileStream, +})) +vi.mock('@sim/audit', () => ({ + recordAudit: vi.fn(), + AuditAction: { FILE_DOWNLOADED: 'file.downloaded' }, + AuditResourceType: { FILE: 'file' }, +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() })) + +import { GET } from '@/app/api/workspaces/[id]/files/download/route' + +const WORKSPACE_ID = 'ws-1' +const context = { params: Promise.resolve({ id: WORKSPACE_ID }) } + +let workDir: string +let bigPath: string +let bigBytes: Buffer +/** The OS unzip is the point of this file; skip rather than fail where it is absent. */ +let hasUnzip = true + +beforeAll(async () => { + hasUnzip = await run('unzip', ['-v']).then( + () => true, + () => false + ) + workDir = await mkdtemp(join(tmpdir(), 'sim-archive-')) + bigPath = join(workDir, 'big.bin') + // Several MB of non-repeating bytes: forces many 64 KiB chunks through the generator, + // the archiver queue and the web bridge, and would expose a truncation or ordering bug. + bigBytes = Buffer.alloc(5 * 1024 * 1024) + for (let i = 0; i < bigBytes.length; i++) bigBytes[i] = (i * 31 + (i >> 8)) & 0xff + await writeFile(bigPath, bigBytes) +}) + +afterAll(async () => { + await rm(workDir, { recursive: true, force: true }) +}) + +describe('workspace files download — real archive', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) + mockVerifyWorkspaceMembership.mockResolvedValue({ role: 'member' }) + mockListFolders.mockResolvedValue([{ id: 'folder-1', name: 'Reports', parentId: null }]) + // A real fs stream, not a single pre-made buffer. + mockDownloadFileStream.mockImplementation(async () => createReadStream(bigPath)) + }) + + it('produces an archive the OS unzip accepts, with byte-exact contents', async (ctx) => { + if (!hasUnzip) ctx.skip() + mockListWorkspaceFiles.mockResolvedValue([ + { + id: 'f1', + name: 'big.bin', + key: `workspace/${WORKSPACE_ID}/f1`, + path: '/serve/f1', + size: bigBytes.length, + type: 'application/octet-stream', + folderId: 'folder-1', + }, + ]) + + const response = await GET( + createMockRequest( + 'GET', + undefined, + {}, + `http://localhost:3000/api/workspaces/${WORKSPACE_ID}/files/download?fileIds=f1` + ), + context + ) + expect(response.status).toBe(200) + + const zipPath = join(workDir, 'out.zip') + await writeFile(zipPath, Buffer.from(await response.arrayBuffer())) + + // Independent validation: the CRCs and central directory have to satisfy a real + // zip reader, which is exactly what failed for the customer. + await expect(run('unzip', ['-t', zipPath])).resolves.toBeTruthy() + + await run('unzip', ['-o', '-q', zipPath, '-d', join(workDir, 'out')]) + const extracted = await readFile(join(workDir, 'out', 'Reports', 'big.bin')) + expect(extracted.length).toBe(bigBytes.length) + expect(extracted.equals(bigBytes)).toBe(true) + }) + + it('keeps entries intact and correctly named across several streamed files', async (ctx) => { + if (!hasUnzip) ctx.skip() + mockListWorkspaceFiles.mockResolvedValue( + ['a.bin', 'b.bin', 'c.bin'].map((name, index) => ({ + id: `f${index}`, + name, + key: `workspace/${WORKSPACE_ID}/f${index}`, + path: `/serve/f${index}`, + size: bigBytes.length, + type: 'application/octet-stream', + folderId: 'folder-1', + })) + ) + + const response = await GET( + createMockRequest( + 'GET', + undefined, + {}, + `http://localhost:3000/api/workspaces/${WORKSPACE_ID}/files/download?fileIds=f0&fileIds=f1&fileIds=f2` + ), + context + ) + + const zipPath = join(workDir, 'multi.zip') + await writeFile(zipPath, Buffer.from(await response.arrayBuffer())) + await expect(run('unzip', ['-t', zipPath])).resolves.toBeTruthy() + + const { stdout } = await run('unzip', ['-l', zipPath]) + for (const name of ['Reports/a.bin', 'Reports/b.bin', 'Reports/c.bin']) { + expect(stdout).toContain(name) + } + // Each entry carries the full payload — a shared or truncated stream would not. + expect(stdout.match(/5242880/g)).toHaveLength(3) + }) +}) From 7d336b0cec63371d30ccfcd877d894a60fc2d964 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 19:29:50 -0700 Subject: [PATCH 13/18] fix(files): bound the markdown export's embedded-asset bundling The embed list is produced by scanning the document body, so its length and the bytes behind it are whatever the author put there. Every referenced asset was downloaded at once with no concurrency bound, no per-file cap and no total cap, so one request could materialize an unbounded number of unbounded files. Its sibling bulk-download route already had a count cap and a byte cap. Metadata is now resolved first, which bounds the download from declared sizes before a byte is read and moves the authorization check off the download path. Assets are then fetched with bounded concurrency and a per-file cap; a single unreadable or oversized asset drops out of the bundle rather than failing the export, which is what the previous allSettled did. --- apps/sim/app/api/files/export/[id]/route.ts | 94 ++++++++++++++++----- 1 file changed, 72 insertions(+), 22 deletions(-) diff --git a/apps/sim/app/api/files/export/[id]/route.ts b/apps/sim/app/api/files/export/[id]/route.ts index 414d781c3a0..05db7c61373 100644 --- a/apps/sim/app/api/files/export/[id]/route.ts +++ b/apps/sim/app/api/files/export/[id]/route.ts @@ -9,17 +9,28 @@ import { fileExportContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { extractEmbeddedImageIds } from '@/lib/copilot/tools/server/files/embedded-image-refs' +import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' import type { StorageContext } from '@/lib/uploads/config' import { getServeStoragePrefix } from '@/lib/uploads/config' import { downloadFile } from '@/lib/uploads/core/storage-service' import { getFileMetadataById } from '@/lib/uploads/server/metadata' +import { formatFileSize } from '@/lib/uploads/utils/file-utils' import { verifyFileAccess } from '@/app/api/files/authorization' import { encodeFilenameForHeader } from '@/app/api/files/utils' const logger = createLogger('FilesExportAPI') +/** + * Bundling caps. The embed list comes from scanning the document body, so its length + * and the bytes behind it are whatever the author put there — without these the export + * would materialize an unbounded number of unbounded assets in one request. + */ +const MAX_EXPORT_ASSETS = 100 +const MAX_EXPORT_ASSET_BYTES = 25 * 1024 * 1024 +const MAX_EXPORT_TOTAL_BYTES = 100 * 1024 * 1024 + const MARKDOWN_MIME_TYPES = new Set(['text/markdown', 'text/x-markdown']) const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']) @@ -136,34 +147,73 @@ export const GET = withRouteHandler( }) } - const fetchResults = await Promise.allSettled( - imageIds.map(async (imageId) => { - const imgRecord = await getFileMetadataById(imageId) - if (!imgRecord) return null - const imgHasAccess = await verifyFileAccess(imgRecord.key, userId) - if (!imgHasAccess) return null - const imgBuffer = await downloadFile({ - key: imgRecord.key, - context: imgRecord.context as StorageContext, - }) - return { imageId, originalName: imgRecord.originalName, buffer: imgBuffer } + if (imageIds.length > MAX_EXPORT_ASSETS) { + return NextResponse.json( + { + error: `This document embeds ${imageIds.length} files, more than the ${MAX_EXPORT_ASSETS} an export can bundle.`, + }, + { status: 400 } + ) + } + + // Metadata first: declared sizes bound the download before a byte is read, and the + // authorization check costs nothing to run here. + const assetTargets = ( + await mapWithConcurrency(imageIds, MATERIALIZE_CONCURRENCY, async (imageId) => { + try { + const imgRecord = await getFileMetadataById(imageId) + if (!imgRecord) return null + if (!(await verifyFileAccess(imgRecord.key, userId))) return null + return { imageId, record: imgRecord } + } catch (error) { + logger.warn('Failed to resolve asset for export', { + imageId, + error: toError(error).message, + }) + return null + } }) + ).filter((target): target is NonNullable => target !== null) + + const declaredAssetBytes = assetTargets.reduce((sum, target) => sum + target.record.size, 0) + if (declaredAssetBytes > MAX_EXPORT_TOTAL_BYTES) { + return NextResponse.json( + { + error: `Embedded files total ${formatFileSize(declaredAssetBytes)}, which exceeds the ${formatFileSize(MAX_EXPORT_TOTAL_BYTES)} export limit.`, + }, + { status: 400 } + ) + } + + const fetched = await mapWithConcurrency( + assetTargets, + MATERIALIZE_CONCURRENCY, + async ({ imageId, record: imgRecord }) => { + try { + const buffer = await downloadFile({ + key: imgRecord.key, + context: imgRecord.context as StorageContext, + maxBytes: MAX_EXPORT_ASSET_BYTES, + }) + return { imageId, originalName: imgRecord.originalName, buffer } + } catch (error) { + // A single unreadable or oversized asset drops out of the bundle rather than + // failing the whole export; the markdown keeps its original link. + logger.warn('Failed to fetch asset for export', { + imageId, + error: toError(error).message, + }) + return null + } + } ) const assetMap = new Map() const usedFilenames = new Set() - for (let i = 0; i < fetchResults.length; i++) { - const result = fetchResults[i] - if (result.status === 'rejected') { - logger.warn('Failed to fetch asset for export', { - imageId: imageIds[i], - error: toError(result.reason).message, - }) - continue - } - if (!result.value) continue - const { imageId, originalName, buffer } = result.value + for (const result of fetched) { + if (!result) continue + const { imageId, originalName, buffer } = result const preferred = safeFilename(originalName) const filename = deduplicatedFilename(preferred, usedFilenames, imageId) usedFilenames.add(filename) From f89b40477a7e83d299571cc130e3603e4797b672 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 19:31:37 -0700 Subject: [PATCH 14/18] fix(files): mount rendered documents into the sandbox, not generator source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mounting a generated document handed the sandbox its generator source under a .docx name, so a python-docx or openpyxl script failed on a file that looked fine and the agent debugged code that was correct. Both branches were wrong, not just the buffered one: with cloud storage the mount presigns record.key, which is the raw source object, so swapping the buffered read alone would have fixed only local storage. Source-backed documents now always take the servable path — they are bounded by the render ceiling, so routing them through the web process instead of presigning is affordable. The text/binary decision also keyed off record.type, which for these files is text/x-docxjs, so a resolved binary would have been decoded as UTF-8. It reads the resolved content type now. --- .../tools/handlers/function-execute.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index 0fbb4cebafd..c2d5034a39f 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -10,6 +10,7 @@ import { getTableById, listTables } from '@/lib/table/service' import { getOrCreateTableSnapshot, SNAPSHOT_MAX_BYTES } from '@/lib/table/snapshot-cache' import { listWorkspaceFileFolders } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager' import { + fetchServableWorkspaceFileBuffer, fetchWorkspaceFileBuffer, findWorkspaceFileRecord, getSandboxWorkspaceFilePath, @@ -21,6 +22,7 @@ import { generatePresignedDownloadUrl, hasCloudStorage, } from '@/lib/uploads/core/storage-service' +import { isGeneratedDocumentSourceType } from '@/lib/uploads/utils/file-utils' import { executeTool as executeAppTool } from '@/tools' import type { ToolExecutionContext, ToolExecutionResult } from '../../tool-executor/types' @@ -87,7 +89,14 @@ async function pushWorkspaceFileMount( mountPath: string, mounted: MountedBytes ): Promise { - if (hasCloudStorage()) { + // A generated document stores its generator source, so a presigned URL for + // `record.key` would hand the sandbox source text under a `.docx` name and the + // user's script would fail on a file that looks fine. Those resolve through the + // servable reader instead — they are bounded by the render ceiling, so routing them + // through the web process rather than presigning is affordable. + const rendersFromSource = isGeneratedDocumentSourceType(record.type) + + if (hasCloudStorage() && !rendersFromSource) { if (record.size > MOUNT_URL_MAX_BYTES) { throw new Error( `Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MOUNT_URL_MAX_BYTES / 1024 / 1024}MB per-file mount limit.` @@ -118,9 +127,13 @@ async function pushWorkspaceFileMount( `Mounting "${mountPath}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.` ) } - const buffer = await fetchWorkspaceFileBuffer(record) + const { buffer, contentType } = rendersFromSource + ? await fetchServableWorkspaceFileBuffer(record, { maxBytes: MAX_FILE_SIZE }) + : { buffer: await fetchWorkspaceFileBuffer(record), contentType: record.type } + // Keyed off the resolved type: a rendered document's source MIME is `text/x-…`, and + // decoding the binary as UTF-8 would corrupt it just as surely as shipping the source. const isText = /^text\/|application\/json|application\/xml|application\/csv/.test( - record.type || '' + contentType || '' ) sandboxFiles.push({ path: mountPath, From 2ec32cdff365721f88bb44025edf9bcc0586ec28 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 19:36:54 -0700 Subject: [PATCH 15/18] chore(deps): regenerate the lockfile against staging's dependency rework Staging reworked the OTel split, dropped dead deps and declared emcn peers, so the lockfile is regenerated on top of that rather than carrying a stale merge. The archive dependency is the only addition; lazystream stays as archiver's transitive dep now that it is no longer declared directly. Regenerated incrementally rather than from scratch: a clean resolve fails the bunfig age gate because minimumReleaseAgeExcludes lists only the darwin and linux-gnu @next/swc binaries, not the musl and windows variants that a full re-resolve also pulls. --- bun.lock | 102 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 26 deletions(-) diff --git a/bun.lock b/bun.lock index 20cc0d013c2..07229c616eb 100644 --- a/bun.lock +++ b/bun.lock @@ -207,6 +207,7 @@ "@trigger.dev/sdk": "4.4.3", "@typescript/typescript6": "^6.0.2", "ajv": "8.18.0", + "archiver": "8.0.0", "better-auth": "1.6.23", "binary-extensions": "3.1.0", "browser-image-compression": "^2.0.2", @@ -306,6 +307,7 @@ "@tailwindcss/typography": "0.5.19", "@testing-library/jest-dom": "^6.6.3", "@trigger.dev/build": "4.4.3", + "@types/archiver": "8.0.0", "@types/busboy": "1.5.4", "@types/fluent-ffmpeg": "2.1.28", "@types/html-to-text": "9.0.4", @@ -1880,6 +1882,8 @@ "@tybys/wasm-util": ["@tybys/wasm-util@0.10.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg=="], + "@types/archiver": ["@types/archiver@8.0.0", "", { "dependencies": { "@types/node": "*", "@types/readdir-glob": "*" } }, "sha512-YpXPbEuv9+eUIPPQWUPahj3cvs9isWRuF+J4z+KbdYVDO3rWorWQFxUVHnwPu2AgKwvgpki5F2VMX0Xx+mX45A=="], + "@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="], "@types/babel__generator": ["@types/babel__generator@7.27.0", "", { "dependencies": { "@babel/types": "^7.0.0" } }, "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg=="], @@ -2008,6 +2012,8 @@ "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], + "@types/readdir-glob": ["@types/readdir-glob@1.1.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg=="], + "@types/request": ["@types/request@2.48.13", "", { "dependencies": { "@types/caseless": "*", "@types/node": "*", "@types/tough-cookie": "*", "form-data": "^2.5.5" } }, "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg=="], "@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="], @@ -2162,6 +2168,8 @@ "anynum": ["anynum@1.0.0", "", {}, "sha512-xjR9/zBVnUOP6ztMIIgShjsxui80nQUQH+5xJnvrYLs+90bF25/KJqaAi8mk+B4RDtX1Nspi6fmp4YTEts8SfA=="], + "archiver": ["archiver@8.0.0", "", { "dependencies": { "async": "^3.2.4", "buffer-crc32": "^1.0.0", "is-stream": "^4.0.0", "lazystream": "^1.0.0", "normalize-path": "^3.0.0", "readable-stream": "^4.0.0", "readdir-glob": "^3.0.0", "tar-stream": "^3.0.0", "zip-stream": "^7.0.2" } }, "sha512-fV1orZfsnPn9BaSByR/qE67rJCLJEy2Ox5bq7nJh+jquWaNh6Sfec75kJ2T6PtdGUbPQlrVoSVCEOa5SdiTQ1g=="], + "arg": ["arg@5.0.2", "", {}, "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg=="], "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], @@ -2184,7 +2192,7 @@ "astring": ["astring@1.9.0", "", { "bin": { "astring": "bin/astring" } }, "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg=="], - "async": ["async@0.2.10", "", {}, "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ=="], + "async": ["async@3.2.6", "", {}, "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA=="], "async-retry": ["async-retry@1.3.3", "", { "dependencies": { "retry": "0.13.1" } }, "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw=="], @@ -2200,10 +2208,22 @@ "axios": ["axios@1.18.0", "", { "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.5", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, "sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw=="], + "b4a": ["b4a@1.8.1", "", { "peerDependencies": { "react-native-b4a": "*" }, "optionalPeers": ["react-native-b4a"] }, "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw=="], + "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + "bare-events": ["bare-events@2.9.1", "", { "peerDependencies": { "bare-abort-controller": "*" }, "optionalPeers": ["bare-abort-controller"] }, "sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg=="], + + "bare-fs": ["bare-fs@4.7.4", "", { "dependencies": { "bare-events": "^2.5.4", "bare-path": "^3.0.0", "bare-stream": "^2.6.4", "bare-url": "^2.2.2", "fast-fifo": "^1.3.2" }, "peerDependencies": { "bare-buffer": "*" }, "optionalPeers": ["bare-buffer"] }, "sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ=="], + + "bare-path": ["bare-path@3.1.1", "", {}, "sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ=="], + + "bare-stream": ["bare-stream@2.13.3", "", { "dependencies": { "b4a": "^1.8.1", "streamx": "^2.25.0", "teex": "^1.0.1" }, "peerDependencies": { "bare-abort-controller": "*", "bare-buffer": "*", "bare-events": "*" }, "optionalPeers": ["bare-abort-controller", "bare-buffer", "bare-events"] }, "sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ=="], + + "bare-url": ["bare-url@2.4.5", "", { "dependencies": { "bare-path": "^3.0.0" } }, "sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ=="], + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], "base64id": ["base64id@2.0.0", "", {}, "sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog=="], @@ -2246,6 +2266,8 @@ "buffer": ["buffer@5.6.0", "", { "dependencies": { "base64-js": "^1.0.2", "ieee754": "^1.1.4" } }, "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw=="], + "buffer-crc32": ["buffer-crc32@1.0.0", "", {}, "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w=="], + "buffer-equal-constant-time": ["buffer-equal-constant-time@1.0.1", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="], "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], @@ -2334,6 +2356,8 @@ "compare-versions": ["compare-versions@6.1.1", "", {}, "sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg=="], + "compress-commons": ["compress-commons@7.0.1", "", { "dependencies": { "crc-32": "^1.2.0", "crc32-stream": "^7.0.1", "is-stream": "^4.0.0", "normalize-path": "^3.0.0", "readable-stream": "^4.0.0" } }, "sha512-g0S8KAD8qf4+V//pr3BfB1aBnARLXNz2Gx+jmHU0LEriUuoQUOPOulVquHKTJ8+EAIIO7fhseNDr9wK5Q9FKBQ=="], + "compute-scroll-into-view": ["compute-scroll-into-view@3.1.1", "", {}, "sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw=="], "concat-stream": ["concat-stream@2.0.0", "", { "dependencies": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.0.2", "typedarray": "^0.0.6" } }, "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A=="], @@ -2362,6 +2386,10 @@ "cpu-features": ["cpu-features@0.0.10", "", { "dependencies": { "buildcheck": "~0.0.6", "nan": "^2.19.0" } }, "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA=="], + "crc-32": ["crc-32@1.2.2", "", { "bin": { "crc32": "bin/crc32.njs" } }, "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ=="], + + "crc32-stream": ["crc32-stream@7.0.1", "", { "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^4.0.0" } }, "sha512-IBWsY8xznyQrcHn8h4bC8/4ErNke5elzgG8GcqF4RFPw6aHkWWRc7Tgw6upjaTX/CT/yQgqYENkxYsTYN+hW2g=="], + "croner": ["croner@9.1.0", "", {}, "sha512-p9nwwR4qyT5W996vBZhdvBCnMhicY5ytZkR4D1Xj0wuTDEiMnjwR57Q3RXYY/s0EpX6Ay3vgIcfaR+ewGHsi+g=="], "cronstrue": ["cronstrue@3.3.0", "", { "bin": { "cronstrue": "bin/cli.js" } }, "sha512-iwJytzJph1hosXC09zY8F5ACDJKerr0h3/2mOxg9+5uuFObYlgK0m35uUPk4GCvhHc2abK7NfnR9oMqY0qZFAg=="], @@ -2642,6 +2670,8 @@ "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], + "events-universal": ["events-universal@1.0.1", "", { "dependencies": { "bare-events": "^2.7.0" } }, "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw=="], + "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], "eventsource-parser": ["eventsource-parser@3.1.0", "", {}, "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg=="], @@ -2676,6 +2706,8 @@ "fast-equals": ["fast-equals@5.4.0", "", {}, "sha512-jt2DW/aNFNwke7AUd+Z+e6pz39KO5rzdbbFCg2sGafS4mk13MI7Z8O5z9cADNn5lhGODIgLwug6TZO2ctf7kcw=="], + "fast-fifo": ["fast-fifo@1.3.2", "", {}, "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="], + "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], "fast-safe-stringify": ["fast-safe-stringify@2.1.1", "", {}, "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="], @@ -2950,7 +2982,7 @@ "is-property": ["is-property@1.0.2", "", {}, "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g=="], - "is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], + "is-stream": ["is-stream@4.0.1", "", {}, "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A=="], "is-unicode-supported": ["is-unicode-supported@2.1.0", "", {}, "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ=="], @@ -3028,6 +3060,8 @@ "layout-base": ["layout-base@1.0.2", "", {}, "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg=="], + "lazystream": ["lazystream@1.0.1", "", { "dependencies": { "readable-stream": "^2.0.5" } }, "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw=="], + "leac": ["leac@0.6.0", "", {}, "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg=="], "libbase64": ["libbase64@1.3.0", "", {}, "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg=="], @@ -3624,10 +3658,12 @@ "read-cache": ["read-cache@1.0.0", "", { "dependencies": { "pify": "^2.3.0" } }, "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA=="], - "readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + "readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="], "readable-web-to-node-stream": ["readable-web-to-node-stream@3.0.4", "", { "dependencies": { "readable-stream": "^4.7.0" } }, "sha512-9nX56alTf5bwXQ3ZDipHJhusu9NTQJ/CVPtb/XHAJCXihZeitfJvIRS4GqQ/mfIoOE3IelHMrpayVrosdHBuLw=="], + "readdir-glob": ["readdir-glob@3.0.0", "", { "dependencies": { "minimatch": "^10.2.2" } }, "sha512-AhNB2KgKeVJr16nK9LLZbJNWnYoT23ZrumNKFDebHBdkC8KHSqWo871JAUhoWC/RtjEVdqNMFpM6qrwRbaUqpw=="], + "readdirp": ["readdirp@5.0.0", "", {}, "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ=="], "real-require": ["real-require@0.2.0", "", {}, "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg=="], @@ -3868,6 +3904,8 @@ "streamsearch": ["streamsearch@1.1.0", "", {}, "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg=="], + "streamx": ["streamx@2.28.0", "", { "dependencies": { "events-universal": "^1.0.0", "fast-fifo": "^1.3.2", "text-decoder": "^1.1.0" } }, "sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw=="], + "string-argv": ["string-argv@0.3.2", "", {}, "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q=="], "string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], @@ -3876,7 +3914,7 @@ "string.prototype.codepointat": ["string.prototype.codepointat@0.2.1", "", {}, "sha512-2cBVCj6I4IOvEnjgO/hWqXjqBGsY+zwPmHl12Srk9IXSZ56Jwwmy+66XO5Iut/oQVR7t5ihYdLB0GMa4alEUcg=="], - "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], "stringify-entities": ["stringify-entities@4.0.4", "", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], @@ -3936,12 +3974,16 @@ "tar-fs": ["tar-fs@2.1.4", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ=="], - "tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="], + "tar-stream": ["tar-stream@3.2.0", "", { "dependencies": { "b4a": "^1.6.4", "bare-fs": "^4.5.5", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg=="], "tdigest": ["tdigest@0.1.2", "", { "dependencies": { "bintrees": "1.0.2" } }, "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA=="], "teeny-request": ["teeny-request@9.0.0", "", { "dependencies": { "http-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0", "node-fetch": "^2.6.9", "stream-events": "^1.0.5", "uuid": "^9.0.0" } }, "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g=="], + "teex": ["teex@1.0.1", "", { "dependencies": { "streamx": "^2.12.5" } }, "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg=="], + + "text-decoder": ["text-decoder@1.2.7", "", { "dependencies": { "b4a": "^1.6.4" } }, "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ=="], + "thenify": ["thenify@3.3.1", "", { "dependencies": { "any-promise": "^1.0.0" } }, "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw=="], "thenify-all": ["thenify-all@1.6.0", "", { "dependencies": { "thenify": ">= 3.1.0 < 4" } }, "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA=="], @@ -4164,6 +4206,8 @@ "yoga-wasm-web": ["yoga-wasm-web@0.3.3", "", {}, "sha512-N+d4UJSJbt/R3wqY7Coqs5pcV0aUj2j9IaQ3rNj9bVCLld8tTGKRa2USARjnvZJWVx1NDmQev8EknoczaOQDOA=="], + "zip-stream": ["zip-stream@7.0.5", "", { "dependencies": { "compress-commons": "^7.0.0", "normalize-path": "^3.0.0", "readable-stream": "^4.0.0" } }, "sha512-dSvYKdvLsAHCDqPOhIwk/q5CvuWtTB3Dgpoe0uVEFjTzIOAmsQpprX25InCvrvJsirEbu1OHyy67n/kAj1Sw/w=="], + "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "zod-error": ["zod-error@1.5.0", "", { "dependencies": { "zod": "^3.20.2" } }, "sha512-zzopKZ/skI9iXpqCEPj+iLCKl9b88E43ehcU+sbRoHuwGd9F1IDVGQ70TyO6kmfiRL1g4IXkjsXK+g1gLYl4WQ=="], @@ -4528,12 +4572,16 @@ "@trigger.dev/sdk/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], + "@types/archiver/@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], + "@types/cors/@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], "@types/node-fetch/@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], "@types/nodemailer/@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], + "@types/readdir-glob/@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], + "@types/request/@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], "@types/request/form-data": ["form-data@2.5.6", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.4", "mime-types": "^2.1.35", "safe-buffer": "^5.2.1" } }, "sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA=="], @@ -4634,6 +4682,8 @@ "fetch-cookie/tough-cookie": ["tough-cookie@6.0.1", "", { "dependencies": { "tldts": "^7.0.5" } }, "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw=="], + "fluent-ffmpeg/async": ["async@0.2.10", "", {}, "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ=="], + "foreground-child/signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], @@ -4660,6 +4710,8 @@ "fumadocs-ui/tailwind-merge": ["tailwind-merge@3.6.0", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="], + "gaxios/is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], + "gaxios/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], "gcp-metadata/gaxios": ["gaxios@7.1.3", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "rimraf": "^5.0.1" } }, "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ=="], @@ -4688,8 +4740,12 @@ "jsonwebtoken/semver": ["semver@7.8.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA=="], + "jszip/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + "katex/commander": ["commander@8.3.0", "", {}, "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww=="], + "lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + "libmime/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], "linebreak/base64-js": ["base64-js@0.0.8", "", {}, "sha512-3XSA2cR/h/73EzlXXdU6YNycmYI7+kicTxks4eJg2g39biHR84slg2+des+p7iHYhbRg/udIS4TD53WabcOUkw=="], @@ -4718,8 +4774,6 @@ "neo4j-driver-bolt-connection/buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], - "neo4j-driver-bolt-connection/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "next/postcss": ["postcss@8.4.31", "", { "dependencies": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } }, "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ=="], "next/sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], @@ -4780,9 +4834,7 @@ "react-promise-suspense/fast-deep-equal": ["fast-deep-equal@2.0.1", "", {}, "sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w=="], - "readable-stream/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], - - "readable-web-to-node-stream/readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="], + "readable-stream/buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], "resend/@react-email/render": ["@react-email/render@1.1.2", "", { "dependencies": { "html-to-text": "^9.0.5", "prettier": "^3.5.3", "react-promise-suspense": "^0.3.4" }, "peerDependencies": { "react": "^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^18.0 || ^19.0 || ^19.0.0-rc" } }, "sha512-RnRehYN3v9gVlNMehHPHhyp2RQo7+pSkHDtXPvg3s0GbzM9SQMW4Qrf8GRNvtpLC4gsI+Wt0VatNRUFqjvevbw=="], @@ -4812,8 +4864,6 @@ "string-width-cjs/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], - "string_decoder/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], - "strip-ansi-cjs/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "sucrase/commander": ["commander@4.1.1", "", {}, "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA=="], @@ -4822,7 +4872,7 @@ "tar-fs/chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], - "tar-stream/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + "tar-fs/tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="], "teeny-request/http-proxy-agent": ["http-proxy-agent@5.0.0", "", { "dependencies": { "@tootallnate/once": "2", "agent-base": "6", "debug": "4" } }, "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w=="], @@ -5070,12 +5120,16 @@ "@trigger.dev/core/socket.io-client/engine.io-client": ["engine.io-client@6.5.4", "", { "dependencies": { "@socket.io/component-emitter": "~3.1.0", "debug": "~4.3.1", "engine.io-parser": "~5.2.1", "ws": "~8.17.1", "xmlhttprequest-ssl": "~2.0.0" } }, "sha512-GeZeeRjpD2qf49cZQ0Wvh/8NJNfeXkXXcoGh+F77oEAgo9gUHwT1fCRxSNU+YEEaysOJTnsFHmM5oAcPy4ntvQ=="], + "@types/archiver/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + "@types/cors/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], "@types/node-fetch/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], "@types/nodemailer/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + "@types/readdir-glob/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + "@types/request/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], "@types/request/form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], @@ -5088,8 +5142,6 @@ "axios/https-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], - "bl/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "c12/chokidar/readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], "chrome-launcher/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], @@ -5104,8 +5156,6 @@ "cmdk/@radix-ui/react-primitive/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.4", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Jl+bCv8HxKnlTLVrcDE8zTMJ09R9/ukw4qBs/oZClOfoQk/cOTbDn+NceXfV7j09YPVQUryJPHurafcSg6EVKA=="], - "concat-stream/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "cytoscape-fcose/cose-base/layout-base": ["layout-base@2.0.1", "", {}, "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg=="], "d3-sankey/d3-array/internmap": ["internmap@1.0.1", "", {}, "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw=="], @@ -5114,8 +5164,6 @@ "docx/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], - "duplexify/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "engine.io/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], "express/accepts/negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], @@ -5186,6 +5234,14 @@ "html-to-text/htmlparser2/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "jszip/readable-stream/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], + + "jszip/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + + "lazystream/readable-stream/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], + + "lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "log-update/slice-ansi/is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], "next/sharp/@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], @@ -5266,10 +5322,6 @@ "react-email/chokidar/readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], - "readable-web-to-node-stream/readable-stream/buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], - - "readable-web-to-node-stream/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "rimraf/glob/jackspeak": ["jackspeak@3.4.3", "", { "dependencies": { "@isaacs/cliui": "^8.0.2" }, "optionalDependencies": { "@pkgjs/parseargs": "^0.11.0" } }, "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw=="], "rimraf/glob/path-scurry": ["path-scurry@1.11.1", "", { "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" } }, "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA=="], @@ -5280,11 +5332,9 @@ "sim/tailwindcss/postcss-selector-parser": ["postcss-selector-parser@6.1.4", "", { "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" } }, "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ=="], - "stream-browserify/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "string-width-cjs/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - "tar-stream/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + "tar-fs/tar-stream/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], "teeny-request/http-proxy-agent/agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], From 30ccdbf3ca9666e977ab9e6eadd7a4f9f998564f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 19:44:08 -0700 Subject: [PATCH 16/18] fix(files): budget sandbox mounts on rendered bytes, not declared source size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source-backed document declares the size of its generator, not of the document, so the per-file and aggregate mount pre-checks were reading a number unrelated to what was about to be mounted — tiny sources cleared the guard and only afterwards was the running total updated with the real length. Those pre-checks now apply only to files that serve what they declare. A document is capped by the read instead, at the smaller of the per-file limit and the budget left, and a size rejection is reported in the same terms as the other mount limits. Same invariant the archive route already had to learn: declared size is not a bound once bytes can be rendered. --- .../tools/handlers/function-execute.ts | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index c2d5034a39f..828e27b5956 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -3,6 +3,7 @@ import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/copilot/vfs/ import { resolveWorkflowAliasForWorkspace } from '@/lib/copilot/vfs/workflow-alias-resolver' import { isPlanAliasPath, workflowAliasSandboxPath } from '@/lib/copilot/vfs/workflow-aliases' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { getColumnId } from '@/lib/table/column-keys' import { formatCsvCell, neutralizeCsvFormula, toCsvRow } from '@/lib/table/export-format' import { queryRows } from '@/lib/table/rows/service' @@ -117,18 +118,33 @@ async function pushWorkspaceFileMount( return } - if (record.size > MAX_FILE_SIZE) { - throw new Error( - `Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` - ) - } - if (mounted.buffered + record.size > MAX_TOTAL_SIZE) { - throw new Error( - `Mounting "${mountPath}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.` - ) + const remainingBudget = Math.max(0, MAX_TOTAL_SIZE - mounted.buffered) + + // A source-backed document declares the size of its generator, not of the document, + // so these pre-checks say nothing about what is about to be mounted. Its read is + // capped instead, and the real length is checked once it is known. + if (!rendersFromSource) { + if (record.size > MAX_FILE_SIZE) { + throw new Error( + `Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` + ) + } + if (record.size > remainingBudget) { + throw new Error( + `Mounting "${mountPath}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.` + ) + } } + const { buffer, contentType } = rendersFromSource - ? await fetchServableWorkspaceFileBuffer(record, { maxBytes: MAX_FILE_SIZE }) + ? await fetchServableWorkspaceFileBuffer(record, { + maxBytes: Math.min(MAX_FILE_SIZE, remainingBudget), + }).catch((error) => { + if (!isPayloadSizeLimitError(error)) throw error + throw new Error( + `Input file "${mountPath}" renders to more than the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit, or than the mount budget left. Mount fewer or smaller files.` + ) + }) : { buffer: await fetchWorkspaceFileBuffer(record), contentType: record.type } // Keyed off the resolved type: a rendered document's source MIME is `text/x-…`, and // decoding the binary as UTF-8 would corrupt it just as surely as shipping the source. From 9db0867567b09c7e39348bd9cc3a6fbc884c5826 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 19:55:09 -0700 Subject: [PATCH 17/18] test(files): cover the two surfaces added without tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The markdown export route had no test file at all, and the sandbox mount's source-backed branch was exercised by nothing — both were changed in this PR and one of them shipped a defect a reviewer caught within minutes. Export: the embed-count rejection, the declared-bytes rejection landing before any asset leaves storage, the per-asset download cap, an unreadable asset dropping out rather than failing the export, and an unauthorized asset never being read. Mount: a generated document never presigning its raw key even on cloud storage, its rendered bytes mounting as base64 rather than being utf-8 decoded off the source MIME, and the budget rejecting on rendered length. --- .../app/api/files/export/[id]/route.test.ts | 158 ++++++++++++++++++ .../tools/handlers/function-execute.test.ts | 59 +++++++ 2 files changed, 217 insertions(+) create mode 100644 apps/sim/app/api/files/export/[id]/route.test.ts diff --git a/apps/sim/app/api/files/export/[id]/route.test.ts b/apps/sim/app/api/files/export/[id]/route.test.ts new file mode 100644 index 00000000000..77e789c3cbd --- /dev/null +++ b/apps/sim/app/api/files/export/[id]/route.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ +import { createMockRequest } from '@sim/testing' +import JSZip from 'jszip' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCheckAuth, + mockGetFileMetadataById, + mockVerifyFileAccess, + mockDownloadFile, + mockExtractEmbeddedImageIds, +} = vi.hoisted(() => ({ + mockCheckAuth: vi.fn(), + mockGetFileMetadataById: vi.fn(), + mockVerifyFileAccess: vi.fn(), + mockDownloadFile: vi.fn(), + mockExtractEmbeddedImageIds: vi.fn(), +})) + +vi.mock('@/lib/auth/hybrid', () => ({ checkSessionOrInternalAuth: mockCheckAuth })) +vi.mock('@/lib/uploads/server/metadata', () => ({ + getFileMetadataById: mockGetFileMetadataById, +})) +vi.mock('@/app/api/files/authorization', () => ({ verifyFileAccess: mockVerifyFileAccess })) +vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile })) +vi.mock('@/lib/copilot/tools/server/files/embedded-image-refs', () => ({ + extractEmbeddedImageIds: mockExtractEmbeddedImageIds, +})) +vi.mock('@sim/audit', () => ({ + recordAudit: vi.fn(), + AuditAction: { FILE_DOWNLOADED: 'file.downloaded' }, + AuditResourceType: { FILE: 'file' }, +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() })) + +import { GET } from '@/app/api/files/export/[id]/route' + +const MB = 1024 * 1024 +const DOC_ID = 'doc-1' +const context = { params: Promise.resolve({ id: DOC_ID }) } + +function request() { + return createMockRequest('GET', undefined, {}, `http://localhost:3000/api/files/export/${DOC_ID}`) +} + +function assetRecord(id: string, size: number) { + return { + id, + key: `workspace/ws-1/${id}`, + originalName: `${id}.png`, + contentType: 'image/png', + context: 'workspace', + size, + workspaceId: 'ws-1', + } +} + +describe('markdown export bundling', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAuth.mockResolvedValue({ success: true, userId: 'user-1' }) + mockVerifyFileAccess.mockResolvedValue(true) + mockGetFileMetadataById.mockImplementation(async (id: string) => + id === DOC_ID + ? { + id: DOC_ID, + key: 'workspace/ws-1/doc.md', + originalName: 'doc.md', + contentType: 'text/markdown', + context: 'workspace', + size: 1024, + workspaceId: 'ws-1', + } + : assetRecord(id, 1 * MB) + ) + mockDownloadFile.mockResolvedValue(Buffer.from('# Doc\n')) + mockExtractEmbeddedImageIds.mockReturnValue([]) + }) + + it('rejects a document embedding more assets than an export may bundle', async () => { + mockExtractEmbeddedImageIds.mockReturnValue( + Array.from({ length: 101 }, (_, index) => `img-${index}`) + ) + + const response = await GET(request(), context) + + expect(response.status).toBe(400) + expect((await response.json()).error).toContain('101') + // Rejected on the embed count alone: nothing was resolved or downloaded. + expect(mockGetFileMetadataById).toHaveBeenCalledTimes(1) + }) + + it('rejects on declared asset bytes before downloading any of them', async () => { + mockExtractEmbeddedImageIds.mockReturnValue(['a', 'b', 'c']) + mockGetFileMetadataById.mockImplementation(async (id: string) => + id === DOC_ID + ? { + id: DOC_ID, + key: 'workspace/ws-1/doc.md', + originalName: 'doc.md', + contentType: 'text/markdown', + context: 'workspace', + size: 1024, + workspaceId: 'ws-1', + } + : assetRecord(id, 40 * MB) + ) + + const response = await GET(request(), context) + + expect(response.status).toBe(400) + expect((await response.json()).error).toContain('exceeds') + // Only the markdown body was read; the 120 MB of assets never left storage. + expect(mockDownloadFile).toHaveBeenCalledTimes(1) + }) + + it('caps each asset download rather than trusting its declared size', async () => { + mockExtractEmbeddedImageIds.mockReturnValue(['a']) + + await GET(request(), context) + + const assetCall = mockDownloadFile.mock.calls.find( + ([options]) => options.key === 'workspace/ws-1/a' + ) + expect(assetCall?.[0].maxBytes).toBe(25 * MB) + }) + + it('drops an unreadable asset instead of failing the whole export', async () => { + mockExtractEmbeddedImageIds.mockReturnValue(['good', 'bad']) + mockDownloadFile.mockImplementation(async ({ key }: { key: string }) => { + if (key.endsWith('doc.md')) return Buffer.from('# Doc\n![x](/api/files/view/good)\n') + if (key.endsWith('bad')) throw new Error('storage down') + return Buffer.from('png-bytes') + }) + + const response = await GET(request(), context) + + expect(response.status).toBe(200) + const zip = await JSZip.loadAsync(Buffer.from(await response.arrayBuffer())) + expect(zip.file('assets/good.png')).not.toBeNull() + expect(zip.file('assets/bad.png')).toBeNull() + }) + + it('skips an asset the caller cannot read', async () => { + mockExtractEmbeddedImageIds.mockReturnValue(['secret']) + mockVerifyFileAccess.mockImplementation(async (key: string) => !key.endsWith('secret')) + + const response = await GET(request(), context) + + expect(response.status).toBe(200) + // Authorization is settled during metadata resolution, before any asset read. + expect(mockDownloadFile.mock.calls.some(([options]) => options.key.endsWith('secret'))).toBe( + false + ) + }) +}) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts index f6494b14aa0..51dabd8de14 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' const { mockIsFeatureEnabled, @@ -16,6 +17,7 @@ const { mockListWorkspaceFiles, mockFindWorkspaceFileRecord, mockFetchWorkspaceFileBuffer, + mockFetchServableWorkspaceFileBuffer, mockGetSandboxWorkspaceFilePath, mockListWorkspaceFileFolders, } = vi.hoisted(() => ({ @@ -31,6 +33,7 @@ const { mockListWorkspaceFiles: vi.fn(), mockFindWorkspaceFileRecord: vi.fn(), mockFetchWorkspaceFileBuffer: vi.fn(), + mockFetchServableWorkspaceFileBuffer: vi.fn(), mockGetSandboxWorkspaceFilePath: vi.fn(), mockListWorkspaceFileFolders: vi.fn(), })) @@ -52,6 +55,7 @@ vi.mock('@/lib/uploads/core/storage-service', () => ({ })) vi.mock('@/tools', () => ({ executeTool: mockExecuteTool })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchServableWorkspaceFileBuffer: mockFetchServableWorkspaceFileBuffer, fetchWorkspaceFileBuffer: mockFetchWorkspaceFileBuffer, findWorkspaceFileRecord: mockFindWorkspaceFileRecord, getSandboxWorkspaceFilePath: mockGetSandboxWorkspaceFilePath, @@ -309,6 +313,61 @@ describe('executeFunctionExecute file mounts', () => { expect(file.type).toBeUndefined() }) + describe('generated documents', () => { + const docRecord = { + ...fileRecord, + name: 'report.docx', + key: 'workspace/ws_1/report.docx', + // The stored bytes are the generator source, so the record declares its size. + type: 'text/x-docxjs', + size: 6_242, + } + + beforeEach(() => { + mockFindWorkspaceFileRecord.mockReturnValue(docRecord) + mockListWorkspaceFiles.mockResolvedValue([docRecord]) + mockGetSandboxWorkspaceFilePath.mockReturnValue('/home/user/files/report.docx') + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: Buffer.from('PK\u0003\u0004rendered-docx'), + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }) + }) + + it('never presigns the raw key, even on cloud storage', async () => { + mockHasCloudStorage.mockReturnValue(true) + + await executeFunctionExecute({ inputFiles: ['files/report.docx'] }, context as never) + + // Presigning record.key would hand the sandbox the generator source. + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + expect(mockFetchWorkspaceFileBuffer).not.toHaveBeenCalled() + expect(mockFetchServableWorkspaceFileBuffer).toHaveBeenCalledTimes(1) + }) + + it('mounts the rendered bytes as base64, not utf-8', async () => { + mockHasCloudStorage.mockReturnValue(true) + + await executeFunctionExecute({ inputFiles: ['files/report.docx'] }, context as never) + + const file = mountedFiles()[0] + // record.type is text/x-docxjs; keying off it would utf-8 decode a binary. + expect(file.encoding).toBe('base64') + expect(Buffer.from(file.content as string, 'base64').toString()).toContain('rendered-docx') + }) + + it('budgets the mount on rendered length, not the declared source size', async () => { + mockHasCloudStorage.mockReturnValue(true) + // A tiny source that renders past the aggregate mount budget. + mockFetchServableWorkspaceFileBuffer.mockRejectedValue( + new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) + ) + + await expect( + executeFunctionExecute({ inputFiles: ['files/report.docx'] }, context as never) + ).rejects.toThrow(/mount limit/) + }) + }) + it('cloud storage: throws when a file exceeds the per-file URL mount limit', async () => { mockFindWorkspaceFileRecord.mockReturnValue({ ...fileRecord, size: 600 * 1024 * 1024 }) From dcc19cc9f1f4c1ec11e3bea789a513e5b8cbf1e8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 20:01:48 -0700 Subject: [PATCH 18/18] fix(files): stop the export caps from rejecting documents that used to work The caps I picked would have failed exports that previously succeeded: a screenshot-heavy document can hold more than 100 embeds, and 100 embeds of unoptimised PNGs can exceed 100 MB. Adding a limit to bound memory is right; setting it below real usage turns a memory risk into a broken feature. The byte ceiling now matches the bulk-download route, so the two export surfaces reject at the same size rather than at two invented ones. The count is only a guard on the metadata lookups that run before the byte check, so it moves well above any hand-authored document instead of sitting where a legitimate one could reach it. --- apps/sim/app/api/files/export/[id]/route.test.ts | 8 ++++---- apps/sim/app/api/files/export/[id]/route.ts | 9 +++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/sim/app/api/files/export/[id]/route.test.ts b/apps/sim/app/api/files/export/[id]/route.test.ts index 77e789c3cbd..1056fd61e05 100644 --- a/apps/sim/app/api/files/export/[id]/route.test.ts +++ b/apps/sim/app/api/files/export/[id]/route.test.ts @@ -81,13 +81,13 @@ describe('markdown export bundling', () => { it('rejects a document embedding more assets than an export may bundle', async () => { mockExtractEmbeddedImageIds.mockReturnValue( - Array.from({ length: 101 }, (_, index) => `img-${index}`) + Array.from({ length: 501 }, (_, index) => `img-${index}`) ) const response = await GET(request(), context) expect(response.status).toBe(400) - expect((await response.json()).error).toContain('101') + expect((await response.json()).error).toContain('501') // Rejected on the embed count alone: nothing was resolved or downloaded. expect(mockGetFileMetadataById).toHaveBeenCalledTimes(1) }) @@ -105,14 +105,14 @@ describe('markdown export bundling', () => { size: 1024, workspaceId: 'ws-1', } - : assetRecord(id, 40 * MB) + : assetRecord(id, 100 * MB) ) const response = await GET(request(), context) expect(response.status).toBe(400) expect((await response.json()).error).toContain('exceeds') - // Only the markdown body was read; the 120 MB of assets never left storage. + // Only the markdown body was read; the 300 MB of assets never left storage. expect(mockDownloadFile).toHaveBeenCalledTimes(1) }) diff --git a/apps/sim/app/api/files/export/[id]/route.ts b/apps/sim/app/api/files/export/[id]/route.ts index 05db7c61373..6b217f9c6c1 100644 --- a/apps/sim/app/api/files/export/[id]/route.ts +++ b/apps/sim/app/api/files/export/[id]/route.ts @@ -26,10 +26,15 @@ const logger = createLogger('FilesExportAPI') * Bundling caps. The embed list comes from scanning the document body, so its length * and the bytes behind it are whatever the author put there — without these the export * would materialize an unbounded number of unbounded assets in one request. + * + * The byte ceilings are the real bound and match the bulk-download route, so the two + * export surfaces reject at the same size. The count is only a guard on the metadata + * lookups that precede the byte check, so it sits far above any hand-authored document + * rather than at a number a screenshot-heavy doc could plausibly reach. */ -const MAX_EXPORT_ASSETS = 100 +const MAX_EXPORT_ASSETS = 500 const MAX_EXPORT_ASSET_BYTES = 25 * 1024 * 1024 -const MAX_EXPORT_TOTAL_BYTES = 100 * 1024 * 1024 +const MAX_EXPORT_TOTAL_BYTES = 250 * 1024 * 1024 const MARKDOWN_MIME_TYPES = new Set(['text/markdown', 'text/x-markdown']) const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown'])