fix(files): serve rendered documents and stream workspace archives

apps/sim/app/api/files/export/[id]/route.test.ts

@@ -0,0 +1,158 @@
+/**
+ * @vitest-environment node
+ */
+import { createMockRequest } from '@sim/testing'
+import JSZip from 'jszip'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockCheckAuth,
+  mockGetFileMetadataById,
+  mockVerifyFileAccess,
+  mockDownloadFile,
+  mockExtractEmbeddedImageIds,
+} = vi.hoisted(() => ({
+  mockCheckAuth: vi.fn(),
+  mockGetFileMetadataById: vi.fn(),
+  mockVerifyFileAccess: vi.fn(),
+  mockDownloadFile: vi.fn(),
+  mockExtractEmbeddedImageIds: vi.fn(),
+}))
+
+vi.mock('@/lib/auth/hybrid', () => ({ checkSessionOrInternalAuth: mockCheckAuth }))
+vi.mock('@/lib/uploads/server/metadata', () => ({
+  getFileMetadataById: mockGetFileMetadataById,
+}))
+vi.mock('@/app/api/files/authorization', () => ({ verifyFileAccess: mockVerifyFileAccess }))
+vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile }))
+vi.mock('@/lib/copilot/tools/server/files/embedded-image-refs', () => ({
+  extractEmbeddedImageIds: mockExtractEmbeddedImageIds,
+}))
+vi.mock('@sim/audit', () => ({
+  recordAudit: vi.fn(),
+  AuditAction: { FILE_DOWNLOADED: 'file.downloaded' },
+  AuditResourceType: { FILE: 'file' },
+}))
+vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() }))
+
+import { GET } from '@/app/api/files/export/[id]/route'
+
+const MB = 1024 * 1024
+const DOC_ID = 'doc-1'
+const context = { params: Promise.resolve({ id: DOC_ID }) }
+
+function request() {
+  return createMockRequest('GET', undefined, {}, `http://localhost:3000/api/files/export/${DOC_ID}`)
+}
+
+function assetRecord(id: string, size: number) {
+  return {
+    id,
+    key: `workspace/ws-1/${id}`,
+    originalName: `${id}.png`,
+    contentType: 'image/png',
+    context: 'workspace',
+    size,
+    workspaceId: 'ws-1',
+  }
+}
+
+describe('markdown export bundling', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockCheckAuth.mockResolvedValue({ success: true, userId: 'user-1' })
+    mockVerifyFileAccess.mockResolvedValue(true)
+    mockGetFileMetadataById.mockImplementation(async (id: string) =>
+      id === DOC_ID
+        ? {
+            id: DOC_ID,
+            key: 'workspace/ws-1/doc.md',
+            originalName: 'doc.md',
+            contentType: 'text/markdown',
+            context: 'workspace',
+            size: 1024,
+            workspaceId: 'ws-1',
+          }
+        : assetRecord(id, 1 * MB)
+    )
+    mockDownloadFile.mockResolvedValue(Buffer.from('# Doc\n'))
+    mockExtractEmbeddedImageIds.mockReturnValue([])
+  })
+
+  it('rejects a document embedding more assets than an export may bundle', async () => {
+    mockExtractEmbeddedImageIds.mockReturnValue(
+      Array.from({ length: 501 }, (_, index) => `img-${index}`)
+    )
+
+    const response = await GET(request(), context)
+
+    expect(response.status).toBe(400)
+    expect((await response.json()).error).toContain('501')
+    // Rejected on the embed count alone: nothing was resolved or downloaded.
+    expect(mockGetFileMetadataById).toHaveBeenCalledTimes(1)
+  })
+
+  it('rejects on declared asset bytes before downloading any of them', async () => {
+    mockExtractEmbeddedImageIds.mockReturnValue(['a', 'b', 'c'])
+    mockGetFileMetadataById.mockImplementation(async (id: string) =>
+      id === DOC_ID
+        ? {
+            id: DOC_ID,
+            key: 'workspace/ws-1/doc.md',
+            originalName: 'doc.md',
+            contentType: 'text/markdown',
+            context: 'workspace',
+            size: 1024,
+            workspaceId: 'ws-1',
+          }
+        : assetRecord(id, 100 * MB)
+    )
+
+    const response = await GET(request(), context)
+
+    expect(response.status).toBe(400)
+    expect((await response.json()).error).toContain('exceeds')
+    // Only the markdown body was read; the 300 MB of assets never left storage.
+    expect(mockDownloadFile).toHaveBeenCalledTimes(1)
+  })
+
+  it('caps each asset download rather than trusting its declared size', async () => {
+    mockExtractEmbeddedImageIds.mockReturnValue(['a'])
+
+    await GET(request(), context)
+
+    const assetCall = mockDownloadFile.mock.calls.find(
+      ([options]) => options.key === 'workspace/ws-1/a'
+    )
+    expect(assetCall?.[0].maxBytes).toBe(25 * MB)
+  })
+
+  it('drops an unreadable asset instead of failing the whole export', async () => {
+    mockExtractEmbeddedImageIds.mockReturnValue(['good', 'bad'])
+    mockDownloadFile.mockImplementation(async ({ key }: { key: string }) => {
+      if (key.endsWith('doc.md')) return Buffer.from('# Doc\n![x](/api/files/view/good)\n')
+      if (key.endsWith('bad')) throw new Error('storage down')
+      return Buffer.from('png-bytes')
+    })
+
+    const response = await GET(request(), context)
+
+    expect(response.status).toBe(200)
+    const zip = await JSZip.loadAsync(Buffer.from(await response.arrayBuffer()))
+    expect(zip.file('assets/good.png')).not.toBeNull()
+    expect(zip.file('assets/bad.png')).toBeNull()
+  })
+
+  it('skips an asset the caller cannot read', async () => {
+    mockExtractEmbeddedImageIds.mockReturnValue(['secret'])
+    mockVerifyFileAccess.mockImplementation(async (key: string) => !key.endsWith('secret'))
+
+    const response = await GET(request(), context)
+
+    expect(response.status).toBe(200)
+    // Authorization is settled during metadata resolution, before any asset read.
+    expect(mockDownloadFile.mock.calls.some(([options]) => options.key.endsWith('secret'))).toBe(
+      false
+    )
+  })
+})

apps/sim/app/api/files/export/[id]/route.ts

@@ -9,17 +9,33 @@ import { fileExportContract } from '@/lib/api/contracts/storage-transfer'
 import { parseRequest } from '@/lib/api/server'
 import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
 import { extractEmbeddedImageIds } from '@/lib/copilot/tools/server/files/embedded-image-refs'
+import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { captureServerEvent } from '@/lib/posthog/server'
 import type { StorageContext } from '@/lib/uploads/config'
 import { getServeStoragePrefix } from '@/lib/uploads/config'
 import { downloadFile } from '@/lib/uploads/core/storage-service'
 import { getFileMetadataById } from '@/lib/uploads/server/metadata'
+import { formatFileSize } from '@/lib/uploads/utils/file-utils'
 import { verifyFileAccess } from '@/app/api/files/authorization'
 import { encodeFilenameForHeader } from '@/app/api/files/utils'
 
 const logger = createLogger('FilesExportAPI')
 
+/**
+ * Bundling caps. The embed list comes from scanning the document body, so its length
+ * and the bytes behind it are whatever the author put there — without these the export
+ * would materialize an unbounded number of unbounded assets in one request.
+ *
+ * The byte ceilings are the real bound and match the bulk-download route, so the two
+ * export surfaces reject at the same size. The count is only a guard on the metadata
+ * lookups that precede the byte check, so it sits far above any hand-authored document
+ * rather than at a number a screenshot-heavy doc could plausibly reach.
+ */
+const MAX_EXPORT_ASSETS = 500
+const MAX_EXPORT_ASSET_BYTES = 25 * 1024 * 1024
+const MAX_EXPORT_TOTAL_BYTES = 250 * 1024 * 1024
+
 const MARKDOWN_MIME_TYPES = new Set(['text/markdown', 'text/x-markdown'])
 const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown'])
 
@@ -136,34 +152,73 @@ export const GET = withRouteHandler(
       })
     }
 
-    const fetchResults = await Promise.allSettled(
-      imageIds.map(async (imageId) => {
-        const imgRecord = await getFileMetadataById(imageId)
-        if (!imgRecord) return null
-        const imgHasAccess = await verifyFileAccess(imgRecord.key, userId)
-        if (!imgHasAccess) return null
-        const imgBuffer = await downloadFile({
-          key: imgRecord.key,
-          context: imgRecord.context as StorageContext,
-        })
-        return { imageId, originalName: imgRecord.originalName, buffer: imgBuffer }
+    if (imageIds.length > MAX_EXPORT_ASSETS) {
+      return NextResponse.json(
+        {
+          error: `This document embeds ${imageIds.length} files, more than the ${MAX_EXPORT_ASSETS} an export can bundle.`,
+        },
+        { status: 400 }
+      )
+    }
+
+    // Metadata first: declared sizes bound the download before a byte is read, and the
+    // authorization check costs nothing to run here.
+    const assetTargets = (
+      await mapWithConcurrency(imageIds, MATERIALIZE_CONCURRENCY, async (imageId) => {
+        try {
+          const imgRecord = await getFileMetadataById(imageId)
+          if (!imgRecord) return null
+          if (!(await verifyFileAccess(imgRecord.key, userId))) return null
+          return { imageId, record: imgRecord }
+        } catch (error) {
+          logger.warn('Failed to resolve asset for export', {
+            imageId,
+            error: toError(error).message,
+          })
+          return null
+        }
       })
+    ).filter((target): target is NonNullable<typeof target> => target !== null)
+
+    const declaredAssetBytes = assetTargets.reduce((sum, target) => sum + target.record.size, 0)
+    if (declaredAssetBytes > MAX_EXPORT_TOTAL_BYTES) {
+      return NextResponse.json(
+        {
+          error: `Embedded files total ${formatFileSize(declaredAssetBytes)}, which exceeds the ${formatFileSize(MAX_EXPORT_TOTAL_BYTES)} export limit.`,
+        },
+        { status: 400 }
+      )
+    }
+
+    const fetched = await mapWithConcurrency(
+      assetTargets,
+      MATERIALIZE_CONCURRENCY,
+      async ({ imageId, record: imgRecord }) => {
+        try {
+          const buffer = await downloadFile({
+            key: imgRecord.key,
+            context: imgRecord.context as StorageContext,
+            maxBytes: MAX_EXPORT_ASSET_BYTES,
+          })
+          return { imageId, originalName: imgRecord.originalName, buffer }
+        } catch (error) {
+          // A single unreadable or oversized asset drops out of the bundle rather than
+          // failing the whole export; the markdown keeps its original link.
+          logger.warn('Failed to fetch asset for export', {
+            imageId,
+            error: toError(error).message,
+          })
+          return null
+        }
+      }
     )
 
     const assetMap = new Map<string, { filename: string; buffer: Buffer }>()
     const usedFilenames = new Set<string>()
 
-    for (let i = 0; i < fetchResults.length; i++) {
-      const result = fetchResults[i]
-      if (result.status === 'rejected') {
-        logger.warn('Failed to fetch asset for export', {
-          imageId: imageIds[i],
-          error: toError(result.reason).message,
-        })
-        continue
-      }
-      if (!result.value) continue
-      const { imageId, originalName, buffer } = result.value
+    for (const result of fetched) {
+      if (!result) continue
+      const { imageId, originalName, buffer } = result
       const preferred = safeFilename(originalName)
       const filename = deduplicatedFilename(preferred, usedFilenames, imageId)
       usedFilenames.add(filename)

apps/sim/app/api/tools/file/manage/route.ts

@@ -11,6 +11,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
 import { splitWorkspaceFilePath } from '@/lib/copilot/tools/server/files/workspace-file'
 import { acquireLock, releaseLock } from '@/lib/core/config/redis'
 import { generateRequestId } from '@/lib/core/utils/request'
+import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { ensureAbsoluteUrl } from '@/lib/core/utils/urls'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers'
@@ -685,7 +686,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
           const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger)
           if (denied) return denied
 
-          const buffer = await downloadFileFromStorage(userFile, requestId, logger, {
+          // Generated docs store their generation source, not the rendered binary, so
+          // the archive must carry the servable bytes instead of the raw source text.
+          // A still-compiling artifact throws, and the handler's catch turns that into
+          // the shared 409 via `docNotReadyResponse`.
+          const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, {
             maxBytes: MAX_COMPRESS_FILE_BYTES,
           })
           totalBytes += buffer.length
@@ -864,6 +869,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     }
     const notReady = docNotReadyResponse(error)
     if (notReady) return notReady
+    // A file over its per-file cap is a size rejection, not a fault. Rendered
+    // documents can cross it even when the stored source was well under.
+    if (isPayloadSizeLimitError(error)) {
+      return NextResponse.json({ success: false, error: error.message }, { status: 413 })
+    }
     if (error instanceof ShareValidationError) {
       return NextResponse.json({ success: false, error: error.message }, { status: 400 })
     }

apps/sim/app/api/v1/files/[fileId]/route.ts

@@ -6,7 +6,11 @@ import { parseRequest } from '@/lib/api/server'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { captureServerEvent } from '@/lib/posthog/server'
-import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace'
+import {
+  fetchServableWorkspaceFileBuffer,
+  getWorkspaceFile,
+} from '@/lib/uploads/contexts/workspace'
+import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/servable-file-response'
 import { performDeleteWorkspaceFileItems } from '@/lib/workspace-files/orchestration'
 import {
   checkRateLimit,
@@ -48,7 +52,9 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo
       return NextResponse.json({ error: 'File not found' }, { status: 404 })
     }
 
-    const buffer = await fetchWorkspaceFileBuffer(fileRecord)
+    // Generated docs store their generation source; serve the rendered artifact.
+    // Its content type is the rendered one, not the source MIME on the record.
+    const { buffer, contentType } = await fetchServableWorkspaceFileBuffer(fileRecord)
 
     recordAudit({
       workspaceId,
@@ -73,21 +79,30 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo
       { groups: { workspace: workspaceId } }
     )
 
-    return new Response(new Uint8Array(buffer), {
-      status: 200,
-      headers: {
-        'Content-Type': fileRecord.type || 'application/octet-stream',
-        'Content-Disposition': `attachment; filename="${fileRecord.name.replace(/[^\w.-]/g, '_')}"; filename*=UTF-8''${encodeURIComponent(fileRecord.name)}`,
-        'Content-Length': String(buffer.length),
-        'X-File-Id': fileRecord.id,
-        'X-File-Name': encodeURIComponent(fileRecord.name),
-        'X-Uploaded-At':
-          fileRecord.uploadedAt instanceof Date
-            ? fileRecord.uploadedAt.toISOString()
-            : String(fileRecord.uploadedAt),
-      },
-    })
+    // View, not copy — a second full copy would double peak memory for a large file.
+    return new Response(
+      new Uint8Array(buffer.buffer as ArrayBuffer, buffer.byteOffset, buffer.byteLength),
+      {
+        status: 200,
+        headers: {
+          'Content-Type': contentType || fileRecord.type || 'application/octet-stream',
+          'Content-Disposition': `attachment; filename="${fileRecord.name.replace(/[^\w.-]/g, '_')}"; filename*=UTF-8''${encodeURIComponent(fileRecord.name)}`,
+          'Content-Length': String(buffer.length),
+          'X-File-Id': fileRecord.id,
+          'X-File-Name': encodeURIComponent(fileRecord.name),
+          'X-Uploaded-At':
+            fileRecord.uploadedAt instanceof Date
+              ? fileRecord.uploadedAt.toISOString()
+              : String(fileRecord.uploadedAt),
+        },
+      }
+    )
   } catch (error) {
+    // A generated doc whose artifact is still compiling is retryable, not a fault:
+    // without this the caller sees a 500 and has no reason to try again.
+    if (isDocNotReadyError(error)) {
+      return NextResponse.json({ error: docNotReadyMessage() }, { status: 409 })
+    }
     logger.error(`[${requestId}] Error downloading file:`, error)
     return NextResponse.json({ error: 'Failed to download file' }, { status: 500 })
   }