Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions apps/sim/lib/copilot/tools/server/files/doc-compile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -541,8 +541,9 @@ export async function resolveServableDocBytes(args: {
}
}

// Reaches here only for xlsx, which has no isolated-vm fallback.
if (!format) return { buffer: rawBuffer, contentType: getContentType(fileName) }
// Reaches here only for xlsx, which has no isolated-vm fallback. Returning these
// bytes would expose generation source as a spreadsheet.
if (!format) throw new DocCompileUserError('Document is still being generated')

const cacheKey = sha256Hex(`${ext}${source}${workspaceId ?? ''}`)
const cached = compiledDocCache.get(cacheKey)
Expand Down
30 changes: 23 additions & 7 deletions apps/sim/lib/copilot/tools/server/files/doc-servable.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,14 +158,30 @@ describe('resolveServableDocBytes', () => {
expect(mockRunSandboxTask).not.toHaveBeenCalled()
})

it('returns raw XLSX source when there is no workspaceId (xlsx has no isolated-vm path)', async () => {
const result = await resolveServableDocBytes({
rawBuffer: XLSX_SOURCE,
fileName: 'sheet.xlsx',
workspaceId: undefined,
})
it('throws instead of returning XLSX source when E2B is disabled', async () => {
mockLoadCompiledDoc.mockResolvedValue(null)
setEnvFlags({ isDocSandboxEnabled: false })

await expect(
resolveServableDocBytes({
rawBuffer: XLSX_SOURCE,
fileName: 'sheet.xlsx',
workspaceId: WORKSPACE_ID,
})
).rejects.toBeInstanceOf(DocCompileUserError)

expect(mockRunSandboxTask).not.toHaveBeenCalled()
})

it('throws instead of returning XLSX source when there is no workspaceId', async () => {
await expect(
resolveServableDocBytes({
rawBuffer: XLSX_SOURCE,
fileName: 'sheet.xlsx',
workspaceId: undefined,
})
).rejects.toBeInstanceOf(DocCompileUserError)

expect(result.buffer).toBe(XLSX_SOURCE)
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
expect(mockRunSandboxTask).not.toHaveBeenCalled()
})
Expand Down
56 changes: 56 additions & 0 deletions apps/sim/lib/execution/payloads/materialization.server.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'

const { mockDownloadServableFileFromStorage, mockVerifyFileAccess } = vi.hoisted(() => ({
mockDownloadServableFileFromStorage: vi.fn(),
mockVerifyFileAccess: vi.fn(),
}))

vi.mock('@/lib/uploads/utils/file-utils.server', () => ({
downloadServableFileFromStorage: mockDownloadServableFileFromStorage,
}))

vi.mock('@/app/api/files/authorization', () => ({
verifyFileAccess: mockVerifyFileAccess,
}))

import { readUserFileContent } from '@/lib/execution/payloads/materialization.server'
import type { UserFile } from '@/executor/types'

const PDF_SOURCE = Buffer.from('from reportlab.pdfgen import canvas')
const PDF_BYTES = Buffer.from('%PDF-1.4 rendered bytes')

const generatedPdf: UserFile = {
id: 'file-1',
name: 'report.pdf',
url: '',
size: PDF_SOURCE.length,
type: 'text/x-python-pdf',
key: 'workspace/2f1d8c3e-5b6a-4c7d-8e9f-0a1b2c3d4e5f/1700000000000-abc1234-report.pdf',
}

describe('readUserFileContent', () => {
beforeEach(() => {
vi.clearAllMocks()
generatedPdf.size = PDF_SOURCE.length
mockVerifyFileAccess.mockResolvedValue(true)
mockDownloadServableFileFromStorage.mockResolvedValue({
buffer: PDF_BYTES,
contentType: 'application/pdf',
})
})

it('returns the compiled artifact instead of the stored generation source', async () => {
const content = await readUserFileContent(generatedPdf, {
userId: 'user-1',
encoding: 'base64',
})

expect(mockDownloadServableFileFromStorage).toHaveBeenCalledOnce()
expect(content).toBe(PDF_BYTES.toString('base64'))
expect(content).not.toBe(PDF_SOURCE.toString('base64'))
expect(generatedPdf.size).toBe(PDF_BYTES.length)
})
})
23 changes: 20 additions & 3 deletions apps/sim/lib/execution/payloads/materialization.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,12 @@ import {
} from '@/lib/execution/payloads/large-value-ref'
import { ExecutionResourceLimitError } from '@/lib/execution/resource-errors'
import type { StorageContext } from '@/lib/uploads'
import { bufferToBase64, inferContextFromKey } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import {
bufferToBase64,
inferContextFromKey,
isGeneratedDocumentSourceType,
} from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import type { UserFile } from '@/executor/types'

const logger = createLogger('ExecutionPayloadMaterialization')
Expand Down Expand Up @@ -267,6 +271,11 @@ export async function assertUserFileContentAccess(
}
}

/**
* Reads the bytes a consumer should receive. For generated documents, updates the
* file's size to the rendered artifact size so downstream attachment routing does
* not make decisions from the smaller generation-source size.
*/
export async function readUserFileContent(
file: unknown,
options: ReadUserFileContentOptions
Expand All @@ -291,9 +300,14 @@ export async function readUserFileContent(
const requestId = options.requestId ?? 'unknown'

try {
buffer = await downloadFileFromStorage(file, requestId, log, { maxBytes: maxSourceBytes })
buffer = (
await downloadServableFileFromStorage(file, requestId, log, { maxBytes: maxSourceBytes })
).buffer
Comment thread
j15z marked this conversation as resolved.
Comment thread
j15z marked this conversation as resolved.
Comment thread
j15z marked this conversation as resolved.
} catch (error) {
if (isPayloadSizeLimitError(error)) {
if (isGeneratedDocumentSourceType(file.type) && error.observedBytes !== undefined) {
file.size = error.observedBytes
}
throw new ExecutionResourceLimitError({
resource: 'execution_payload_bytes',
attemptedBytes: error.observedBytes ?? maxSourceBytes + 1,
Expand All @@ -306,6 +320,9 @@ export async function readUserFileContent(
if (!buffer) {
throw new Error(`File content for ${file.name} is unavailable.`)
}
if (isGeneratedDocumentSourceType(file.type)) {
file.size = buffer.length
}
if (buffer.length > maxSourceBytes) {
throw new ExecutionResourceLimitError({
resource: 'execution_payload_bytes',
Expand Down
51 changes: 47 additions & 4 deletions apps/sim/lib/uploads/utils/file-utils.server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,51 @@
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'

const { mockDownloadFile } = vi.hoisted(() => ({
mockDownloadFile: vi.fn(),
}))
const { mockDownloadFile, mockParseWorkspaceFileKey, mockResolveServableDocBytes } = vi.hoisted(
() => ({
mockDownloadFile: vi.fn(),
mockParseWorkspaceFileKey: vi.fn(),
mockResolveServableDocBytes: vi.fn(),
})
)

vi.mock('@/lib/uploads/core/storage-service', () => ({
downloadFile: mockDownloadFile,
hasCloudStorage: vi.fn(() => true),
}))

vi.mock('@/lib/uploads/contexts/execution/execution-file-manager', () => ({
downloadExecutionFile: mockDownloadFile,
}))

vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
parseWorkspaceFileKey: mockParseWorkspaceFileKey,
}))

vi.mock('@/lib/copilot/tools/server/files/doc-compile', () => ({
resolveServableDocBytes: mockResolveServableDocBytes,
}))

vi.mock('@/app/api/files/authorization', () => ({
verifyFileAccess: vi.fn(),
}))

import { createLogger } from '@sim/logger'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import {
downloadFileFromStorage,
downloadServableFileFromStorage,
} from '@/lib/uploads/utils/file-utils.server'
import type { UserFile } from '@/executor/types'

describe('downloadFileFromStorage context derivation', () => {
beforeEach(() => {
vi.clearAllMocks()
mockDownloadFile.mockResolvedValue(Buffer.from('bytes'))
mockParseWorkspaceFileKey.mockReturnValue(null)
mockResolveServableDocBytes.mockImplementation(async ({ rawBuffer }) => ({
buffer: rawBuffer,
contentType: 'application/pdf',
}))
})

it('downloads with the key-derived context, ignoring a caller-supplied public context', async () => {
Expand All @@ -44,4 +68,23 @@ describe('downloadFileFromStorage context derivation', () => {
expect.objectContaining({ key: userFile.key, context: 'workspace' })
)
})

it('uses the workspace ID embedded in an execution key to resolve generated artifacts', async () => {
const workspaceId = '2f1d8c3e-5b6a-4c7d-8e9f-0a1b2c3d4e5f'
const userFile: UserFile = {
id: 'f1',
name: 'report.pdf',
url: '',
size: 5,
type: 'text/x-python-pdf',
key: `execution/${workspaceId}/3f2e9d4c-6a7b-4d8e-9f0a-1b2c3d4e5f6a/4a3b2c1d-7e8f-4a9b-8c0d-1e2f3a4b5c6d/report.pdf`,
context: 'execution',
}

await downloadServableFileFromStorage(userFile, 'req-1', createLogger('test'))

expect(mockResolveServableDocBytes).toHaveBeenCalledWith(
expect.objectContaining({ workspaceId })
)
})
})
7 changes: 6 additions & 1 deletion apps/sim/lib/uploads/utils/file-utils.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { StorageService } from '@/lib/uploads'
import { isExecutionFile } from '@/lib/uploads/contexts/execution/utils'
import {
extractStorageKey,
extractWorkspaceIdFromExecutionKey,
getFileExtension,
getMimeTypeFromExtension,
inferContextFromKey,
Expand Down Expand Up @@ -384,7 +385,11 @@ export async function downloadServableFileFromStorage(
const { parseWorkspaceFileKey } = await import(
'@/lib/uploads/contexts/workspace/workspace-file-manager'
)
const workspaceId = userFile.key ? (parseWorkspaceFileKey(userFile.key) ?? undefined) : undefined
const workspaceId = userFile.key
? (parseWorkspaceFileKey(userFile.key) ??
extractWorkspaceIdFromExecutionKey(userFile.key) ??
undefined)
: undefined

const { resolveServableDocBytes } = await import('@/lib/copilot/tools/server/files/doc-compile')
const resolved = await resolveServableDocBytes({
Expand Down
Loading
Loading