diff --git a/apps/docs/openapi-v2-knowledge.json b/apps/docs/openapi-v2-knowledge.json index eeac6c843b5..806bbbccd83 100644 --- a/apps/docs/openapi-v2-knowledge.json +++ b/apps/docs/openapi-v2-knowledge.json @@ -709,9 +709,6 @@ "404": { "$ref": "#/components/responses/NotFound" }, - "409": { - "$ref": "#/components/responses/Conflict" - }, "413": { "description": "The uploaded file exceeds the 100 MB limit, or the workspace storage limit has been reached.", "content": { diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts index bf347078d73..ce255768db1 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts @@ -151,7 +151,10 @@ describe('Knowledge Connector By ID API Route', () => { success: true, userId: 'user-1', }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) + mockCheckWriteAccess.mockResolvedValue({ + hasAccess: true, + knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, + }) dbChainMockFns.limit.mockResolvedValueOnce([]) const req = createMockRequest('PATCH', { sourceConfig: { project: 'NEW' } }) @@ -174,7 +177,8 @@ describe('Knowledge Connector By ID API Route', () => { mockHasWorkspaceLiveSyncAccess.mockResolvedValue(true) const updatedConnector = { id: 'conn-456', status: 'paused', syncIntervalMinutes: 5 } - dbChainMockFns.limit.mockResolvedValueOnce([updatedConnector]) + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', connectorType: 'jira' }]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedConnector]) const req = createMockRequest('PATCH', { status: 'paused', syncIntervalMinutes: 5 }) const response = await PATCH(req, { params: mockParams }) @@ -196,6 +200,7 @@ describe('Knowledge Connector By ID API Route', () => { knowledgeBase: { workspaceId: 'ws-free', name: 'Free KB' }, }) mockHasWorkspaceLiveSyncAccess.mockResolvedValue(false) + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', connectorType: 'jira' }]) const req = createMockRequest('PATCH', { syncIntervalMinutes: 5 }) const response = await PATCH(req, { params: mockParams }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts index 3ff1d479cd0..d63513af694 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts @@ -1,20 +1,29 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' -import { document, embedding, knowledgeConnector, knowledgeConnectorSyncLog } from '@sim/db/schema' +import { knowledgeConnectorSyncLog } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { and, desc, eq, inArray, isNull, sql } from 'drizzle-orm' +import { desc, eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' -import { updateKnowledgeConnectorContract } from '@/lib/api/contracts/knowledge' +import { + deleteKnowledgeConnectorContract, + updateKnowledgeConnectorContract, +} from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' import { decryptApiKey } from '@/lib/api-key/crypto' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' -import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service' -import { cleanupUnusedTagDefinitions } from '@/lib/knowledge/tags/service' -import { captureServerEvent } from '@/lib/posthog/server' +import { + getKnowledgeConnector, + type KnowledgeConnectorRow, + performDeleteKnowledgeConnector, + performUpdateKnowledgeConnector, + type SourceConfigRejection, +} from '@/lib/knowledge/orchestration' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' @@ -42,20 +51,8 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) } - const connectorRows = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - if (connectorRows.length === 0) { + const connector = await getKnowledgeConnector(knowledgeBaseId, connectorId) + if (!connector) { return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) } @@ -66,7 +63,7 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou .orderBy(desc(knowledgeConnectorSyncLog.startedAt)) .limit(10) - const { encryptedApiKey: _, ...connectorData } = connectorRows[0] + const { encryptedApiKey: _, ...connectorData } = connector return NextResponse.json({ success: true, data: { @@ -81,357 +78,179 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou }) /** - * PATCH /api/knowledge/[id]/connectors/[connectorId] - Update a connector + * Validates a replacement `sourceConfig` against the live source, resolving the + * connector's own token first. Returns a rejection message, or `null` to accept. + * + * Stays with the route rather than moving into orchestration because resolving + * the token needs the requesting identity: workspace credentials are shared and + * token reads are scoped to `account.userId`, so the credential's own account + * owner is used — not the knowledge base owner, and not the acting user when a + * service account mints its own token. */ -export const PATCH = withRouteHandler(async (request: NextRequest, context: RouteParams) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await context.params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const parsed = await parseRequest(updateKnowledgeConnectorContract, request, context) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - if ( - body.syncIntervalMinutes !== undefined && - body.syncIntervalMinutes > 0 && - body.syncIntervalMinutes < 60 - ) { - const workspaceId = writeCheck.knowledgeBase.workspaceId - if (!workspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace billing context' }, - { status: 409 } - ) - } - const canUseLiveSync = await hasWorkspaceLiveSyncAccess(workspaceId) - if (!canUseLiveSync) { - return NextResponse.json( - { error: 'Live sync requires a Max or Enterprise plan' }, - { status: 403 } - ) +function makeSourceConfigValidator( + actingUserId: string, + workspaceId: string | null, + connectorId: string +) { + return async ( + connector: KnowledgeConnectorRow, + sourceConfig: Record + ): Promise => { + const connectorConfig = CONNECTOR_REGISTRY[connector.connectorType] + if (!connectorConfig) { + return { + message: `Unknown connector type: ${connector.connectorType}`, + errorCode: 'validation', } } - if (body.sourceConfig !== undefined) { - const existingRows = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - if (existingRows.length === 0) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } - - const existing = existingRows[0] - const connectorConfig = CONNECTOR_REGISTRY[existing.connectorType] - - if (!connectorConfig) { - return NextResponse.json( - { error: `Unknown connector type: ${existing.connectorType}` }, - { status: 400 } - ) - } - - let accessToken: string | null = null - if (connectorConfig.auth.mode === 'apiKey') { - if (!existing.encryptedApiKey) { - return NextResponse.json( - { error: 'API key not found. Please reconfigure the connector.' }, - { status: 400 } - ) - } - accessToken = (await decryptApiKey(existing.encryptedApiKey)).decrypted - } else { - if (!existing.credentialId) { - return NextResponse.json( - { error: 'OAuth credential not found. Please reconfigure the connector.' }, - { status: 400 } - ) + let accessToken: string | null = null + if (connectorConfig.auth.mode === 'apiKey') { + if (!connector.encryptedApiKey) { + return { + message: 'API key not found. Please reconfigure the connector.', + errorCode: 'validation', } - const connectorWorkspaceId = writeCheck.knowledgeBase.workspaceId - if (!connectorWorkspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace context' }, - { status: 409 } - ) - } - /** - * Resolve the credential's own account owner, not the knowledge base owner: - * workspace credentials are shared, and token reads are scoped to - * `account.userId`. - */ - const identity = await resolveCredentialTokenIdentity( - existing.credentialId, - connectorWorkspaceId - ) - if (!identity) { - return NextResponse.json( - { error: 'Credential is no longer usable in this workspace. Please reconnect it.' }, - { status: 400 } - ) + } + accessToken = (await decryptApiKey(connector.encryptedApiKey)).decrypted + } else { + if (!connector.credentialId) { + return { + message: 'OAuth credential not found. Please reconfigure the connector.', + errorCode: 'validation', } - accessToken = await refreshAccessTokenIfNeeded( - existing.credentialId, - // Service accounts mint their own token and ignore the acting user. - identity.kind === 'oauth' ? identity.userId : auth.userId, - `patch-${connectorId}` - ) } - - if (!accessToken) { - return NextResponse.json( - { error: 'Failed to refresh access token. Please reconnect your account.' }, - { status: 401 } - ) + if (!workspaceId) { + return { + message: 'Knowledge base is missing workspace context', + errorCode: 'conflict', + } } - - const validation = await connectorConfig.validateConfig(accessToken, body.sourceConfig) - if (!validation.valid) { - return NextResponse.json( - { error: validation.error || 'Invalid source configuration' }, - { status: 400 } - ) + const identity = await resolveCredentialTokenIdentity(connector.credentialId, workspaceId) + if (!identity) { + return { + message: 'Credential is no longer usable in this workspace. Please reconnect it.', + errorCode: 'validation', + } } + accessToken = await refreshAccessTokenIfNeeded( + connector.credentialId, + // Service accounts mint their own token and ignore the acting user. + identity.kind === 'oauth' ? identity.userId : actingUserId, + `patch-${connectorId}` + ) } - const updates: Record = { updatedAt: new Date() } - if (body.sourceConfig !== undefined) { - updates.sourceConfig = body.sourceConfig - } - if (body.syncIntervalMinutes !== undefined) { - updates.syncIntervalMinutes = body.syncIntervalMinutes - if (body.syncIntervalMinutes > 0) { - updates.nextSyncAt = new Date(Date.now() + body.syncIntervalMinutes * 60 * 1000) - } else { - updates.nextSyncAt = null - } - } - if (body.status !== undefined) { - updates.status = body.status - if (body.status === 'active') { - updates.consecutiveFailures = 0 - updates.lastSyncError = null - if (updates.nextSyncAt === undefined) { - updates.nextSyncAt = new Date() - } + if (!accessToken) { + // A stale stored credential, not an unauthenticated caller — but the route + // has always answered 401 here, so keep that rather than silently + // reclassifying it as part of this refactor. + return { + message: 'Failed to refresh access token. Please reconnect your account.', + errorCode: 'unauthorized', } } - await db - .update(knowledgeConnector) - .set(updates) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) + const validation = await connectorConfig.validateConfig(accessToken, sourceConfig) + return validation.valid + ? null + : { message: validation.error || 'Invalid source configuration', errorCode: 'validation' } + } +} - const updated = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) +/** + * PATCH /api/knowledge/[id]/connectors/[connectorId] - Update a connector + */ +export const PATCH = withRouteHandler(async (request: NextRequest, context: RouteParams) => { + const requestId = generateRequestId() + const { id: knowledgeBaseId, connectorId } = await context.params - const { encryptedApiKey: __, ...updatedData } = updated[0] + const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_UPDATED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: updatedData.connectorType, - description: `Updated connector for knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - connectorType: updatedData.connectorType, - updatedFields: Object.keys(parsed.data), - ...(body.syncIntervalMinutes !== undefined && { - syncIntervalMinutes: body.syncIntervalMinutes, - }), - ...(body.status !== undefined && { newStatus: body.status }), - }, - request, - }) + const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) + if (!writeCheck.hasAccess) { + const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 + return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) + } - return NextResponse.json({ success: true, data: updatedData }) - } catch (error) { - logger.error(`[${requestId}] Error updating connector`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) + const parsed = await parseRequest(updateKnowledgeConnectorContract, request, context) + if (!parsed.success) return parsed.response + + const outcome = await performUpdateKnowledgeConnector({ + knowledgeBase: { + id: knowledgeBaseId, + name: writeCheck.knowledgeBase.name, + workspaceId: writeCheck.knowledgeBase.workspaceId ?? null, + }, + connectorId, + updates: parsed.data.body, + validateSourceConfig: makeSourceConfigValidator( + auth.userId, + writeCheck.knowledgeBase.workspaceId ?? null, + connectorId + ), + userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Internal server error') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) } + + return NextResponse.json({ success: true, data: outcome.connector }) }) /** * DELETE /api/knowledge/[id]/connectors/[connectorId] - Hard-delete a connector */ -export const DELETE = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => { +export const DELETE = withRouteHandler(async (request: NextRequest, context: RouteParams) => { const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const existingConnector = await db - .select({ id: knowledgeConnector.id, connectorType: knowledgeConnector.connectorType }) - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - if (existingConnector.length === 0) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } - - const { searchParams } = new URL(request.url) - const deleteDocuments = searchParams.get('deleteDocuments') === 'true' - - const { deletedDocs, docCount } = await db.transaction(async (tx) => { - await tx.execute(sql`SELECT 1 FROM knowledge_connector WHERE id = ${connectorId} FOR UPDATE`) - - // Includes pending-removal (tombstoned) docs — the connector is being - // deleted, so there's no future sync left to confirm or resurrect them. - const docs = await tx - .select({ id: document.id, fileUrl: document.fileUrl }) - .from(document) - .where(and(eq(document.connectorId, connectorId), isNull(document.archivedAt))) - - const documentIds = docs.map((doc) => doc.id) - if (deleteDocuments) { - if (documentIds.length > 0) { - await tx.delete(embedding).where(inArray(embedding.documentId, documentIds)) - await tx.delete(document).where(inArray(document.id, documentIds)) - } - } else if (documentIds.length > 0) { - // Kept documents become normal standalone KB entries once their connector - // is gone — resurrect any pending-removal ones rather than leaving them - // invisible tombstones with no future sync left to ever confirm or - // resurrect them. - await tx.update(document).set({ deletedAt: null }).where(inArray(document.id, documentIds)) - } - - const deletedConnectors = await tx - .delete(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .returning({ id: knowledgeConnector.id }) - - if (deletedConnectors.length === 0) { - throw new Error('Connector not found') - } - - return { deletedDocs: deleteDocuments ? docs : [], docCount: docs.length } - }) - - const kbWorkspaceId = writeCheck.knowledgeBase?.workspaceId ?? null + const { id: knowledgeBaseId, connectorId } = await context.params - if (deleteDocuments) { - await Promise.all([ - deletedDocs.length > 0 - ? deleteDocumentStorageFiles( - deletedDocs.map((doc) => ({ ...doc, workspaceId: kbWorkspaceId })), - requestId - ) - : Promise.resolve(), - cleanupUnusedTagDefinitions(knowledgeBaseId, requestId).catch((error) => { - logger.warn(`[${requestId}] Failed to cleanup tag definitions`, error) - }), - ]) - } + const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - logger.info( - `[${requestId}] Deleted connector ${connectorId}${deleteDocuments ? ` and ${docCount} documents` : `, kept ${docCount} documents`}` - ) + const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) + if (!writeCheck.hasAccess) { + const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 + return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) + } - captureServerEvent( - auth.userId, - 'knowledge_base_connector_removed', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId ?? '', - connector_type: existingConnector[0].connectorType, - documents_deleted: deleteDocuments ? docCount : 0, - }, - kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : undefined + const parsed = await parseRequest(deleteKnowledgeConnectorContract, request, context) + if (!parsed.success) return parsed.response + + const outcome = await performDeleteKnowledgeConnector({ + knowledgeBase: { + id: knowledgeBaseId, + name: writeCheck.knowledgeBase.name, + workspaceId: writeCheck.knowledgeBase.workspaceId ?? null, + }, + connectorId, + deleteDocuments: parsed.data.query.deleteDocuments, + userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Internal server error') }, + { status: statusForOrchestrationError(outcome.errorCode) } ) - - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_DELETED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: existingConnector[0].connectorType, - description: `Deleted connector from knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - connectorType: existingConnector[0].connectorType, - deleteDocuments, - documentsDeleted: deleteDocuments ? docCount : 0, - documentsKept: deleteDocuments ? 0 : docCount, - }, - request, - }) - - return NextResponse.json({ success: true }) - } catch (error) { - logger.error(`[${requestId}] Error deleting connector`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) } + + return NextResponse.json({ success: true }) }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts index c79c85df58a..b8869013644 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts @@ -62,7 +62,10 @@ describe('Connector Manual Sync API Route', () => { success: true, userId: 'user-1', }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) + mockCheckWriteAccess.mockResolvedValue({ + hasAccess: true, + knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, + }) dbChainMockFns.limit.mockResolvedValueOnce([]) const req = createMockRequest('POST') @@ -76,7 +79,10 @@ describe('Connector Manual Sync API Route', () => { success: true, userId: 'user-1', }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) + mockCheckWriteAccess.mockResolvedValue({ + hasAccess: true, + knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, + }) dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', status: 'syncing' }]) const req = createMockRequest('POST') diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts index 714f554040b..21e6bfdb50e 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts @@ -1,8 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { db } from '@sim/db' -import { knowledgeConnector } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { and, eq, isNull } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { triggerKnowledgeConnectorSyncContract } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' @@ -11,14 +6,15 @@ import { requireBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { dispatchSync } from '@/lib/knowledge/connectors/queue' -import { captureServerEvent } from '@/lib/posthog/server' +import { performSyncKnowledgeConnector } from '@/lib/knowledge/orchestration' import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' -const logger = createLogger('ConnectorManualSyncAPI') - type RouteParams = { params: Promise<{ id: string; connectorId: string }> } /** @@ -31,105 +27,50 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route const { id: knowledgeBaseId, connectorId } = parsed.data.params const { rehydrate } = parsed.data.query - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const connectorRows = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) + const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - if (connectorRows.length === 0) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } + const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) + if (!writeCheck.hasAccess) { + const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 + return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) + } - if (connectorRows[0].status === 'syncing') { - return NextResponse.json({ error: 'Sync already in progress' }, { status: 409 }) - } + const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId ?? null - const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId - if (!kbWorkspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace billing context' }, - { status: 409 } - ) - } - const billingAttribution = + const outcome = await performSyncKnowledgeConnector({ + knowledgeBase: { + id: knowledgeBaseId, + name: writeCheck.knowledgeBase.name, + workspaceId: kbWorkspaceId, + }, + connectorId, + resolveBillingAttribution: async () => auth.authType === AuthType.INTERNAL_JWT ? requireBillingAttributionHeader(request.headers, { - actorUserId: auth.userId, - workspaceId: kbWorkspaceId, - }) - : await resolveBillingAttribution({ - actorUserId: auth.userId, - workspaceId: kbWorkspaceId, + actorUserId: auth.userId as string, + workspaceId: kbWorkspaceId as string, }) - - logger.info( - `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` - ) - - captureServerEvent( - auth.userId, - 'knowledge_base_connector_synced', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId, - connector_type: connectorRows[0].connectorType, - }, - kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : undefined + : resolveBillingAttribution({ + actorUserId: auth.userId as string, + workspaceId: kbWorkspaceId as string, + }), + rehydrate, + userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Internal server error') }, + { status: statusForOrchestrationError(outcome.errorCode) } ) - - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_SYNCED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: connectorRows[0].connectorType, - description: `Triggered manual sync for connector on knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - connectorType: connectorRows[0].connectorType, - connectorStatus: connectorRows[0].status, - syncType: rehydrate ? 'manual-rehydrate' : 'manual', - }, - request, - }) - - dispatchSync(connectorId, { billingAttribution, requestId, rehydrate }).catch((error) => { - logger.error( - `[${requestId}] Failed to dispatch manual sync for connector ${connectorId}`, - error - ) - }) - - return NextResponse.json({ - success: true, - message: 'Sync triggered', - }) - } catch (error) { - logger.error(`[${requestId}] Error triggering manual sync`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) } + + return NextResponse.json({ success: true, message: 'Sync triggered' }) }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts index 6087572fb40..361a8e2ad68 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts @@ -118,7 +118,8 @@ describe('Knowledge Connectors API Route', () => { }) mockHasWorkspaceLiveSyncAccess.mockResolvedValue(true) mockResolveBillingAttribution.mockResolvedValue(BILLING_ATTRIBUTION) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'knowledge-base-1' }]).mockResolvedValueOnce([ + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'knowledge-base-1' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ { id: 'connector-1', knowledgeBaseId: 'knowledge-base-1', @@ -173,6 +174,8 @@ describe('Knowledge Connectors API Route', () => { expect(response.status).toBe(403) expect(mockHasWorkspaceLiveSyncAccess).toHaveBeenCalledWith('workspace-free') + // The payer is resolved lazily, so a request the plan gate rejects never + // pays for the lookup. expect(mockResolveBillingAttribution).not.toHaveBeenCalled() expect(mockDispatchSync).not.toHaveBeenCalled() }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.ts index b7df3198990..df2f246ae1d 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/route.ts @@ -1,28 +1,25 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' -import { knowledgeBase, knowledgeBaseTagDefinitions, knowledgeConnector } from '@sim/db/schema' +import { knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { and, desc, eq, isNull, sql } from 'drizzle-orm' +import { and, desc, eq, isNull } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { createKnowledgeConnectorContract } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' -import { encryptApiKey } from '@/lib/api-key/crypto' import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { requireBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' -import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' +import { + messageForOrchestrationError, + OrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { dispatchSync } from '@/lib/knowledge/connectors/queue' -import { allocateTagSlots } from '@/lib/knowledge/constants' -import { createTagDefinition } from '@/lib/knowledge/tags/service' -import { captureServerEvent } from '@/lib/posthog/server' +import { performCreateKnowledgeConnector } from '@/lib/knowledge/orchestration' import { getCredential } from '@/app/api/auth/oauth/utils' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' -import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' const logger = createLogger('KnowledgeConnectorsAPI') @@ -80,263 +77,71 @@ export const POST = withRouteHandler( const requestId = generateRequestId() const { id: knowledgeBaseId } = await context.params - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json( - { error: status === 404 ? 'Not found' : 'Unauthorized' }, - { status } - ) - } + const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - const parsed = await parseRequest(createKnowledgeConnectorContract, request, context) - if (!parsed.success) return parsed.response + const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) + if (!writeCheck.hasAccess) { + const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 + return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) + } - const { connectorType, credentialId, apiKey, sourceConfig, syncIntervalMinutes } = - parsed.data.body + const parsed = await parseRequest(createKnowledgeConnectorContract, request, context) + if (!parsed.success) return parsed.response - const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId - if (!kbWorkspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace billing context' }, - { status: 409 } - ) - } + const { connectorType, credentialId, apiKey, sourceConfig, syncIntervalMinutes } = + parsed.data.body - if (syncIntervalMinutes > 0 && syncIntervalMinutes < 60) { - const canUseLiveSync = await hasWorkspaceLiveSyncAccess(kbWorkspaceId) - if (!canUseLiveSync) { - return NextResponse.json( - { error: 'Live sync requires a Max or Enterprise plan' }, - { status: 403 } - ) - } - } + const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId + if (!kbWorkspaceId) { + return NextResponse.json( + { error: 'Knowledge base is missing workspace billing context' }, + { status: 409 } + ) + } - const billingAttribution = + const outcome = await performCreateKnowledgeConnector({ + knowledgeBase: { + id: knowledgeBaseId, + name: writeCheck.knowledgeBase.name, + workspaceId: kbWorkspaceId, + }, + connectorType, + credentialId, + apiKey, + sourceConfig, + syncIntervalMinutes, + resolveBillingAttribution: async () => auth.authType === AuthType.INTERNAL_JWT ? requireBillingAttributionHeader(request.headers, { - actorUserId: auth.userId, + actorUserId: auth.userId as string, workspaceId: kbWorkspaceId, }) - : await resolveBillingAttribution({ - actorUserId: auth.userId, + : resolveBillingAttribution({ + actorUserId: auth.userId as string, workspaceId: kbWorkspaceId, - }) - - const connectorConfig = CONNECTOR_REGISTRY[connectorType] - if (!connectorConfig) { - return NextResponse.json( - { error: `Unknown connector type: ${connectorType}` }, - { status: 400 } - ) - } - - let resolvedCredentialId: string | null = null - let resolvedEncryptedApiKey: string | null = null - let accessToken: string - - if (connectorConfig.auth.mode === 'apiKey') { - if (!apiKey) { - return NextResponse.json({ error: 'API key is required' }, { status: 400 }) - } - accessToken = apiKey - } else { - if (!credentialId) { - return NextResponse.json({ error: 'Credential is required' }, { status: 400 }) - } - - const credential = await getCredential(requestId, credentialId, auth.userId) - if (!credential) { - return NextResponse.json({ error: 'Credential not found' }, { status: 400 }) - } - - if (!credential.accessToken) { - return NextResponse.json( - { error: 'Credential has no access token. Please reconnect your account.' }, - { status: 400 } - ) - } - - accessToken = credential.accessToken - resolvedCredentialId = credentialId - } - - const configValidation = await connectorConfig.validateConfig(accessToken, sourceConfig) - if (!configValidation.valid) { - return NextResponse.json( - { error: configValidation.error || 'Invalid source configuration' }, - { status: 400 } - ) - } - - let finalSourceConfig: Record = { ...sourceConfig } - - if (connectorConfig.auth.mode === 'apiKey' && apiKey) { - const { encrypted } = await encryptApiKey(apiKey) - resolvedEncryptedApiKey = encrypted - } - - const tagSlotMapping: Record = {} - let newTagSlots: Record = {} - - if (connectorConfig.tagDefinitions?.length) { - const disabledIds = new Set((sourceConfig.disabledTagIds as string[] | undefined) ?? []) - const enabledDefs = connectorConfig.tagDefinitions.filter((td) => !disabledIds.has(td.id)) - - const existingDefs = await db - .select({ - tagSlot: knowledgeBaseTagDefinitions.tagSlot, - displayName: knowledgeBaseTagDefinitions.displayName, - fieldType: knowledgeBaseTagDefinitions.fieldType, - }) - .from(knowledgeBaseTagDefinitions) - .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)) - - const usedSlots = new Set(existingDefs.map((d) => d.tagSlot)) - const existingByName = new Map( - existingDefs.map((d) => [d.displayName, { tagSlot: d.tagSlot, fieldType: d.fieldType }]) - ) - - const defsNeedingSlots: typeof enabledDefs = [] - for (const td of enabledDefs) { - const existing = existingByName.get(td.displayName) - if (existing && existing.fieldType === td.fieldType) { - tagSlotMapping[td.id] = existing.tagSlot - } else { - defsNeedingSlots.push(td) - } - } - - const { mapping, skipped: skippedTags } = allocateTagSlots(defsNeedingSlots, usedSlots) - Object.assign(tagSlotMapping, mapping) - newTagSlots = mapping - - for (const name of skippedTags) { - logger.warn(`[${requestId}] No available slots for "${name}"`) - } - - if (skippedTags.length > 0 && Object.keys(tagSlotMapping).length === 0) { - return NextResponse.json( - { error: `No available tag slots. Could not assign: ${skippedTags.join(', ')}` }, - { status: 422 } - ) - } - - finalSourceConfig = { ...finalSourceConfig, tagSlotMapping } - } - - const now = new Date() - const connectorId = generateId() - const nextSyncAt = - syncIntervalMinutes > 0 ? new Date(now.getTime() + syncIntervalMinutes * 60 * 1000) : null - - await db.transaction(async (tx) => { - await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`) - - const activeKb = await tx - .select({ id: knowledgeBase.id }) - .from(knowledgeBase) - .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) - .limit(1) - - if (activeKb.length === 0) { - throw new Error('Knowledge base not found') - } - - for (const [semanticId, slot] of Object.entries(newTagSlots)) { - const td = connectorConfig.tagDefinitions!.find((d) => d.id === semanticId)! - await createTagDefinition( - { - knowledgeBaseId, - tagSlot: slot, - displayName: td.displayName, - fieldType: td.fieldType, - }, - requestId, - tx - ) - } - - await tx.insert(knowledgeConnector).values({ - id: connectorId, - knowledgeBaseId, - connectorType, - credentialId: resolvedCredentialId, - encryptedApiKey: resolvedEncryptedApiKey, - sourceConfig: finalSourceConfig, - syncIntervalMinutes, - status: 'active', - nextSyncAt, - createdAt: now, - updatedAt: now, - }) - }) - - logger.info(`[${requestId}] Created connector ${connectorId} for KB ${knowledgeBaseId}`) - - captureServerEvent( - auth.userId, - 'knowledge_base_connector_added', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId, - connector_type: connectorType, - sync_interval_minutes: syncIntervalMinutes, - }, - { - groups: kbWorkspaceId ? { workspace: kbWorkspaceId } : undefined, - setOnce: { first_connector_added_at: new Date().toISOString() }, - } + }), + resolveAccessToken: async (id) => { + const credential = await getCredential(requestId, id, auth.userId as string) + if (!credential) throw new OrchestrationError('validation', 'Credential not found') + return credential.accessToken ?? null + }, + userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Internal server error') }, + { status: statusForOrchestrationError(outcome.errorCode) } ) - - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_CREATED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: connectorType, - description: `Created ${connectorType} connector for knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - connectorType, - syncIntervalMinutes, - authMode: connectorConfig.auth.mode, - }, - request, - }) - - dispatchSync(connectorId, { billingAttribution, requestId }).catch((error) => { - logger.error( - `[${requestId}] Failed to dispatch initial sync for connector ${connectorId}`, - error - ) - }) - - const created = await db - .select() - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, connectorId)) - .limit(1) - - const { encryptedApiKey: _, ...createdData } = created[0] - return NextResponse.json({ success: true, data: createdData }, { status: 201 }) - } catch (error) { - if (error instanceof Error && error.message === 'Knowledge base not found') { - return NextResponse.json({ error: 'Not found' }, { status: 404 }) - } - logger.error(`[${requestId}] Error creating connector`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) } + + return NextResponse.json({ success: true, data: outcome.connector }, { status: 201 }) } ) diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts index 7acdc821391..dab693e462f 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { updateKnowledgeDocumentContract } from '@/lib/api/contracts/knowledge' @@ -8,15 +7,19 @@ import { requireBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' +import { + messageForOrchestrationError, + type OrchestrationErrorCode, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { - deleteDocument, - markDocumentAsFailedTimeout, - retryDocumentProcessing, - updateDocument, -} from '@/lib/knowledge/documents/service' -import { captureServerEvent } from '@/lib/posthog/server' + performDeleteKnowledgeDocument, + performMarkKnowledgeDocumentTimedOut, + performRetryKnowledgeDocumentProcessing, + performUpdateKnowledgeDocument, +} from '@/lib/knowledge/orchestration' import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils' const logger = createLogger('DocumentByIdAPI') @@ -108,58 +111,30 @@ export const PUT = withRouteHandler( ) if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const updateData: any = {} + const { markFailedDueToTimeout, retryProcessing, ...documentUpdates } = parsed.data.body + const doc = accessCheck.document + const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? null - if (validatedData.markFailedDueToTimeout) { - const doc = accessCheck.document - - if (doc.processingStatus !== 'processing') { - return NextResponse.json( - { error: `Document is not in processing state (current: ${doc.processingStatus})` }, - { status: 400 } - ) - } - - if (!doc.processingStartedAt) { - return NextResponse.json( - { error: 'Document has no processing start time' }, - { status: 400 } - ) - } - - try { - await markDocumentAsFailedTimeout(documentId, doc.processingStartedAt, requestId) + const failed = (outcome: { error?: string; errorCode?: OrchestrationErrorCode }) => + NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to update document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) - return NextResponse.json({ - success: true, - data: { - documentId, - status: 'failed', - message: 'Document marked as failed due to timeout', - }, - }) - } catch (error) { - if (error instanceof Error) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - throw error - } - } else if (validatedData.retryProcessing) { - const doc = accessCheck.document + if (markFailedDueToTimeout) { + const outcome = await performMarkKnowledgeDocumentTimedOut({ + document: doc, + requestId, + }) + if (!outcome.success) return failed(outcome) - if (doc.processingStatus !== 'failed') { - return NextResponse.json({ error: 'Document is not in failed state' }, { status: 400 }) - } + return NextResponse.json({ + success: true, + data: { documentId, status: outcome.status, message: outcome.message }, + }) + } - const docData = { - filename: doc.filename, - fileUrl: doc.fileUrl, - fileSize: doc.fileSize, - mimeType: doc.mimeType, - } - const workspaceId = accessCheck.knowledgeBase?.workspaceId + if (retryProcessing) { const billingAttribution = workspaceId ? auth.authType === AuthType.INTERNAL_JWT ? requireBillingAttributionHeader(req.headers, { @@ -172,56 +147,38 @@ export const PUT = withRouteHandler( }) : undefined - const result = await retryDocumentProcessing( + const outcome = await performRetryKnowledgeDocumentProcessing({ knowledgeBaseId, - documentId, - docData, + document: doc, + billingAttribution, requestId, - billingAttribution - ) - - return NextResponse.json({ - success: true, - data: { - documentId, - status: result.status, - message: result.message, - }, - }) - } else { - const updatedDocument = await updateDocument(documentId, validatedData, requestId) - - logger.info( - `[${requestId}] Document updated: ${documentId} in knowledge base ${knowledgeBaseId}` - ) - - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.DOCUMENT_UPDATED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: documentId, - resourceName: validatedData.filename ?? accessCheck.document?.filename, - description: `Updated document "${validatedData.filename ?? accessCheck.document?.filename}" in knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileName: validatedData.filename ?? accessCheck.document?.filename, - updatedFields: Object.keys(validatedData).filter( - (k) => validatedData[k as keyof typeof validatedData] !== undefined - ), - ...(validatedData.enabled !== undefined && { enabled: validatedData.enabled }), - }, - request: req, }) + if (!outcome.success) return failed(outcome) return NextResponse.json({ success: true, - data: updatedDocument, + data: { documentId, status: outcome.status, message: outcome.message }, }) } + + const outcome = await performUpdateKnowledgeDocument({ + knowledgeBase: { + id: knowledgeBaseId, + name: accessCheck.knowledgeBase?.name, + workspaceId, + }, + document: { id: documentId, filename: doc.filename }, + updates: documentUpdates, + userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request: req, + }) + if (!outcome.success) return failed(outcome) + + return NextResponse.json({ success: true, data: outcome.document }) } catch (error) { logger.error(`[${requestId}] Error updating document ${documentId}`, error) return NextResponse.json({ error: 'Failed to update document' }, { status: 500 }) @@ -257,43 +214,30 @@ export const DELETE = withRouteHandler( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - const result = await deleteDocument(documentId, requestId) - - logger.info( - `[${requestId}] Document deleted: ${documentId} from knowledge base ${knowledgeBaseId}` - ) - - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, + const outcome = await performDeleteKnowledgeDocument({ + knowledgeBase: { + id: knowledgeBaseId, + name: accessCheck.knowledgeBase?.name, + workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, + }, + document: accessCheck.document, + userId, actorName: auth.userName, actorEmail: auth.userEmail, - action: AuditAction.DOCUMENT_DELETED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: documentId, - resourceName: accessCheck.document?.filename, - description: `Deleted document "${accessCheck.document?.filename}" from knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileName: accessCheck.document?.filename, - fileSize: accessCheck.document?.fileSize, - mimeType: accessCheck.document?.mimeType, - }, + source: 'ui', + requestId, request: req, }) - - const kbWorkspaceId = accessCheck.knowledgeBase?.workspaceId ?? '' - captureServerEvent( - userId, - 'knowledge_base_document_deleted', - { knowledge_base_id: knowledgeBaseId, workspace_id: kbWorkspaceId }, - kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : undefined - ) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to delete document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, - data: result, + data: { success: true, message: 'Document deleted successfully' }, }) } catch (error) { logger.error(`[${requestId}] Error deleting document`, error) diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/route.test.ts index 971c4a8f28b..84b523c7870 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.test.ts @@ -570,7 +570,9 @@ describe('Knowledge Base Documents API Route', () => { const data = await response.json() expect(response.status).toBe(500) - expect(data.error).toBe('Database error') + // An unclassified fault renders the route's own wording; the driver's + // message is logged, not returned. + expect(data.error).toBe('Failed to create document') }) }) }) diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.ts b/apps/sim/app/api/knowledge/[id]/documents/route.ts index 9025cea9891..a46d08abae4 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' import { getErrorMessage } from '@sim/utils/errors' @@ -19,19 +18,22 @@ import { requireBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { bulkDocumentOperation, bulkDocumentOperationByFilter, - createDocumentRecords, - createSingleDocument, getDocuments, getProcessingConfig, - KnowledgeBaseFileOwnershipError, - processDocumentsWithQueue, } from '@/lib/knowledge/documents/service' import type { TagFilterCondition } from '@/lib/knowledge/documents/tag-filter' -import { captureServerEvent } from '@/lib/posthog/server' +import { + performUploadKnowledgeDocument, + performUploadKnowledgeDocuments, +} from '@/lib/knowledge/orchestration' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' const logger = createLogger('DocumentsAPI') @@ -210,168 +212,77 @@ export const POST = withRouteHandler( ) } - if (body.bulk === true) { - const createdDocuments = await createDocumentRecords( - body.documents, - knowledgeBaseId, - requestId, - userId - ) - - logger.info( - `[${requestId}] Starting controlled async processing of ${createdDocuments.length} documents` - ) - - try { - const { PlatformEvents } = await import('@/lib/core/telemetry') - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId, - documentsCount: createdDocuments.length, - uploadType: 'bulk', - recipe: body.processingOptions?.recipe, - }) - } catch (_e) { - // Silently fail - } - - captureServerEvent( - userId, - 'knowledge_base_document_uploaded', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId ?? '', - document_count: createdDocuments.length, - upload_type: 'bulk', - }, - { - ...(kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : {}), - setOnce: { first_document_uploaded_at: new Date().toISOString() }, - } - ) - - processDocumentsWithQueue( - createdDocuments, - knowledgeBaseId, - body.processingOptions ?? {}, - requestId, - billingAttribution - ).catch((error: unknown) => { - logger.error(`[${requestId}] Critical error in document processing pipeline:`, error) - }) + const knowledgeBase = { + id: knowledgeBaseId, + name: accessCheck.knowledgeBase?.name, + workspaceId: kbWorkspaceId ?? null, + } + const actor = { + userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui' as const, + requestId, + request: req, + } - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: knowledgeBaseId, - resourceName: `${createdDocuments.length} document(s)`, - description: `Uploaded ${createdDocuments.length} document(s) to knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileCount: createdDocuments.length, - }, - request: req, + if (body.bulk === true) { + const outcome = await performUploadKnowledgeDocuments({ + ...actor, + knowledgeBase, + documents: body.documents, + processingOptions: body.processingOptions, + billingAttribution, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to create document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } + const { batchSize, maxConcurrentDocuments } = getProcessingConfig() return NextResponse.json({ success: true, data: { - total: createdDocuments.length, - documentsCreated: createdDocuments.map((doc) => ({ + total: outcome.documents.length, + documentsCreated: outcome.documents.map((doc) => ({ documentId: doc.documentId, filename: doc.filename, status: 'pending', })), processingMethod: 'background', processingConfig: { - maxConcurrentDocuments: getProcessingConfig().maxConcurrentDocuments, - batchSize: getProcessingConfig().batchSize, - totalBatches: Math.ceil(createdDocuments.length / getProcessingConfig().batchSize), + maxConcurrentDocuments, + batchSize, + totalBatches: Math.ceil(outcome.documents.length / batchSize), }, }, }) } const { bulk: _bulk, workflowId: _workflowId, ...singleDocumentData } = body - const newDocument = await createSingleDocument( - singleDocumentData, - knowledgeBaseId, - requestId, - userId - ) - - try { - const { PlatformEvents } = await import('@/lib/core/telemetry') - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId, - documentsCount: 1, - uploadType: 'single', - mimeType: singleDocumentData.mimeType, - fileSize: singleDocumentData.fileSize, - }) - } catch (_e) { - // Silently fail - } - - captureServerEvent( - userId, - 'knowledge_base_document_uploaded', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId ?? '', - document_count: 1, - upload_type: 'single', - }, - { - ...(kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : {}), - setOnce: { first_document_uploaded_at: new Date().toISOString() }, - } - ) - - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: knowledgeBaseId, - resourceName: singleDocumentData.filename, - description: `Uploaded document "${singleDocumentData.filename}" to knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileName: singleDocumentData.filename, - fileType: singleDocumentData.mimeType, - fileSize: singleDocumentData.fileSize, - }, - request: req, - }) - - return NextResponse.json({ - success: true, - data: newDocument, + // Indexing is deliberately not started here: this path only records the + // document, and its caller drives processing separately. + const outcome = await performUploadKnowledgeDocument({ + ...actor, + knowledgeBase, + document: singleDocumentData, + billingAttribution, }) - } catch (error) { - logger.error(`[${requestId}] Error creating document`, error) - - if (error instanceof KnowledgeBaseFileOwnershipError) { + if (!outcome.success) { return NextResponse.json( - { error: 'File URL does not reference a file owned by this knowledge base' }, - { status: 403 } + { error: messageForOrchestrationError(outcome, 'Failed to create document') }, + { status: statusForOrchestrationError(outcome.errorCode) } ) } - const errorMessage = getErrorMessage(error, 'Failed to create document') - const isStorageLimitError = - errorMessage.includes('Storage limit exceeded') || errorMessage.includes('storage limit') - const isMissingKnowledgeBase = errorMessage === 'Knowledge base not found' - + return NextResponse.json({ success: true, data: outcome.document }) + } catch (error) { + logger.error(`[${requestId}] Error creating document`, error) return NextResponse.json( - { error: errorMessage }, - { status: isMissingKnowledgeBase ? 404 : isStorageLimitError ? 413 : 500 } + { error: getErrorMessage(error, 'Failed to create document') }, + { status: 500 } ) } } diff --git a/apps/sim/app/api/knowledge/[id]/restore/route.ts b/apps/sim/app/api/knowledge/[id]/restore/route.ts index 5dee08582a6..a5ed8b85808 100644 --- a/apps/sim/app/api/knowledge/[id]/restore/route.ts +++ b/apps/sim/app/api/knowledge/[id]/restore/route.ts @@ -4,6 +4,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { restoreKnowledgeBaseContract } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { @@ -45,12 +49,17 @@ export const POST = withRouteHandler( const result = await performRestoreKnowledgeBase({ knowledgeBaseId: id, userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', requestId, + request, }) if (!result.success) { - const status = - result.errorCode === 'not_found' ? 404 : result.errorCode === 'conflict' ? 409 : 500 - return NextResponse.json({ error: result.error }, { status }) + return NextResponse.json( + { error: messageForOrchestrationError(result, 'Failed to restore knowledge base') }, + { status: statusForOrchestrationError(result.errorCode) } + ) } logger.info(`[${requestId}] Restored knowledge base ${id}`) diff --git a/apps/sim/app/api/knowledge/[id]/route.ts b/apps/sim/app/api/knowledge/[id]/route.ts index cd47c173ab4..3b91289af20 100644 --- a/apps/sim/app/api/knowledge/[id]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/route.ts @@ -1,20 +1,19 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { updateKnowledgeBaseContract } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { PlatformEvents } from '@/lib/core/telemetry' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { - deleteKnowledgeBase, - getKnowledgeBaseById, - KnowledgeBaseConflictError, - KnowledgeBaseFolderError, - KnowledgeBasePermissionError, - updateKnowledgeBase, -} from '@/lib/knowledge/service' + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration' +import { getKnowledgeBaseById } from '@/lib/knowledge/service' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' const logger = createLogger('KnowledgeBaseByIdAPI') @@ -69,93 +68,56 @@ export const PUT = withRouteHandler( const requestId = generateRequestId() const { id } = await context.params - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized knowledge base update attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId + const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + logger.warn(`[${requestId}] Unauthorized knowledge base update attempt`) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + const userId = auth.userId - const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) + const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${id}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to update unauthorized knowledge base ${id}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + if (!accessCheck.hasAccess) { + if ('notFound' in accessCheck && accessCheck.notFound) { + logger.warn(`[${requestId}] Knowledge base not found: ${id}`) + return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) } - - const parsed = await parseRequest(updateKnowledgeBaseContract, req, context) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - - const updatedKnowledgeBase = await updateKnowledgeBase( - id, - { - name: validatedData.name, - description: validatedData.description, - workspaceId: validatedData.workspaceId, - folderId: validatedData.folderId, - chunkingConfig: validatedData.chunkingConfig, - }, - requestId, - { actorUserId: userId } + logger.warn( + `[${requestId}] User ${userId} attempted to update unauthorized knowledge base ${id}` ) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - logger.info(`[${requestId}] Knowledge base updated: ${id} for user ${userId}`) - - recordAudit({ - workspaceId: accessCheck.knowledgeBase.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.KNOWLEDGE_BASE_UPDATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: validatedData.name ?? updatedKnowledgeBase.name, - description: `Updated knowledge base "${validatedData.name ?? updatedKnowledgeBase.name}"`, - metadata: { - updatedFields: Object.keys(validatedData).filter( - (k) => validatedData[k as keyof typeof validatedData] !== undefined - ), - ...(validatedData.name && { newName: validatedData.name }), - ...(validatedData.description !== undefined && { - description: validatedData.description, - }), - ...(validatedData.chunkingConfig && { - chunkMaxSize: validatedData.chunkingConfig.maxSize, - chunkMinSize: validatedData.chunkingConfig.minSize, - chunkOverlap: validatedData.chunkingConfig.overlap, - }), - }, - request: req, - }) - - return NextResponse.json({ - success: true, - data: updatedKnowledgeBase, - }) - } catch (error) { - if (error instanceof KnowledgeBaseConflictError) { - return NextResponse.json({ error: error.message }, { status: 409 }) - } - if (error instanceof KnowledgeBaseFolderError) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - if (error instanceof KnowledgeBasePermissionError) { - logger.warn(`[${requestId}] Forbidden knowledge base update on ${id}: ${error.message}`) - return NextResponse.json({ error: error.message }, { status: 403 }) - } - - logger.error(`[${requestId}] Error updating knowledge base`, error) - return NextResponse.json({ error: 'Failed to update knowledge base' }, { status: 500 }) + const parsed = await parseRequest(updateKnowledgeBaseContract, req, context) + if (!parsed.success) return parsed.response + + const body = parsed.data.body + + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: id, + workspaceId: accessCheck.knowledgeBase.workspaceId ?? null, + userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + updates: { + name: body.name, + description: body.description, + workspaceId: body.workspaceId, + folderId: body.folderId, + chunkingConfig: body.chunkingConfig, + }, + requestId, + request: req, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to update knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) } + + return NextResponse.json({ success: true, data: outcome.knowledgeBase }) } ) @@ -164,62 +126,49 @@ export const DELETE = withRouteHandler( const requestId = generateRequestId() const { id } = await params - try { - const auth = await checkSessionOrInternalAuth(_request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized knowledge base delete attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${id}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to delete unauthorized knowledge base ${id}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } + const auth = await checkSessionOrInternalAuth(_request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + logger.warn(`[${requestId}] Unauthorized knowledge base delete attempt`) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + const userId = auth.userId - await deleteKnowledgeBase(id, requestId) + const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) - try { - PlatformEvents.knowledgeBaseDeleted({ - knowledgeBaseId: id, - }) - } catch { - // Telemetry should not fail the operation + if (!accessCheck.hasAccess) { + if ('notFound' in accessCheck && accessCheck.notFound) { + logger.warn(`[${requestId}] Knowledge base not found: ${id}`) + return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) } + logger.warn( + `[${requestId}] User ${userId} attempted to delete unauthorized knowledge base ${id}` + ) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - logger.info(`[${requestId}] Knowledge base deleted: ${id} for user ${userId}`) - - recordAudit({ + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { + id, + name: accessCheck.knowledgeBase.name, workspaceId: accessCheck.knowledgeBase.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.KNOWLEDGE_BASE_DELETED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: accessCheck.knowledgeBase.name, - description: `Deleted knowledge base "${accessCheck.knowledgeBase.name || id}"`, - metadata: { - knowledgeBaseName: accessCheck.knowledgeBase.name, - }, - request: _request, - }) - - return NextResponse.json({ - success: true, - data: { message: 'Knowledge base deleted successfully' }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting knowledge base`, error) - return NextResponse.json({ error: 'Failed to delete knowledge base' }, { status: 500 }) + }, + userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request: _request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to delete knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) } + + return NextResponse.json({ + success: true, + data: { message: 'Knowledge base deleted successfully' }, + }) } ) diff --git a/apps/sim/app/api/knowledge/route.ts b/apps/sim/app/api/knowledge/route.ts index b2f9177b49d..09178f9dff1 100644 --- a/apps/sim/app/api/knowledge/route.ts +++ b/apps/sim/app/api/knowledge/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { @@ -7,19 +6,14 @@ import { } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { PlatformEvents } from '@/lib/core/telemetry' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' -import { - createKnowledgeBase, - getKnowledgeBases, - KnowledgeBaseConflictError, - KnowledgeBaseFolderError, - KnowledgeBasePermissionError, - type KnowledgeBaseScope, -} from '@/lib/knowledge/service' -import { captureServerEvent } from '@/lib/posthog/server' +import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' +import { getKnowledgeBases, type KnowledgeBaseScope } from '@/lib/knowledge/service' const logger = createLogger('KnowledgeBaseAPI') @@ -65,113 +59,49 @@ export const GET = withRouteHandler(async (req: NextRequest) => { export const POST = withRouteHandler(async (req: NextRequest) => { const requestId = generateRequestId() - try { - const session = await getSession() - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthorized knowledge base creation attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - createKnowledgeBaseContract, - req, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid knowledge base data`, { errors: error.issues }) - return NextResponse.json( - { error: 'Invalid request data', details: error.issues }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - - try { - const embeddingModel = getConfiguredEmbeddingModel() - - const createData = { - ...validatedData, - userId: session.user.id, - embeddingModel, - embeddingDimension: EMBEDDING_DIMENSIONS, - } - - const newKnowledgeBase = await createKnowledgeBase(createData, requestId) - - try { - PlatformEvents.knowledgeBaseCreated({ - knowledgeBaseId: newKnowledgeBase.id, - name: validatedData.name, - workspaceId: validatedData.workspaceId, - }) - } catch { - // Telemetry should not fail the operation - } - - captureServerEvent( - session.user.id, - 'knowledge_base_created', - { - knowledge_base_id: newKnowledgeBase.id, - workspace_id: validatedData.workspaceId, - name: validatedData.name, - }, - { - groups: { workspace: validatedData.workspaceId }, - setOnce: { first_kb_created_at: new Date().toISOString() }, - } - ) - - logger.info( - `[${requestId}] Knowledge base created: ${newKnowledgeBase.id} for user ${session.user.id}` - ) - - recordAudit({ - workspaceId: validatedData.workspaceId, - actorId: session.user.id, - actorName: session.user.name, - actorEmail: session.user.email, - action: AuditAction.KNOWLEDGE_BASE_CREATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: newKnowledgeBase.id, - resourceName: validatedData.name, - description: `Created knowledge base "${validatedData.name}"`, - metadata: { - name: validatedData.name, - description: validatedData.description, - embeddingModel, - embeddingDimension: EMBEDDING_DIMENSIONS, - chunkingStrategy: validatedData.chunkingConfig.strategy, - chunkMaxSize: validatedData.chunkingConfig.maxSize, - chunkMinSize: validatedData.chunkingConfig.minSize, - chunkOverlap: validatedData.chunkingConfig.overlap, - }, - request: req, - }) + const session = await getSession() + if (!session?.user?.id) { + logger.warn(`[${requestId}] Unauthorized knowledge base creation attempt`) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - return NextResponse.json({ - success: true, - data: newKnowledgeBase, - }) - } catch (createError) { - if (createError instanceof KnowledgeBaseConflictError) { - return NextResponse.json({ error: createError.message }, { status: 409 }) - } - if (createError instanceof KnowledgeBaseFolderError) { - return NextResponse.json({ error: createError.message }, { status: 400 }) - } - if (createError instanceof KnowledgeBasePermissionError) { - logger.warn(`[${requestId}] Forbidden knowledge base creation: ${createError.message}`) - return NextResponse.json({ error: createError.message }, { status: 403 }) - } - throw createError + const parsed = await parseRequest( + createKnowledgeBaseContract, + req, + {}, + { + validationErrorResponse: (error) => { + logger.warn(`[${requestId}] Invalid knowledge base data`, { errors: error.issues }) + return NextResponse.json( + { error: 'Invalid request data', details: error.issues }, + { status: 400 } + ) + }, } - } catch (error) { - logger.error(`[${requestId}] Error creating knowledge base`, error) - return NextResponse.json({ error: 'Failed to create knowledge base' }, { status: 500 }) + ) + if (!parsed.success) return parsed.response + + const body = parsed.data.body + + const outcome = await performCreateKnowledgeBase({ + userId: session.user.id, + actorName: session.user.name, + actorEmail: session.user.email, + source: 'ui', + workspaceId: body.workspaceId, + name: body.name, + description: body.description, + folderId: body.folderId, + chunkingConfig: body.chunkingConfig, + requestId, + request: req, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to create knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) } + + return NextResponse.json({ success: true, data: outcome.knowledgeBase }) }) diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts index 94c4832f265..33ac4611ffe 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' import { document, knowledgeConnector } from '@sim/db/schema' import { and, eq, isNull } from 'drizzle-orm' @@ -8,8 +7,12 @@ import { v1GetKnowledgeDocumentContract, } from '@/lib/api/contracts/v1/knowledge' import { parseRequest } from '@/lib/api/server' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteDocument } from '@/lib/knowledge/documents/service' +import { performDeleteKnowledgeDocument } from '@/lib/knowledge/orchestration' import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, v1ValidationErrorResponse } from '@/app/api/v1/middleware' @@ -152,19 +155,24 @@ export const DELETE = withRouteHandler( return NextResponse.json({ error: 'Document not found' }, { status: 404 }) } - await deleteDocument(documentId, requestId) - - recordAudit({ - workspaceId: parsed.data.query.workspaceId, - actorId: userId, - action: AuditAction.DOCUMENT_DELETED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: documentId, - resourceName: docs[0].filename, - description: `Deleted document "${docs[0].filename}" from knowledge base via API`, - metadata: { knowledgeBaseId }, + const outcome = await performDeleteKnowledgeDocument({ + knowledgeBase: { + id: knowledgeBaseId, + name: result.kb.name, + workspaceId: parsed.data.query.workspaceId, + }, + document: { id: documentId, filename: docs[0].filename }, + userId, + source: 'api', + requestId, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to delete document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts index 18898d704af..5cba10e6338 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts @@ -75,6 +75,9 @@ vi.mock('@/lib/uploads/contexts/workspace', () => ({ vi.mock('@/lib/uploads/utils/validation', () => ({ validateFileType: mockValidateFileType, + // Read at module scope by `lib/uploads/utils/file-utils`, which the route now + // reaches transitively through the knowledge orchestration module. + SUPPORTED_ARCHIVE_EXTENSIONS: [], })) vi.mock('@/lib/knowledge/documents/service', () => ({ diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts index dfd08d4c892..8f77bb467c6 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { type NextRequest, NextResponse } from 'next/server' import { v1ListKnowledgeDocumentsContract, @@ -10,19 +9,19 @@ import { resolveBillingAttribution, resolveSystemBillingAttribution, } from '@/lib/billing/core/billing-attribution' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { isPayloadSizeLimitError, MAX_MULTIPART_OVERHEAD_BYTES, readFormDataWithLimit, } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSingleDocument, - type DocumentData, - getDocuments, - processDocumentsWithQueue, -} from '@/lib/knowledge/documents/service' +import { getDocuments } from '@/lib/knowledge/documents/service' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' +import { performUploadKnowledgeDocument } from '@/lib/knowledge/orchestration' import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { validateFileType } from '@/lib/uploads/utils/validation' import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' @@ -189,47 +188,29 @@ export const POST = withRouteHandler( contentType ) - const newDocument = await createSingleDocument( - { + const outcome = await performUploadKnowledgeDocument({ + knowledgeBase: { id: knowledgeBaseId, name: result.kb.name, workspaceId }, + document: { filename: file.name, fileUrl: uploadedFile.url, fileSize: file.size, mimeType: contentType, }, - knowledgeBaseId, - requestId, - billingActorUserId - ) - - const documentData: DocumentData = { - documentId: newDocument.id, - filename: file.name, - fileUrl: uploadedFile.url, - fileSize: file.size, - mimeType: contentType, - } - - processDocumentsWithQueue( - [documentData], - knowledgeBaseId, - {}, + startProcessing: 'queue', + billingAttribution, + uploadedBy: billingActorUserId, + userId, + source: 'api', requestId, - billingAttribution - ).catch(() => { - // Processing errors are logged internally - }) - - recordAudit({ - workspaceId, - actorId: userId, - action: AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: newDocument.id, - resourceName: file.name, - description: `Uploaded document "${file.name}" to knowledge base via API`, - metadata: { knowledgeBaseId, fileSize: file.size, mimeType: contentType }, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to upload document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } + const newDocument = outcome.document return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/knowledge/[id]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/route.ts index 8dbb280559f..373d0951636 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { type NextRequest, NextResponse } from 'next/server' import { v1DeleteKnowledgeBaseContract, @@ -6,8 +5,15 @@ import { v1UpdateKnowledgeBaseContract, } from '@/lib/api/contracts/v1/knowledge' import { parseRequest } from '@/lib/api/server' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteKnowledgeBase, updateKnowledgeBase } from '@/lib/knowledge/service' +import { + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration' import { formatKnowledgeBase, handleError, @@ -67,33 +73,26 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle const result = await resolveKnowledgeBase(id, workspaceId, userId, rateLimit, 'write') if (result instanceof NextResponse) return result - const updates: { - name?: string - description?: string - chunkingConfig?: { maxSize: number; minSize: number; overlap: number } - } = {} - if (name !== undefined) updates.name = name - if (description !== undefined) updates.description = description - if (chunkingConfig !== undefined) updates.chunkingConfig = chunkingConfig - - const updatedKb = await updateKnowledgeBase(id, updates, requestId) - - recordAudit({ + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: id, workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_UPDATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: updatedKb.name, - description: `Updated knowledge base "${updatedKb.name}" via API`, - metadata: { updatedFields: Object.keys(updates) }, + userId, + source: 'api', + updates: { name, description, chunkingConfig }, + requestId, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to update knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, data: { - knowledgeBase: formatKnowledgeBase(updatedKb), + knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase), message: 'Knowledge base updated successfully', }, }) @@ -125,18 +124,23 @@ export const DELETE = withRouteHandler( ) if (result instanceof NextResponse) return result - await deleteKnowledgeBase(id, requestId) - - recordAudit({ - workspaceId: parsed.data.query.workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_DELETED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: result.kb.name, - description: `Deleted knowledge base "${result.kb.name}" via API`, + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { + id, + name: result.kb.name, + workspaceId: parsed.data.query.workspaceId, + }, + userId, + source: 'api', + requestId, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to delete knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/knowledge/route.ts b/apps/sim/app/api/v1/knowledge/route.ts index 5b608484025..cacb36ed482 100644 --- a/apps/sim/app/api/v1/knowledge/route.ts +++ b/apps/sim/app/api/v1/knowledge/route.ts @@ -1,13 +1,16 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { type NextRequest, NextResponse } from 'next/server' import { v1CreateKnowledgeBaseContract, v1ListKnowledgeBasesContract, } from '@/lib/api/contracts/v1/knowledge' import { parseRequest } from '@/lib/api/server' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' -import { createKnowledgeBase, getKnowledgeBases } from '@/lib/knowledge/service' +import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' +import { getKnowledgeBases } from '@/lib/knowledge/service' import { formatKnowledgeBase, handleError } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, @@ -76,35 +79,27 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId, 'write') if (accessError) return accessError - const kb = await createKnowledgeBase( - { - name, - description, - workspaceId, - userId, - embeddingModel: getConfiguredEmbeddingModel(), - embeddingDimension: EMBEDDING_DIMENSIONS, - chunkingConfig: chunkingConfig ?? { maxSize: 1024, minSize: 100, overlap: 200 }, - }, - requestId - ) - - recordAudit({ + const outcome = await performCreateKnowledgeBase({ + userId, + source: 'api', workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_CREATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: kb.id, - resourceName: kb.name, - description: `Created knowledge base "${kb.name}" via API`, - metadata: { chunkingConfig }, + name, + description, + chunkingConfig, + requestId, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to create knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, data: { - knowledgeBase: formatKnowledgeBase(kb), + knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase), message: 'Knowledge base created successfully', }, }) diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/[documentId]/route.ts index 41560fb7558..ef9318b1dc6 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/[documentId]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' import { document, knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' @@ -13,12 +12,18 @@ import { import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteDocument } from '@/lib/knowledge/documents/service' +import { performDeleteKnowledgeDocument } from '@/lib/knowledge/orchestration' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' import { checkRateLimit, type RateLimitResult } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' -import { v2Data, v2Error, v2RateLimitError, v2ValidationError } from '@/app/api/v2/lib/response' +import { + v2Data, + v2Error, + v2ErrorForOrchestration, + v2RateLimitError, + v2ValidationError, +} from '@/app/api/v2/lib/response' const logger = createLogger('V2KnowledgeDocumentDetailAPI') @@ -193,19 +198,21 @@ export const DELETE = withRouteHandler( const doc = docs[0] if (!doc) return v2Error('NOT_FOUND', 'Document not found') - await deleteDocument(documentId, requestId) - - recordAudit({ - workspaceId: parsed.data.query.workspaceId, - actorId: userId, - action: AuditAction.DOCUMENT_DELETED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: documentId, - resourceName: doc.filename, - description: `Deleted document "${doc.filename}" from knowledge base via API`, - metadata: { knowledgeBaseId }, + const outcome = await performDeleteKnowledgeDocument({ + knowledgeBase: { + id: knowledgeBaseId, + name: result.kb.name, + workspaceId: parsed.data.query.workspaceId, + }, + document: { id: documentId, filename: doc.filename }, + userId, + source: 'api', + requestId, request, }) + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) + } return v2Data({ id: documentId, deleted: true as const }, { rateLimit }) } catch (error) { diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts index 60103508d3d..1f513d5f2ed 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' @@ -20,13 +19,9 @@ import { readFormDataWithLimit, } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSingleDocument, - type DocumentData, - getDocuments, - processDocumentsWithQueue, -} from '@/lib/knowledge/documents/service' +import { getDocuments } from '@/lib/knowledge/documents/service' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' +import { performUploadKnowledgeDocument } from '@/lib/knowledge/orchestration' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { validateFileType } from '@/lib/uploads/utils/validation' @@ -39,6 +34,7 @@ import { v2CursorList, v2Data, v2Error, + v2ErrorForOrchestration, v2RateLimitError, v2ValidationError, } from '@/app/api/v2/lib/response' @@ -248,47 +244,26 @@ export const POST = withRouteHandler( contentType ) - const newDocument = await createSingleDocument( - { + const outcome = await performUploadKnowledgeDocument({ + knowledgeBase: { id: knowledgeBaseId, name: result.kb.name, workspaceId }, + document: { filename: file.name, fileUrl: uploadedFile.url, fileSize: file.size, mimeType: contentType, }, - knowledgeBaseId, - requestId, - billingAttribution.actorUserId - ) - - const documentData: DocumentData = { - documentId: newDocument.id, - filename: file.name, - fileUrl: uploadedFile.url, - fileSize: file.size, - mimeType: contentType, - } - - processDocumentsWithQueue( - [documentData], - knowledgeBaseId, - {}, + startProcessing: 'queue', + billingAttribution, + uploadedBy: billingAttribution.actorUserId, + userId, + source: 'api', requestId, - billingAttribution - ).catch(() => { - // Processing errors are logged internally by the queue. - }) - - recordAudit({ - workspaceId, - actorId: userId, - action: AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: newDocument.id, - resourceName: file.name, - description: `Uploaded document "${file.name}" to knowledge base via API`, - metadata: { knowledgeBaseId, fileSize: file.size, mimeType: contentType }, request, }) + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) + } + const newDocument = outcome.document const document: V2KnowledgeDocumentSummary = { id: newDocument.id, @@ -310,18 +285,6 @@ export const POST = withRouteHandler( return v2Error('PAYLOAD_TOO_LARGE', error.message) } - if (error instanceof Error) { - if ( - error.message.includes('Storage limit exceeded') || - error.message.includes('storage limit') - ) { - return v2Error('PAYLOAD_TOO_LARGE', 'Storage limit exceeded') - } - if (error.message.includes('already exists')) { - return v2Error('CONFLICT', 'Resource already exists') - } - } - logger.error(`[${requestId}] Error uploading document`, { error: getErrorMessage(error, 'Unknown error'), }) diff --git a/apps/sim/app/api/v2/knowledge/[id]/route.ts b/apps/sim/app/api/v2/knowledge/[id]/route.ts index e6ae3849bae..65364a2b808 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' @@ -7,15 +6,24 @@ import { v2GetKnowledgeBaseContract, v2UpdateKnowledgeBaseContract, } from '@/lib/api/contracts/v2/knowledge' -import { isZodError, parseRequest } from '@/lib/api/server' +import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteKnowledgeBase, updateKnowledgeBase } from '@/lib/knowledge/service' +import { + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { formatKnowledgeBase, resolveKnowledgeBase } from '@/app/api/v1/knowledge/utils' import { checkRateLimit, type RateLimitResult } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' -import { v2Data, v2Error, v2RateLimitError, v2ValidationError } from '@/app/api/v2/lib/response' +import { + v2Data, + v2Error, + v2ErrorForOrchestration, + v2RateLimitError, + v2ValidationError, +} from '@/app/api/v2/lib/response' const logger = createLogger('V2KnowledgeDetailAPI') @@ -110,42 +118,21 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle const result = await resolveKnowledgeBaseScoped(id, workspaceId, userId, rateLimit, 'write') if (result instanceof NextResponse) return result - const updates: { - name?: string - description?: string - chunkingConfig?: { maxSize: number; minSize: number; overlap: number } - } = {} - if (name !== undefined) updates.name = name - if (description !== undefined) updates.description = description - if (chunkingConfig !== undefined) updates.chunkingConfig = chunkingConfig - - const updatedKb = await updateKnowledgeBase(id, updates, requestId) - - recordAudit({ + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: id, workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_UPDATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: updatedKb.name, - description: `Updated knowledge base "${updatedKb.name}" via API`, - metadata: { updatedFields: Object.keys(updates) }, + userId, + source: 'api', + updates: { name, description, chunkingConfig }, + requestId, request, }) - - return v2Data({ knowledgeBase: formatKnowledgeBase(updatedKb) }, { rateLimit }) - } catch (error) { - if (isZodError(error)) return v2ValidationError(error) - - if (error instanceof Error) { - if (error.message.includes('does not have permission')) { - return v2Error('FORBIDDEN', 'Access denied') - } - if (error.message.includes('already exists')) { - return v2Error('CONFLICT', 'Resource already exists') - } + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) } + return v2Data({ knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase) }, { rateLimit }) + } catch (error) { logger.error(`[${requestId}] Error updating knowledge base`, { error: getErrorMessage(error, 'Unknown error'), }) @@ -182,18 +169,16 @@ export const DELETE = withRouteHandler( ) if (result instanceof NextResponse) return result - await deleteKnowledgeBase(id, requestId) - - recordAudit({ - workspaceId: parsed.data.query.workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_DELETED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: result.kb.name, - description: `Deleted knowledge base "${result.kb.name}" via API`, + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { id, name: result.kb.name, workspaceId: parsed.data.query.workspaceId }, + userId, + source: 'api', + requestId, request, }) + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) + } return v2Data({ id, deleted: true as const }, { rateLimit }) } catch (error) { diff --git a/apps/sim/app/api/v2/knowledge/route.ts b/apps/sim/app/api/v2/knowledge/route.ts index 65aae6b8d5a..64e6445687b 100644 --- a/apps/sim/app/api/v2/knowledge/route.ts +++ b/apps/sim/app/api/v2/knowledge/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { NextRequest } from 'next/server' @@ -6,11 +5,11 @@ import { v2CreateKnowledgeBaseContract, v2ListKnowledgeBasesContract, } from '@/lib/api/contracts/v2/knowledge' -import { isZodError, parseRequest } from '@/lib/api/server' +import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' -import { createKnowledgeBase, getKnowledgeBases } from '@/lib/knowledge/service' +import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' +import { getKnowledgeBases } from '@/lib/knowledge/service' import { formatKnowledgeBase } from '@/app/api/v1/knowledge/utils' import { checkRateLimit, resolveWorkspaceAccess } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' @@ -18,6 +17,7 @@ import { v2CursorList, v2Data, v2Error, + v2ErrorForOrchestration, v2RateLimitError, v2ValidationError, v2WorkspaceAccessError, @@ -97,50 +97,25 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const access = await resolveWorkspaceAccess(rateLimit, userId, workspaceId, 'write') if (access) return v2WorkspaceAccessError(access) - const kb = await createKnowledgeBase( - { - name, - description, - workspaceId, - userId, - embeddingModel: getConfiguredEmbeddingModel(), - embeddingDimension: EMBEDDING_DIMENSIONS, - chunkingConfig: chunkingConfig ?? { maxSize: 1024, minSize: 100, overlap: 200 }, - }, - requestId - ) - - recordAudit({ + const outcome = await performCreateKnowledgeBase({ + userId, + source: 'api', workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_CREATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: kb.id, - resourceName: kb.name, - description: `Created knowledge base "${kb.name}" via API`, - metadata: { chunkingConfig }, + name, + description, + chunkingConfig, + requestId, request, }) - - return v2Data({ knowledgeBase: formatKnowledgeBase(kb) }, { rateLimit, status: 201 }) - } catch (error) { - if (isZodError(error)) return v2ValidationError(error) - - if (error instanceof Error) { - if (error.message.includes('does not have permission')) { - return v2Error('FORBIDDEN', 'Access denied') - } - if ( - error.message.includes('Storage limit exceeded') || - error.message.includes('storage limit') - ) { - return v2Error('PAYLOAD_TOO_LARGE', 'Storage limit exceeded') - } - if (error.message.includes('already exists')) { - return v2Error('CONFLICT', 'Resource already exists') - } + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) } + return v2Data( + { knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase) }, + { rateLimit, status: 201 } + ) + } catch (error) { logger.error(`[${requestId}] Error creating knowledge base`, { error: getErrorMessage(error, 'Unknown error'), }) diff --git a/apps/sim/app/api/v2/lib/response.ts b/apps/sim/app/api/v2/lib/response.ts index 45ed1e6fcb3..3bdc2b90b91 100644 --- a/apps/sim/app/api/v2/lib/response.ts +++ b/apps/sim/app/api/v2/lib/response.ts @@ -159,10 +159,12 @@ export function decodeCursor>(cursor: string): T | n const V2_CODE_BY_ORCHESTRATION_ERROR: Record = { validation: 'BAD_REQUEST', + unauthorized: 'UNAUTHORIZED', forbidden: 'FORBIDDEN', not_found: 'NOT_FOUND', conflict: 'CONFLICT', locked: 'LOCKED', + payload_too_large: 'PAYLOAD_TOO_LARGE', internal: 'INTERNAL_ERROR', } diff --git a/apps/sim/lib/api/contracts/knowledge/base.ts b/apps/sim/lib/api/contracts/knowledge/base.ts index 701073e3a23..2c142049afc 100644 --- a/apps/sim/lib/api/contracts/knowledge/base.ts +++ b/apps/sim/lib/api/contracts/knowledge/base.ts @@ -7,7 +7,10 @@ import { } from '@/lib/api/contracts/knowledge/shared' import { defineRouteContract } from '@/lib/api/contracts/types' import type { StrategyOptions } from '@/lib/chunkers/types' -import { KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH } from '@/lib/knowledge/constants' +import { + DEFAULT_CHUNKING_CONFIG, + KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH, +} from '@/lib/knowledge/constants' export const knowledgeScopeSchema = z.enum(['active', 'archived', 'all']) export type KnowledgeScope = z.output @@ -67,11 +70,7 @@ export const createKnowledgeBaseBodySchema = z.object({ folderId: z.string().min(1, 'Folder ID cannot be empty').nullable().optional(), embeddingModel: z.literal('text-embedding-3-small').default('text-embedding-3-small'), embeddingDimension: z.literal(1536).default(1536), - chunkingConfig: chunkingConfigSchema.default({ - maxSize: 1024, - minSize: 100, - overlap: 200, - }), + chunkingConfig: chunkingConfigSchema.default(DEFAULT_CHUNKING_CONFIG), }) export const updateKnowledgeBaseBodySchema = createKnowledgeBaseBodySchema diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index ed147c741a1..cef3d8718d1 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -22,7 +22,8 @@ export const updateConnectorBodySchema = z.object({ }) export const deleteConnectorQuerySchema = z.object({ - deleteDocuments: z.boolean().optional(), + /** Also hard-delete the documents the connector produced; kept by default. */ + deleteDocuments: booleanQueryFlagSchema.optional().default(false), }) export const connectorDocumentsQuerySchema = z.object({ diff --git a/apps/sim/lib/api/contracts/v1/knowledge/index.ts b/apps/sim/lib/api/contracts/v1/knowledge/index.ts index 76abfcf16a0..926786090b4 100644 --- a/apps/sim/lib/api/contracts/v1/knowledge/index.ts +++ b/apps/sim/lib/api/contracts/v1/knowledge/index.ts @@ -7,7 +7,10 @@ import { } from '@/lib/api/contracts/knowledge/shared' import { requiredFieldSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' -import { KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH } from '@/lib/knowledge/constants' +import { + DEFAULT_CHUNKING_CONFIG, + KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH, +} from '@/lib/knowledge/constants' /** * Public API v1 schemas (`/api/v1/knowledge/**`) @@ -25,9 +28,9 @@ import { KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH } from '@/lib/knowledge/constants /** Simpler chunking config used by the public API (no `strategy`). */ export const v1ChunkingConfigSchema = z.object({ - maxSize: z.number().min(100).max(4000).default(1024), - minSize: z.number().min(1).max(2000).default(100), - overlap: z.number().min(0).max(500).default(200), + maxSize: z.number().min(100).max(4000).default(DEFAULT_CHUNKING_CONFIG.maxSize), + minSize: z.number().min(1).max(2000).default(DEFAULT_CHUNKING_CONFIG.minSize), + overlap: z.number().min(0).max(500).default(DEFAULT_CHUNKING_CONFIG.overlap), }) /** GET `/api/v1/knowledge` — list knowledge bases scoped to a workspace. */ @@ -46,11 +49,7 @@ export const v1CreateKnowledgeBaseBodySchema = z.object({ `Description must be ${KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH} characters or less` ) .optional(), - chunkingConfig: v1ChunkingConfigSchema.optional().default({ - maxSize: 1024, - minSize: 100, - overlap: 200, - }), + chunkingConfig: v1ChunkingConfigSchema.optional().default(DEFAULT_CHUNKING_CONFIG), }) /** GET/DELETE `/api/v1/knowledge/[id]` — workspace scope param. */ diff --git a/apps/sim/lib/billing/storage/index.ts b/apps/sim/lib/billing/storage/index.ts index 5496ad64a5d..59e829499df 100644 --- a/apps/sim/lib/billing/storage/index.ts +++ b/apps/sim/lib/billing/storage/index.ts @@ -6,6 +6,7 @@ export { getStorageUsageForBillingContext, getUserStorageLimit, getUserStorageUsage, + StorageLimitExceededError, } from './limits' export { applyStorageUsageDeltasInTx, diff --git a/apps/sim/lib/billing/storage/limits.ts b/apps/sim/lib/billing/storage/limits.ts index 34b01d97b07..96d27a82265 100644 --- a/apps/sim/lib/billing/storage/limits.ts +++ b/apps/sim/lib/billing/storage/limits.ts @@ -21,9 +21,24 @@ import type { StorageBillingContext } from '@/lib/billing/storage/context' import { getLegacyStorageBillingEntity } from '@/lib/billing/storage/entity' import { getEnv } from '@/lib/core/config/env' import { isBillingEnabled } from '@/lib/core/config/env-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' const logger = createLogger('StorageLimits') +/** + * Thrown when accepting a write would push its payer past its storage quota. + * + * An {@link OrchestrationError} so every surface reaches 413 by class. The bare + * `Error` this replaced was classified by searching the message for "storage + * limit", which the UI, v1, and v2 knowledge routes each re-implemented. + */ +export class StorageLimitExceededError extends OrchestrationError { + constructor(message: string) { + super('payload_too_large', message) + this.name = 'StorageLimitExceededError' + } +} + type StorageLimits = ReturnType interface StorageLimitResolutionInput { diff --git a/apps/sim/lib/billing/storage/tracking.ts b/apps/sim/lib/billing/storage/tracking.ts index 1fab48fa652..0de9b03507b 100644 --- a/apps/sim/lib/billing/storage/tracking.ts +++ b/apps/sim/lib/billing/storage/tracking.ts @@ -18,6 +18,7 @@ import { getUserStorageLimit, getUserStorageUsage, isStorageEnforcementEnabled, + StorageLimitExceededError, } from '@/lib/billing/storage/limits' import { getFreeTierLimit, isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils' import type { DbOrTx } from '@/lib/db/types' @@ -362,7 +363,7 @@ export async function applyStorageUsageDeltasInTx( payerDelta.maximumUsage !== undefined && nextUsage > payerDelta.maximumUsage ) { - throw new Error( + throw new StorageLimitExceededError( `Storage limit exceeded. Used: ${(nextUsage / 1024 ** 3).toFixed(2)}GB, Limit: ${(payerDelta.maximumUsage / 1024 ** 3).toFixed(0)}GB` ) } @@ -471,7 +472,7 @@ async function mutateWorkspaceStorageUsage( currentPayerUsage + bytes > maximumUsage ) { const newUsage = currentPayerUsage + bytes - throw new Error( + throw new StorageLimitExceededError( `Storage limit exceeded. Used: ${(newUsage / 1024 ** 3).toFixed(2)}GB, Limit: ${(maximumUsage / 1024 ** 3).toFixed(0)}GB` ) } diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts index 186e785d29c..20980bc6a95 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts @@ -2,34 +2,33 @@ * @vitest-environment node */ import { knowledgeConnector } from '@sim/db/schema' -import { queueTableRows, resetDbChainMock, resetUrlsMock, urlsMockFns } from '@sim/testing' -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockAssertBillingAttributionSnapshot, mockCheckKnowledgeBaseWriteAccess, - mockFetch, - mockGenerateInternalToken, - mockSerializeBillingAttributionHeader, + mockGetKnowledgeBaseById, + mockPerformCreateKnowledgeConnector, + mockPerformDeleteKnowledgeBase, + mockPerformDeleteKnowledgeConnector, + mockPerformSyncKnowledgeConnector, } = vi.hoisted(() => ({ mockAssertBillingAttributionSnapshot: vi.fn(), mockCheckKnowledgeBaseWriteAccess: vi.fn(), - mockFetch: vi.fn(), - mockGenerateInternalToken: vi.fn(), - mockSerializeBillingAttributionHeader: vi.fn(), + mockGetKnowledgeBaseById: vi.fn(), + mockPerformCreateKnowledgeConnector: vi.fn(), + mockPerformDeleteKnowledgeBase: vi.fn(), + mockPerformDeleteKnowledgeConnector: vi.fn(), + mockPerformSyncKnowledgeConnector: vi.fn(), })) -vi.mock('@/lib/auth/internal', () => ({ - generateInternalToken: mockGenerateInternalToken, -})) vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ checkActorUsageLimits: vi.fn(), })) vi.mock('@/lib/billing/core/billing-attribution', () => ({ - BILLING_ATTRIBUTION_HEADER: 'x-sim-billing-attribution', assertBillingAttributionSnapshot: mockAssertBillingAttributionSnapshot, checkAttributedUsageLimits: vi.fn(), - serializeBillingAttributionHeader: mockSerializeBillingAttributionHeader, })) vi.mock('@/lib/copilot/generated/tool-catalog-v1', () => ({ KnowledgeBase: { id: 'knowledge_base' }, @@ -37,28 +36,24 @@ vi.mock('@/lib/copilot/generated/tool-catalog-v1', () => ({ vi.mock('@/lib/copilot/tools/server/base-tool', () => ({ assertServerToolNotAborted: vi.fn(), })) -beforeAll(() => { - urlsMockFns.mockGetInternalApiBaseUrl.mockReturnValue('http://internal.test') -}) - -afterAll(resetUrlsMock) -vi.mock('@/lib/knowledge/documents/service', () => ({ - createSingleDocument: vi.fn(), - deleteDocument: vi.fn(), - processDocumentAsync: vi.fn(), - updateDocument: vi.fn(), -})) vi.mock('@/lib/knowledge/embeddings', () => ({ - EMBEDDING_DIMENSIONS: 1536, generateSearchEmbedding: vi.fn(), - getConfiguredEmbeddingModel: vi.fn(), recordSearchEmbeddingUsage: vi.fn(), })) +vi.mock('@/lib/knowledge/orchestration', () => ({ + performCreateKnowledgeBase: vi.fn(), + performDeleteKnowledgeBase: mockPerformDeleteKnowledgeBase, + performCreateKnowledgeConnector: mockPerformCreateKnowledgeConnector, + performDeleteKnowledgeConnector: mockPerformDeleteKnowledgeConnector, + performDeleteKnowledgeDocument: vi.fn(), + performSyncKnowledgeConnector: mockPerformSyncKnowledgeConnector, + performUpdateKnowledgeBase: vi.fn(), + performUpdateKnowledgeConnector: vi.fn(), + performUpdateKnowledgeDocument: vi.fn(), + performUploadKnowledgeDocument: vi.fn(), +})) vi.mock('@/lib/knowledge/service', () => ({ - createKnowledgeBase: vi.fn(), - deleteKnowledgeBase: vi.fn(), - getKnowledgeBaseById: vi.fn(), - updateKnowledgeBase: vi.fn(), + getKnowledgeBaseById: mockGetKnowledgeBaseById, })) vi.mock('@/lib/knowledge/tags/service', () => ({ createTagDefinition: vi.fn(), @@ -73,6 +68,7 @@ vi.mock('@/lib/uploads', () => ({ StorageService: {} })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ resolveWorkspaceFileReference: vi.fn(), })) +vi.mock('@/app/api/auth/oauth/utils', () => ({ getCredential: vi.fn() })) vi.mock('@/app/api/knowledge/search/utils', () => ({ executeKnowledgeSearch: vi.fn(), })) @@ -97,6 +93,12 @@ const BILLING_ATTRIBUTION = { payerSubscription: null, } +const CONTEXT = { + userId: 'external-admin', + workspaceId: 'workspace-paid', + billingAttribution: BILLING_ATTRIBUTION, +} + describe('knowledge base connector Copilot operations', () => { afterAll(() => { resetDbChainMock() @@ -105,11 +107,8 @@ describe('knowledge base connector Copilot operations', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - vi.stubGlobal('fetch', mockFetch) queueTableRows(knowledgeConnector, [{ knowledgeBaseId: 'knowledge-base-1' }]) mockAssertBillingAttributionSnapshot.mockReturnValue(BILLING_ATTRIBUTION) - mockSerializeBillingAttributionHeader.mockReturnValue('serialized-attribution') - mockGenerateInternalToken.mockResolvedValue('internal-token') mockCheckKnowledgeBaseWriteAccess.mockResolvedValue({ hasAccess: true, knowledgeBase: { @@ -118,21 +117,21 @@ describe('knowledge base connector Copilot operations', () => { name: 'Paid KB', }, }) - mockFetch.mockResolvedValue({ - ok: true, - json: vi.fn().mockResolvedValue({ - success: true, - data: { - id: 'connector-1', - connectorType: 'notion', - status: 'active', - }, - }), + mockPerformCreateKnowledgeConnector.mockResolvedValue({ + success: true, + connector: { id: 'connector-1', connectorType: 'notion', status: 'active' }, + }) + mockPerformSyncKnowledgeConnector.mockResolvedValue({ success: true }) + mockPerformDeleteKnowledgeConnector.mockResolvedValue({ + success: true, + documentsDeleted: 0, + documentsKept: 3, }) }) it.each([ { + operation: 'add_connector', params: { operation: 'add_connector', args: { @@ -141,35 +140,87 @@ describe('knowledge base connector Copilot operations', () => { apiKey: 'api-key', }, }, - expectedPath: '/api/knowledge/knowledge-base-1/connectors', + perform: mockPerformCreateKnowledgeConnector, }, { - params: { - operation: 'sync_connector', - args: { connectorId: 'connector-1' }, - }, - expectedPath: '/api/knowledge/knowledge-base-1/connectors/connector-1/sync', + operation: 'sync_connector', + params: { operation: 'sync_connector', args: { connectorId: 'connector-1' } }, + perform: mockPerformSyncKnowledgeConnector, }, - ])( - 'forwards immutable billing attribution for $params.operation', - async ({ params, expectedPath }) => { - const result = await knowledgeBaseServerTool.execute(params, { - userId: 'external-admin', - workspaceId: 'workspace-paid', - billingAttribution: BILLING_ATTRIBUTION, - }) - - expect(result.success).toBe(true) - expect(mockFetch).toHaveBeenCalledWith( - `http://internal.test${expectedPath}`, - expect.objectContaining({ - headers: expect.objectContaining({ - Authorization: 'Bearer internal-token', - 'x-sim-billing-attribution': 'serialized-attribution', - }), - }) - ) - expect(mockSerializeBillingAttributionHeader).toHaveBeenCalledWith(BILLING_ATTRIBUTION) - } - ) + ])('forwards immutable billing attribution for $operation', async ({ params, perform }) => { + const result = await knowledgeBaseServerTool.execute(params, CONTEXT) + + expect(result.success).toBe(true) + // The operation runs in-process now. The payer travels as a value on the + // orchestration call rather than as a serialized header on an internal + // HTTP self-call back into this same process. + const call = perform.mock.calls[0][0] + expect(await call.resolveBillingAttribution()).toEqual(BILLING_ATTRIBUTION) + expect(call.source).toBe('agent') + expect(mockAssertBillingAttributionSnapshot).toHaveBeenCalledWith(BILLING_ATTRIBUTION) + }) + + it('reports a failed knowledge base delete as failed, not as missing', async () => { + mockGetKnowledgeBaseById.mockResolvedValue({ + id: 'knowledge-base-1', + name: 'Paid KB', + workspaceId: 'workspace-paid', + }) + mockPerformDeleteKnowledgeBase.mockResolvedValue({ + success: false, + error: 'Knowledge base is locked', + errorCode: 'conflict', + }) + + const result = await knowledgeBaseServerTool.execute( + { operation: 'delete', args: { knowledgeBaseId: 'knowledge-base-1' } }, + CONTEXT + ) + + // A knowledge base that exists but could not be archived is neither deleted + // nor missing — folding it into notFound told the user it was never there. + expect(result.data.notFound).toEqual([]) + expect(result.data.failed).toEqual([ + { id: 'knowledge-base-1', name: 'Paid KB', reason: 'Knowledge base is locked' }, + ]) + expect(result.message).toContain('Knowledge base is locked') + }) + + it('never relays an unclassified fault to the agent verbatim', async () => { + mockGetKnowledgeBaseById.mockResolvedValue({ + id: 'knowledge-base-1', + name: 'Paid KB', + workspaceId: 'workspace-paid', + }) + mockPerformDeleteKnowledgeBase.mockResolvedValue({ + success: false, + error: 'select "id" from "knowledge_base" — connection terminated', + errorCode: 'internal', + }) + + const result = await knowledgeBaseServerTool.execute( + { operation: 'delete', args: { knowledgeBaseId: 'knowledge-base-1' } }, + CONTEXT + ) + + expect(result.data.failed[0].reason).toBe('Failed to delete knowledge base') + expect(result.message).not.toContain('connection terminated') + }) + + it('reports that a deleted connector kept its documents, because it did', async () => { + const result = await knowledgeBaseServerTool.execute( + { operation: 'delete_connector', args: { connectorId: 'connector-1' } }, + CONTEXT + ) + + // The old wording claimed the documents "have been removed". They never + // were: the tool reached the route over HTTP with no query string, so the + // route's keep-documents default always applied. + expect(result.success).toBe(true) + expect(result.message).toContain('3 document(s) were kept') + expect(result.message).not.toContain('removed') + expect(mockPerformDeleteKnowledgeConnector).toHaveBeenCalledWith( + expect.objectContaining({ connectorId: 'connector-1', source: 'agent' }) + ) + }) }) diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index bf19100c9ec..a7f41a21768 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -1,18 +1,16 @@ import { db } from '@sim/db' import { knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { filterUndefined } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import { and, eq, isNull } from 'drizzle-orm' -import { generateInternalToken } from '@/lib/auth/internal' import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor' import { assertBillingAttributionSnapshot, - BILLING_ATTRIBUTION_HEADER, type BillingAttributionSnapshot, checkAttributedUsageLimits, - serializeBillingAttributionHeader, } from '@/lib/billing/core/billing-attribution' import { KnowledgeBase } from '@/lib/copilot/generated/tool-catalog-v1' import { @@ -20,25 +18,24 @@ import { type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' -import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import { - createSingleDocument, - deleteDocument, - processDocumentAsync, - updateDocument, -} from '@/lib/knowledge/documents/service' + messageForOrchestrationError, + type OrchestrationErrorCode, +} from '@/lib/core/orchestration/types' +import { generateSearchEmbedding, recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings' import { - EMBEDDING_DIMENSIONS, - generateSearchEmbedding, - getConfiguredEmbeddingModel, - recordSearchEmbeddingUsage, -} from '@/lib/knowledge/embeddings' -import { - createKnowledgeBase, - deleteKnowledgeBase, - getKnowledgeBaseById, - updateKnowledgeBase, -} from '@/lib/knowledge/service' + performCreateKnowledgeBase, + performCreateKnowledgeConnector, + performDeleteKnowledgeBase, + performDeleteKnowledgeConnector, + performDeleteKnowledgeDocument, + performSyncKnowledgeConnector, + performUpdateKnowledgeBase, + performUpdateKnowledgeConnector, + performUpdateKnowledgeDocument, + performUploadKnowledgeDocument, +} from '@/lib/knowledge/orchestration' +import { getKnowledgeBaseById } from '@/lib/knowledge/service' import { createTagDefinition, deleteTagDefinition, @@ -50,6 +47,7 @@ import { } from '@/lib/knowledge/tags/service' import { StorageService } from '@/lib/uploads' import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { getCredential } from '@/app/api/auth/oauth/utils' import { executeKnowledgeSearch } from '@/app/api/knowledge/search/utils' import { checkDocumentWriteAccess, @@ -73,6 +71,20 @@ function requireKnowledgeBillingAttribution( return attribution } +/** + * The message the agent — and therefore the user — is shown for a failed + * operation. Mirrors `messageForOrchestrationError` on the HTTP surfaces: a + * classified failure is caller-fixable and safe to relay, an unclassified one + * carries whatever text the fault happened to have (a driver's failed SQL, say) + * and is replaced by the operation's own wording. + */ +function agentFacingError( + outcome: { error?: string; errorCode?: OrchestrationErrorCode }, + fallback: string +): string { + return messageForOrchestrationError(outcome, fallback) +} + type KnowledgeBaseArgs = { operation: string args?: Record @@ -109,6 +121,17 @@ export const knowledgeBaseServerTool: BaseServerTool ({ + userId: context.userId as string, + source: 'agent' as const, + requestId, + }) try { switch (operation) { @@ -129,29 +152,21 @@ export const knowledgeBaseServerTool: BaseServerTool { - logger.error('Background document processing failed', { - documentId: doc.id, - error: toError(err).message, - }) + startProcessing: 'async', + billingAttribution, }) + if (!outcome.success) { + failedFiles.push(fileRef) + continue + } - added.push({ documentId: doc.id, filename: fileRecord.name }) - - logger.info('Workspace file added to knowledge base via copilot', { - knowledgeBaseId: args.knowledgeBaseId, - documentId: doc.id, - fileName: fileRecord.name, - userId: context.userId, - }) + added.push({ documentId: outcome.document.id, filename: fileRecord.name }) } const addedNames = added.map((a) => a.filename).join(', ') @@ -461,13 +460,20 @@ export const knowledgeBaseServerTool: BaseServerTool = [] const notFound: string[] = [] + // A knowledge base that exists but could not be archived is neither + // deleted nor missing. Folding it into `notFound` told the user it was + // never there instead of why the delete failed. + const failed: Array<{ id: string; name: string; reason: string }> = [] for (const kbId of kbIds) { const writeAccess = await checkKnowledgeBaseWriteAccess(kbId, context.userId) @@ -510,23 +520,42 @@ export const knowledgeBaseServerTool: BaseServerTool 0 ? `Deleted: ${deleted.map((d) => d.name).join(', ')}` : null, + failed.length > 0 + ? `Failed: ${failed.map((f) => `${f.name} (${f.reason})`).join(', ')}` + : null, + ] + .filter(Boolean) + .join('. ') + return { success: deleted.length > 0, - message: - deleted.length > 0 - ? `Deleted: ${deleted.map((d) => d.name).join(', ')}` - : 'No knowledge bases found', - data: { deleted, notFound }, + message: deleteSummary || 'No knowledge bases found', + data: { deleted, notFound, failed }, } } @@ -557,8 +586,16 @@ export const knowledgeBaseServerTool: BaseServerTool = { - connectorType: args.connectorType, - sourceConfig: args.sourceConfig ?? {}, - syncIntervalMinutes: args.syncIntervalMinutes ?? 1440, - } - - if (args.credentialId) { - createBody.credentialId = args.credentialId - } - if (args.apiKey) { - createBody.apiKey = args.apiKey - } - + const sourceConfig: Record = { ...(args.sourceConfig ?? {}) } if (args.disabledTagIds?.length) { - ;(createBody.sourceConfig as Record).disabledTagIds = - args.disabledTagIds + sourceConfig.disabledTagIds = args.disabledTagIds } + const requestId = generateId().slice(0, 8) assertNotAborted() - const createRes = await connectorApiCall( - context.userId, - `/api/knowledge/${args.knowledgeBaseId}/connectors`, - 'POST', - createBody, - billingAttribution - ) - - if (!createRes.success) { - return { success: false, message: createRes.error ?? 'Failed to create connector' } - } - - const connector = createRes.data - logger.info('Connector created via copilot', { - connectorId: connector.id, + const outcome = await performCreateKnowledgeConnector({ + ...actor(requestId), + knowledgeBase: { + id: args.knowledgeBaseId, + name: writeAccess.knowledgeBase.name, + workspaceId: connectorWorkspaceId, + }, connectorType: args.connectorType, - knowledgeBaseId: args.knowledgeBaseId, - userId: context.userId, + credentialId: args.credentialId, + apiKey: args.apiKey, + sourceConfig, + syncIntervalMinutes: args.syncIntervalMinutes ?? 1440, + resolveBillingAttribution: async () => billingAttribution, + resolveAccessToken: async (credentialId) => + (await getCredential(requestId, credentialId, context.userId as string)) + ?.accessToken ?? null, }) + if (!outcome.success) { + return { success: false, message: agentFacingError(outcome, 'Failed to add connector') } + } + const connector = outcome.connector return { success: true, message: `Connector "${args.connectorType}" added to knowledge base. Initial sync started.`, data: { id: connector.id, - connectorType: connector.connectorType ?? connector.connector_type, + connectorType: connector.connectorType, status: connector.status, knowledgeBaseId: args.knowledgeBaseId, }, @@ -962,41 +1005,38 @@ export const knowledgeBaseServerTool: BaseServerTool = {} - if (args.sourceConfig !== undefined) updateBody.sourceConfig = args.sourceConfig - if (args.syncIntervalMinutes !== undefined) - updateBody.syncIntervalMinutes = args.syncIntervalMinutes - if (args.connectorStatus !== undefined) updateBody.status = args.connectorStatus - - if (Object.keys(updateBody).length === 0) { - return { - success: false, - message: - 'At least one of sourceConfig, syncIntervalMinutes, or connectorStatus is required', - } + const updates = { + sourceConfig: args.sourceConfig, + syncIntervalMinutes: args.syncIntervalMinutes, + status: args.connectorStatus, } + const requestId = generateId().slice(0, 8) assertNotAborted() - const updateRes = await connectorApiCall( - context.userId, - `/api/knowledge/${kbId}/connectors/${args.connectorId}`, - 'PATCH', - updateBody - ) - - if (!updateRes.success) { - return { success: false, message: updateRes.error ?? 'Failed to update connector' } - } - - logger.info('Connector updated via copilot', { + // No `validateSourceConfig`: the agent has no requesting identity to + // resolve the connector's OAuth token with, so a replacement config is + // stored unvalidated and the next sync reports any problem with it. + const outcome = await performUpdateKnowledgeConnector({ + ...actor(requestId), + knowledgeBase: { + id: kbId, + name: writeAccess.knowledgeBase.name, + workspaceId: writeAccess.knowledgeBase.workspaceId ?? null, + }, connectorId: args.connectorId, - userId: context.userId, + updates, }) + if (!outcome.success) { + return { + success: false, + message: agentFacingError(outcome, 'Failed to update connector'), + } + } return { success: true, message: 'Connector updated successfully', - data: { id: args.connectorId, ...updateBody }, + data: { id: args.connectorId, ...filterUndefined(updates) }, } } @@ -1015,26 +1055,39 @@ export const knowledgeBaseServerTool: BaseServerTool 0 + ? `Connector deleted successfully. Its ${outcome.documentsKept} document(s) were kept in the knowledge base.` + : 'Connector deleted successfully.', + data: { + id: args.connectorId, + documentsKept: outcome.documentsKept, + documentsDeleted: outcome.documentsDeleted, + }, } } @@ -1064,23 +1117,24 @@ export const knowledgeBaseServerTool: BaseServerTool billingAttribution, }) + if (!outcome.success) { + return { + success: false, + message: agentFacingError(outcome, 'Failed to sync connector'), + } + } return { success: true, @@ -1111,42 +1165,6 @@ export const knowledgeBaseServerTool: BaseServerTool, - billingAttribution?: BillingAttributionSnapshot -): Promise<{ success: boolean; data?: any; error?: string }> { - const token = await generateInternalToken(userId) - const baseUrl = getInternalApiBaseUrl() - - const res = await fetch(`${baseUrl}${path}`, { - method, - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${token}`, - ...(billingAttribution - ? { - [BILLING_ATTRIBUTION_HEADER]: serializeBillingAttributionHeader(billingAttribution), - } - : {}), - }, - ...(body ? { body: JSON.stringify(body) } : {}), - }) - - const json = await res.json().catch(() => ({})) - - if (!res.ok) { - return { - success: false, - error: json.error || `API returned ${res.status}`, - } - } - - return { success: true, data: json.data } -} - async function resolveKnowledgeBaseId(connectorId: string): Promise { const rows = await db .select({ knowledgeBaseId: knowledgeConnector.knowledgeBaseId }) diff --git a/apps/sim/lib/core/orchestration/types.ts b/apps/sim/lib/core/orchestration/types.ts index 59ccf54aad6..eb42af6bf65 100644 --- a/apps/sim/lib/core/orchestration/types.ts +++ b/apps/sim/lib/core/orchestration/types.ts @@ -1,9 +1,16 @@ export type OrchestrationErrorCode = | 'validation' + /** + * The credentials this operation depends on are no longer usable — a stored + * third-party token that will not refresh, not an unauthenticated caller. + * Distinct from `forbidden`, which is the caller lacking permission. + */ + | 'unauthorized' | 'not_found' | 'forbidden' | 'conflict' | 'locked' + | 'payload_too_large' | 'internal' /** @@ -13,13 +20,32 @@ export type OrchestrationErrorCode = */ export function statusForOrchestrationError(code: OrchestrationErrorCode | undefined): number { if (code === 'validation') return 400 + if (code === 'unauthorized') return 401 if (code === 'forbidden') return 403 if (code === 'not_found') return 404 if (code === 'conflict') return 409 if (code === 'locked') return 423 + if (code === 'payload_too_large') return 413 return 500 } +/** + * The message a JSON route should render for an orchestration failure. + * + * A classified failure is caller-fixable, so its message is written for the + * caller and is safe to return. An unclassified one carries whatever text the + * fault happened to have — a driver's failed SQL, say — so the caller gets the + * route's own generic wording instead. `v2ErrorForOrchestration` applies the + * same rule for the v2 envelope. + */ +export function messageForOrchestrationError( + result: { error?: string; errorCode?: OrchestrationErrorCode }, + fallback: string +): string { + if (!result.errorCode || result.errorCode === 'internal') return fallback + return result.error ?? fallback +} + /** * A domain failure that already knows its own class. * diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index 84b9d8c9830..57ee50321be 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -1,6 +1,20 @@ /** Max character length for a knowledge base description, enforced at every layer (UI, internal API, v1 API). */ export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000 +/** + * Chunking a knowledge base gets when its creator names no configuration. + * + * Applied in `lib/knowledge/orchestration` so the UI, the v1 and v2 APIs, and + * the copilot agent all index identical input identically. Previously each + * caller carried its own literal and the agent's `minSize` was 1, so the same + * document chunked differently depending on who uploaded it. + */ +export const DEFAULT_CHUNKING_CONFIG = { + maxSize: 1024, + minSize: 100, + overlap: 200, +} as const + export const TAG_SLOT_CONFIG = { text: { slots: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6', 'tag7'] as const, diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 1b8651a8424..9ca64bb6d4b 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -32,6 +32,7 @@ import { maybeNotifyStorageLimitForBillingContext, resolveStorageBillingContext, type StorageBillingContext, + StorageLimitExceededError, } from '@/lib/billing/storage' import { checkAndBillOverageThreshold, @@ -41,6 +42,7 @@ import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' import { env, envNumber } from '@/lib/core/config/env' import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/env-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { processDocument } from '@/lib/knowledge/documents/document-processor' import { @@ -85,9 +87,9 @@ const logger = createLogger('DocumentService') * storage object that is not owned by the target knowledge base's workspace. * Routes map this to a 403. */ -export class KnowledgeBaseFileOwnershipError extends Error { +export class KnowledgeBaseFileOwnershipError extends OrchestrationError { constructor(public readonly storageKey: string) { - super('Document file is not owned by this knowledge base') + super('forbidden', 'Document file is not owned by this knowledge base') this.name = 'KnowledgeBaseFileOwnershipError' } } @@ -1052,7 +1054,7 @@ async function resolveDocumentStorageAdmission( .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .limit(1) if (!kb) { - throw new Error('Knowledge base not found') + throw new OrchestrationError('not_found', 'Knowledge base not found') } if (bytes <= 0) { @@ -1064,7 +1066,7 @@ async function resolveDocumentStorageAdmission( const context = await resolveStorageBillingContext(kb.workspaceId) const quotaCheck = await checkStorageQuotaForBillingContext(context, bytes) if (!quotaCheck.allowed) { - throw new Error(quotaCheck.error || 'Storage limit exceeded') + throw new StorageLimitExceededError(quotaCheck.error || 'Storage limit exceeded') } return { workspaceId: kb.workspaceId, @@ -1078,7 +1080,7 @@ async function resolveDocumentStorageAdmission( getHighestPrioritySubscription(billedUserId), ]) if (!quotaCheck.allowed) { - throw new Error(quotaCheck.error || 'Storage limit exceeded') + throw new StorageLimitExceededError(quotaCheck.error || 'Storage limit exceeded') } return { workspaceId: null, @@ -1126,7 +1128,7 @@ export async function createDocumentRecords( .limit(1) if (kb.length === 0) { - throw new Error('Knowledge base not found') + throw new OrchestrationError('not_found', 'Knowledge base not found') } if ( @@ -1176,7 +1178,7 @@ export async function createDocumentRecords( preparedBilling.bytes ) if (!quotaCheck.allowed) { - throw new Error(quotaCheck.error || 'Storage limit exceeded') + throw new StorageLimitExceededError(quotaCheck.error || 'Storage limit exceeded') } } } @@ -1618,7 +1620,7 @@ export async function createSingleDocument( .limit(1) if (kb.length === 0) { - throw new Error('Knowledge base not found') + throw new OrchestrationError('not_found', 'Knowledge base not found') } if ( @@ -1665,7 +1667,7 @@ export async function createSingleDocument( preparedBilling.bytes ) if (!quotaCheck.allowed) { - throw new Error(quotaCheck.error || 'Storage limit exceeded') + throw new StorageLimitExceededError(quotaCheck.error || 'Storage limit exceeded') } } } diff --git a/apps/sim/lib/knowledge/folders.test.ts b/apps/sim/lib/knowledge/folders.test.ts index a81ebba83bf..009827d4a90 100644 --- a/apps/sim/lib/knowledge/folders.test.ts +++ b/apps/sim/lib/knowledge/folders.test.ts @@ -105,7 +105,7 @@ describe('createKnowledgeBase — folder assignment', () => { await expect( createKnowledgeBase({ ...CREATE_INPUT, folderId: 'f-1' }, 'req-1') - ).rejects.toMatchObject({ code: 'KNOWLEDGE_BASE_FORBIDDEN' }) + ).rejects.toMatchObject({ code: 'forbidden' }) expect(mockFindActiveFolder).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts new file mode 100644 index 00000000000..68f7a10f242 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -0,0 +1,298 @@ +/** + * @vitest-environment node + */ +import { document } from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCaptureServerEvent, + mockDispatchSync, + mockHasWorkspaceLiveSyncAccess, + mockRecordAudit, +} = vi.hoisted(() => ({ + mockCaptureServerEvent: vi.fn(), + mockDispatchSync: vi.fn(), + mockHasWorkspaceLiveSyncAccess: vi.fn(), + mockRecordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { + CONNECTOR_CREATED: 'connector.created', + CONNECTOR_UPDATED: 'connector.updated', + CONNECTOR_DELETED: 'connector.deleted', + CONNECTOR_SYNCED: 'connector.synced', + }, + AuditResourceType: { CONNECTOR: 'connector' }, + recordAudit: mockRecordAudit, +})) +vi.mock('@/lib/api-key/crypto', () => ({ encryptApiKey: vi.fn() })) +vi.mock('@/lib/billing/core/subscription', () => ({ + hasWorkspaceLiveSyncAccess: mockHasWorkspaceLiveSyncAccess, +})) +vi.mock('@/lib/knowledge/connectors/queue', () => ({ dispatchSync: mockDispatchSync })) +vi.mock('@/lib/knowledge/documents/service', () => ({ + deleteDocumentStorageFiles: vi.fn().mockResolvedValue(undefined), +})) +vi.mock('@/lib/knowledge/tags/service', () => ({ + cleanupUnusedTagDefinitions: vi.fn().mockResolvedValue(undefined), + createTagDefinition: vi.fn(), +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) + +import { + performDeleteKnowledgeConnector, + performSyncKnowledgeConnector, + performUpdateKnowledgeConnector, +} from '@/lib/knowledge/orchestration/connectors' + +const KB = { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' } +const ACTOR = { userId: 'user-1', source: 'agent' as const, requestId: 'req-1' } +const BILLING = { actorUserId: 'user-1', workspaceId: 'ws-1' } as never +const resolveBillingAttribution = vi.fn().mockResolvedValue(BILLING) + +describe('performDeleteKnowledgeConnector', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + afterAll(resetDbChainMock) + + it('reports the documents it kept, so the caller cannot claim otherwise', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + queueTableRows(document, [ + { id: 'doc-1', fileUrl: '/a.txt' }, + { id: 'doc-2', fileUrl: '/b.txt' }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-1' }]) + + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + }) + + // The default keeps the documents. The copilot tool used to assert they had + // been removed while taking exactly this path. + expect(outcome).toMatchObject({ success: true, documentsKept: 2, documentsDeleted: 0 }) + expect(dbChainMockFns.delete).not.toHaveBeenCalledWith(document) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: expect.objectContaining({ deleteDocuments: false, documentsKept: 2 }), + }) + ) + }) + + it('reports the documents it deleted when asked to delete them', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + queueTableRows(document, [{ id: 'doc-1', fileUrl: '/a.txt' }]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-1' }]) + + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + deleteDocuments: true, + }) + + expect(outcome).toMatchObject({ success: true, documentsDeleted: 1, documentsKept: 0 }) + }) + + it('reports a missing connector as not found', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'not_found' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) +}) + +describe('performUpdateKnowledgeConnector', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + afterAll(resetDbChainMock) + + it('rejects an update that names nothing before reading the connector', async () => { + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: {}, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('classifies a sub-hourly interval on an unentitled workspace as forbidden', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + mockHasWorkspaceLiveSyncAccess.mockResolvedValue(false) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { syncIntervalMinutes: 5 }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'forbidden' }) + expect(mockHasWorkspaceLiveSyncAccess).toHaveBeenCalledWith('ws-1') + }) + + it('leaves a caller-supplied validator to reject a bad source config', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { sourceConfig: { database: 'gone' } }, + validateSourceConfig: async () => ({ + message: 'Database not found', + errorCode: 'validation' as const, + }), + }) + + expect(outcome).toMatchObject({ + success: false, + errorCode: 'validation', + error: 'Database not found', + }) + }) + + it('preserves the failure class the validator chose', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + + // A stale stored credential kept the route's 401; collapsing every + // rejection to `validation` had flattened it (and the 409) into a 400. + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { sourceConfig: { database: 'x' } }, + validateSourceConfig: async () => ({ + message: 'Failed to refresh access token. Please reconnect your account.', + errorCode: 'unauthorized' as const, + }), + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'unauthorized' }) + }) + + it('clears the failure counters when a paused connector is resumed', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { status: 'active' }, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ consecutiveFailures: 0, lastSyncError: null }) + ) + }) +}) + +describe('performSyncKnowledgeConnector', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockDispatchSync.mockResolvedValue(undefined) + }) + + afterAll(resetDbChainMock) + + it('resolves the payer before writing the audit, not after', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + const rejects = vi.fn().mockRejectedValue(new Error('billing attribution header is malformed')) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + resolveBillingAttribution: rejects, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockDispatchSync).not.toHaveBeenCalled() + }) + + it('refuses to stack a sync on one already running', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'syncing' }, + ]) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + resolveBillingAttribution, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + // A rejected request never pays for the payer lookup. + expect(resolveBillingAttribution).not.toHaveBeenCalled() + expect(mockDispatchSync).not.toHaveBeenCalled() + }) + + it('dispatches and records who asked for it', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + resolveBillingAttribution, + rehydrate: true, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockDispatchSync).toHaveBeenCalledWith('conn-1', { + billingAttribution: BILLING, + requestId: 'req-1', + rehydrate: true, + }) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: 'user-1', + metadata: expect.objectContaining({ syncType: 'manual-rehydrate' }), + }) + ) + }) + + it('rejects a knowledge base with no workspace to bill', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: { ...KB, workspaceId: null }, + connectorId: 'conn-1', + resolveBillingAttribution, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts new file mode 100644 index 00000000000..01d18f5ccea --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -0,0 +1,735 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeBase, + knowledgeBaseTagDefinitions, + knowledgeConnector, +} from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import { encryptApiKey } from '@/lib/api-key/crypto' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' +import { OrchestrationError, type OrchestrationErrorCode } from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' +import { allocateTagSlots } from '@/lib/knowledge/constants' +import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service' +import { + auditActorFields, + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { cleanupUnusedTagDefinitions, createTagDefinition } from '@/lib/knowledge/tags/service' +import { captureServerEvent } from '@/lib/posthog/server' + +const logger = createLogger('KnowledgeConnectorOrchestration') + +/** + * The connector registry and the sync queue are loaded on demand rather than + * imported at module scope. Both pull in every connector's SDK and the whole + * sync engine, and this module is re-exported from the knowledge orchestration + * barrel — a static edge would drag that graph into the bundle of every route + * that merely creates a knowledge base or uploads a document. + */ +async function loadDispatchSync() { + return (await import('@/lib/knowledge/connectors/queue')).dispatchSync +} + +/** A connector row exactly as stored, including its encrypted API key. */ +export type KnowledgeConnectorRow = typeof knowledgeConnector.$inferSelect +type ConnectorRow = KnowledgeConnectorRow +/** The connector row as it reaches every caller: never carrying the stored API key. */ +export type ConnectorWithoutSecret = Omit + +/** A refused `sourceConfig`, with the failure class the caller wants surfaced. */ +export interface SourceConfigRejection { + message: string + errorCode: OrchestrationErrorCode +} + +/** The knowledge base a connector operation targets, already authorized by the caller. */ +export interface ConnectorKnowledgeBase { + id: string + name: string + workspaceId: string | null +} + +function withoutSecret(row: ConnectorRow): ConnectorWithoutSecret { + const { encryptedApiKey: _encryptedApiKey, ...rest } = row + return rest +} + +/** + * Rejects a sub-hourly sync interval on a workspace without the plan for it. + * `0` disables scheduled syncs and is always allowed. + */ +async function assertLiveSyncAllowed( + workspaceId: string, + syncIntervalMinutes: number | undefined +): Promise { + if (syncIntervalMinutes === undefined || syncIntervalMinutes <= 0 || syncIntervalMinutes >= 60) { + return + } + if (!(await hasWorkspaceLiveSyncAccess(workspaceId))) { + throw new OrchestrationError('forbidden', 'Live sync requires a Max or Enterprise plan') + } +} + +export interface PerformCreateKnowledgeConnectorParams extends KnowledgeOperationContext { + knowledgeBase: ConnectorKnowledgeBase + connectorType: string + credentialId?: string + apiKey?: string + sourceConfig: Record + syncIntervalMinutes: number + /** + * Resolves the payer the sync is billed to. A thunk so a request rejected by + * a guard never pays for the lookup, and so the payer is read at the moment + * the sync is dispatched. + */ + resolveBillingAttribution: () => Promise + /** + * Resolves an OAuth credential to its access token. Supplied by the caller + * because credential lookup is scoped to the requesting identity. + */ + resolveAccessToken: (credentialId: string) => Promise +} + +export type PerformConnectorResult = KnowledgeOrchestrationResult<{ + connector: ConnectorWithoutSecret +}> + +/** + * Creates a connector on a knowledge base and dispatches its first sync. + * + * The tag-slot allocation and the connector insert share one transaction under + * the knowledge base's row lock, so a knowledge base archived mid-request can + * never end up with a live connector, and a partial slot allocation cannot + * outlive a failed insert. + */ +export async function performCreateKnowledgeConnector( + params: PerformCreateKnowledgeConnectorParams +): Promise { + const { + knowledgeBase: kb, + connectorType, + credentialId, + apiKey, + sourceConfig, + syncIntervalMinutes, + resolveBillingAttribution, + resolveAccessToken, + request, + source, + } = params + const requestId = params.requestId ?? generateRequestId() + + if (!kb.workspaceId) { + return fail('Knowledge base is missing workspace billing context', 'conflict') + } + const workspaceId = kb.workspaceId + + const { CONNECTOR_REGISTRY } = await import('@/connectors/registry.server') + const connectorConfig = CONNECTOR_REGISTRY[connectorType] + if (!connectorConfig) { + return fail(`Unknown connector type: ${connectorType}`, 'validation') + } + + try { + await assertLiveSyncAllowed(workspaceId, syncIntervalMinutes) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + + let resolvedCredentialId: string | null = null + let resolvedEncryptedApiKey: string | null = null + let accessToken: string + + if (connectorConfig.auth.mode === 'apiKey') { + if (!apiKey) { + return fail('API key is required', 'validation') + } + accessToken = apiKey + } else { + if (!credentialId) { + return fail('Credential is required', 'validation') + } + let token: string | null + try { + token = await resolveAccessToken(credentialId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + if (!token) { + return fail('Credential has no access token. Please reconnect your account.', 'validation') + } + accessToken = token + resolvedCredentialId = credentialId + } + + const configValidation = await connectorConfig.validateConfig(accessToken, sourceConfig) + if (!configValidation.valid) { + return fail(configValidation.error || 'Invalid source configuration', 'validation') + } + + if (connectorConfig.auth.mode === 'apiKey' && apiKey) { + resolvedEncryptedApiKey = (await encryptApiKey(apiKey)).encrypted + } + + let finalSourceConfig: Record = { ...sourceConfig } + const tagSlotMapping: Record = {} + let newTagSlots: Record = {} + + if (connectorConfig.tagDefinitions?.length) { + const disabledIds = new Set((sourceConfig.disabledTagIds as string[] | undefined) ?? []) + const enabledDefs = connectorConfig.tagDefinitions.filter((td) => !disabledIds.has(td.id)) + + const existingDefs = await db + .select({ + tagSlot: knowledgeBaseTagDefinitions.tagSlot, + displayName: knowledgeBaseTagDefinitions.displayName, + fieldType: knowledgeBaseTagDefinitions.fieldType, + }) + .from(knowledgeBaseTagDefinitions) + .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, kb.id)) + + const usedSlots = new Set(existingDefs.map((d) => d.tagSlot)) + const existingByName = new Map( + existingDefs.map((d) => [d.displayName, { tagSlot: d.tagSlot, fieldType: d.fieldType }]) + ) + + const defsNeedingSlots: typeof enabledDefs = [] + for (const td of enabledDefs) { + const existing = existingByName.get(td.displayName) + if (existing && existing.fieldType === td.fieldType) { + tagSlotMapping[td.id] = existing.tagSlot + } else { + defsNeedingSlots.push(td) + } + } + + const { mapping, skipped: skippedTags } = allocateTagSlots(defsNeedingSlots, usedSlots) + Object.assign(tagSlotMapping, mapping) + newTagSlots = mapping + + for (const name of skippedTags) { + logger.warn(`[${requestId}] No available slots for "${name}"`) + } + + if (skippedTags.length > 0 && Object.keys(tagSlotMapping).length === 0) { + return fail( + `No available tag slots. Could not assign: ${skippedTags.join(', ')}`, + 'validation' + ) + } + + finalSourceConfig = { ...finalSourceConfig, tagSlotMapping } + } + + // Resolved before the write, not after: every guard that can cheaply reject + // the request has already run, and `requireBillingAttributionHeader` throws on + // a malformed header. Resolving it post-commit would leave a live connector + // behind a 500 and let a retry create a duplicate. + let billingAttribution: BillingAttributionSnapshot + try { + billingAttribution = await resolveBillingAttribution() + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + + const now = new Date() + const connectorId = generateId() + const nextSyncAt = + syncIntervalMinutes > 0 ? new Date(now.getTime() + syncIntervalMinutes * 60 * 1000) : null + + let created: ConnectorRow + try { + created = await db.transaction(async (tx) => { + await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${kb.id} FOR UPDATE`) + + const activeKb = await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where(and(eq(knowledgeBase.id, kb.id), isNull(knowledgeBase.deletedAt))) + .limit(1) + + if (activeKb.length === 0) { + throw new OrchestrationError('not_found', 'Knowledge base not found') + } + + for (const [semanticId, slot] of Object.entries(newTagSlots)) { + const td = connectorConfig.tagDefinitions?.find((d) => d.id === semanticId) + if (!td) continue + await createTagDefinition( + { + knowledgeBaseId: kb.id, + tagSlot: slot, + displayName: td.displayName, + fieldType: td.fieldType, + }, + requestId, + tx + ) + } + + const [row] = await tx + .insert(knowledgeConnector) + .values({ + id: connectorId, + knowledgeBaseId: kb.id, + connectorType, + credentialId: resolvedCredentialId, + encryptedApiKey: resolvedEncryptedApiKey, + sourceConfig: finalSourceConfig, + syncIntervalMinutes, + status: 'active', + nextSyncAt, + createdAt: now, + updatedAt: now, + }) + .returning() + + return row + }) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + + logger.info(`[${requestId}] Created connector ${connectorId} for KB ${kb.id}`) + + captureServerEvent( + params.userId, + 'knowledge_base_connector_added', + { + knowledge_base_id: kb.id, + workspace_id: workspaceId, + connector_type: connectorType, + sync_interval_minutes: syncIntervalMinutes, + }, + { + groups: { workspace: workspaceId }, + setOnce: { first_connector_added_at: new Date().toISOString() }, + } + ) + + recordAudit({ + workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_CREATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: connectorType, + description: `Created ${connectorType} connector for knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType, + syncIntervalMinutes, + authMode: connectorConfig.auth.mode, + }, + ...(request ? { request } : {}), + }) + + const dispatchSync = await loadDispatchSync() + dispatchSync(connectorId, { billingAttribution, requestId }).catch((error) => { + logger.error( + `[${requestId}] Failed to dispatch initial sync for connector ${connectorId}`, + error + ) + }) + + return { success: true, connector: withoutSecret(created) } +} + +export interface PerformUpdateKnowledgeConnectorParams extends KnowledgeOperationContext { + knowledgeBase: ConnectorKnowledgeBase + connectorId: string + updates: { + sourceConfig?: Record + syncIntervalMinutes?: number + status?: 'active' | 'paused' + } + /** + * Validates a replacement `sourceConfig` against the live source. Supplied by + * the caller because resolving the connector's token needs the requesting + * identity. Returning a rejection fails the update. + * + * The rejection carries its own `errorCode` so a stale credential and a bad + * config stay distinguishable — collapsing every rejection to `validation` + * flattened the route's 401 and 409 into a 400. + */ + validateSourceConfig?: ( + connector: KnowledgeConnectorRow, + sourceConfig: Record + ) => Promise +} + +/** Loads an active connector scoped to its knowledge base. */ +export async function getKnowledgeConnector( + knowledgeBaseId: string, + connectorId: string +): Promise { + const [row] = await db + .select() + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .limit(1) + + return row ?? null +} + +/** Applies a connector configuration change and records it against the actor. */ +export async function performUpdateKnowledgeConnector( + params: PerformUpdateKnowledgeConnectorParams +): Promise { + const { knowledgeBase: kb, connectorId, updates, validateSourceConfig, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + const updatedFields = Object.keys(updates).filter( + (key) => updates[key as keyof typeof updates] !== undefined + ) + if (updatedFields.length === 0) { + return fail( + 'At least one of sourceConfig, syncIntervalMinutes, or status is required', + 'validation' + ) + } + + const existing = await getKnowledgeConnector(kb.id, connectorId) + if (!existing) { + return fail('Connector not found', 'not_found') + } + + if (updates.syncIntervalMinutes !== undefined) { + if (!kb.workspaceId && updates.syncIntervalMinutes > 0 && updates.syncIntervalMinutes < 60) { + return fail('Knowledge base is missing workspace billing context', 'conflict') + } + if (kb.workspaceId) { + try { + await assertLiveSyncAllowed(kb.workspaceId, updates.syncIntervalMinutes) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Update connector ${connectorId}`) + } + } + } + + if (updates.sourceConfig !== undefined && validateSourceConfig) { + const rejection = await validateSourceConfig(existing, updates.sourceConfig) + if (rejection) { + return fail(rejection.message, rejection.errorCode) + } + } + + const values: Partial = { updatedAt: new Date() } + if (updates.sourceConfig !== undefined) { + values.sourceConfig = updates.sourceConfig + } + if (updates.syncIntervalMinutes !== undefined) { + values.syncIntervalMinutes = updates.syncIntervalMinutes + values.nextSyncAt = + updates.syncIntervalMinutes > 0 + ? new Date(Date.now() + updates.syncIntervalMinutes * 60 * 1000) + : null + } + if (updates.status !== undefined) { + values.status = updates.status + if (updates.status === 'active') { + values.consecutiveFailures = 0 + values.lastSyncError = null + // Resuming a paused connector syncs immediately unless this same request + // set a schedule, which then owns the next run. + if (values.nextSyncAt === undefined) { + values.nextSyncAt = new Date() + } + } + } + + let updated: ConnectorRow + try { + const [row] = await db + .update(knowledgeConnector) + .set(values) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, kb.id), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() + + if (!row) { + return fail('Connector not found', 'not_found') + } + updated = row + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Update connector ${connectorId}`) + } + + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_UPDATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: updated.connectorType, + description: `Updated connector for knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: updated.connectorType, + updatedFields, + ...(updates.syncIntervalMinutes !== undefined && { + syncIntervalMinutes: updates.syncIntervalMinutes, + }), + ...(updates.status !== undefined && { newStatus: updates.status }), + }, + ...(request ? { request } : {}), + }) + + return { success: true, connector: withoutSecret(updated) } +} + +export interface PerformDeleteKnowledgeConnectorParams extends KnowledgeOperationContext { + knowledgeBase: ConnectorKnowledgeBase + connectorId: string + /** + * Also hard-delete the documents the connector produced. Defaults to keeping + * them, which turns them into ordinary standalone knowledge base entries. + */ + deleteDocuments?: boolean +} + +/** What actually happened to the connector's documents, for the caller to report. */ +export type PerformDeleteKnowledgeConnectorResult = KnowledgeOrchestrationResult<{ + documentsDeleted: number + documentsKept: number +}> + +/** + * Hard-deletes a connector, either removing the documents it produced or + * releasing them as standalone entries. + * + * Returns the counts so callers state what happened rather than assert it. The + * copilot tool used to reach this through an internal HTTP self-call that sent + * no query string, so it always took the keep-documents default while telling + * the user the documents had been removed. + */ +export async function performDeleteKnowledgeConnector( + params: PerformDeleteKnowledgeConnectorParams +): Promise { + const { knowledgeBase: kb, connectorId, request, source } = params + const deleteDocuments = params.deleteDocuments ?? false + const requestId = params.requestId ?? generateRequestId() + + const existing = await getKnowledgeConnector(kb.id, connectorId) + if (!existing) { + return fail('Connector not found', 'not_found') + } + + let deletedDocs: Array<{ id: string; fileUrl: string }> + let docCount: number + try { + ;({ deletedDocs, docCount } = await db.transaction(async (tx) => { + await tx.execute(sql`SELECT 1 FROM knowledge_connector WHERE id = ${connectorId} FOR UPDATE`) + + // Includes pending-removal (tombstoned) docs — the connector is being + // deleted, so there's no future sync left to confirm or resurrect them. + const docs = await tx + .select({ id: document.id, fileUrl: document.fileUrl }) + .from(document) + .where(and(eq(document.connectorId, connectorId), isNull(document.archivedAt))) + + const documentIds = docs.map((doc) => doc.id) + if (deleteDocuments) { + if (documentIds.length > 0) { + await tx.delete(embedding).where(inArray(embedding.documentId, documentIds)) + await tx.delete(document).where(inArray(document.id, documentIds)) + } + } else if (documentIds.length > 0) { + // Kept documents become normal standalone KB entries once their connector + // is gone — resurrect any pending-removal ones rather than leaving them + // invisible tombstones with no future sync left to ever confirm or + // resurrect them. + await tx.update(document).set({ deletedAt: null }).where(inArray(document.id, documentIds)) + } + + const deletedConnectors = await tx + .delete(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, kb.id), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning({ id: knowledgeConnector.id }) + + if (deletedConnectors.length === 0) { + throw new OrchestrationError('not_found', 'Connector not found') + } + + return { deletedDocs: deleteDocuments ? docs : [], docCount: docs.length } + })) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Delete connector ${connectorId}`) + } + + if (deleteDocuments) { + await Promise.all([ + deletedDocs.length > 0 + ? deleteDocumentStorageFiles( + deletedDocs.map((doc) => ({ ...doc, workspaceId: kb.workspaceId })), + requestId + ) + : Promise.resolve(), + cleanupUnusedTagDefinitions(kb.id, requestId).catch((error) => { + logger.warn(`[${requestId}] Failed to cleanup tag definitions`, error) + }), + ]) + } + + logger.info( + `[${requestId}] Deleted connector ${connectorId}${deleteDocuments ? ` and ${docCount} documents` : `, kept ${docCount} documents`}` + ) + + captureServerEvent( + params.userId, + 'knowledge_base_connector_removed', + { + knowledge_base_id: kb.id, + workspace_id: kb.workspaceId ?? '', + connector_type: existing.connectorType, + documents_deleted: deleteDocuments ? docCount : 0, + }, + kb.workspaceId ? { groups: { workspace: kb.workspaceId } } : undefined + ) + + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_DELETED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: existing.connectorType, + description: `Deleted connector from knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: existing.connectorType, + deleteDocuments, + documentsDeleted: deleteDocuments ? docCount : 0, + documentsKept: deleteDocuments ? 0 : docCount, + }, + ...(request ? { request } : {}), + }) + + return { + success: true, + documentsDeleted: deleteDocuments ? docCount : 0, + documentsKept: deleteDocuments ? 0 : docCount, + } +} + +export interface PerformSyncKnowledgeConnectorParams extends KnowledgeOperationContext { + knowledgeBase: ConnectorKnowledgeBase + connectorId: string + /** + * Resolves the payer the sync is billed to. A thunk so a request rejected by + * a guard never pays for the lookup. + */ + resolveBillingAttribution: () => Promise + /** Re-fetch and re-index every already-synced document, not only changed ones. */ + rehydrate?: boolean +} + +export type PerformSyncKnowledgeConnectorResult = KnowledgeOrchestrationResult + +/** Triggers a manual sync for a connector and records who asked for it. */ +export async function performSyncKnowledgeConnector( + params: PerformSyncKnowledgeConnectorParams +): Promise { + const { knowledgeBase: kb, connectorId, resolveBillingAttribution, request, source } = params + const rehydrate = params.rehydrate ?? false + const requestId = params.requestId ?? generateRequestId() + + const connector = await getKnowledgeConnector(kb.id, connectorId) + if (!connector) { + return fail('Connector not found', 'not_found') + } + if (connector.status === 'syncing') { + return fail('Sync already in progress', 'conflict') + } + if (!kb.workspaceId) { + return fail('Knowledge base is missing workspace billing context', 'conflict') + } + // Resolved before the audit is written, so a rejected payer lookup returns a + // classified failure rather than escaping as a 500 with a sync already recorded. + let billingAttribution: BillingAttributionSnapshot + try { + billingAttribution = await resolveBillingAttribution() + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Sync connector ${connectorId}`) + } + + logger.info( + `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` + ) + + captureServerEvent( + params.userId, + 'knowledge_base_connector_synced', + { + knowledge_base_id: kb.id, + workspace_id: kb.workspaceId ?? '', + connector_type: connector.connectorType, + }, + kb.workspaceId ? { groups: { workspace: kb.workspaceId } } : undefined + ) + + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_SYNCED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: connector.connectorType, + description: `Triggered manual sync for connector on knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: connector.connectorType, + connectorStatus: connector.status, + syncType: rehydrate ? 'manual-rehydrate' : 'manual', + }, + ...(request ? { request } : {}), + }) + + const dispatchSync = await loadDispatchSync() + dispatchSync(connectorId, { billingAttribution, requestId, rehydrate }).catch((error) => { + logger.error( + `[${requestId}] Failed to dispatch manual sync for connector ${connectorId}`, + error + ) + }) + + return { success: true } +} diff --git a/apps/sim/lib/knowledge/orchestration/documents.test.ts b/apps/sim/lib/knowledge/orchestration/documents.test.ts new file mode 100644 index 00000000000..bfbe1bdc403 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/documents.test.ts @@ -0,0 +1,329 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCaptureServerEvent, + mockCreateDocumentRecords, + mockCreateSingleDocument, + mockDeleteDocument, + mockMarkDocumentAsFailedTimeout, + mockProcessDocumentAsync, + mockProcessDocumentsWithQueue, + mockRecordAudit, + mockRetryDocumentProcessing, + mockUpdateDocument, +} = vi.hoisted(() => ({ + mockCaptureServerEvent: vi.fn(), + mockCreateDocumentRecords: vi.fn(), + mockCreateSingleDocument: vi.fn(), + mockDeleteDocument: vi.fn(), + mockMarkDocumentAsFailedTimeout: vi.fn(), + mockProcessDocumentAsync: vi.fn(), + mockProcessDocumentsWithQueue: vi.fn(), + mockRecordAudit: vi.fn(), + mockRetryDocumentProcessing: vi.fn(), + mockUpdateDocument: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { + DOCUMENT_UPLOADED: 'document.uploaded', + DOCUMENT_UPDATED: 'document.updated', + DOCUMENT_DELETED: 'document.deleted', + }, + AuditResourceType: { DOCUMENT: 'document' }, + recordAudit: mockRecordAudit, +})) +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { knowledgeBaseDocumentsUploaded: vi.fn() }, +})) +vi.mock('@/lib/knowledge/documents/service', () => ({ + createDocumentRecords: mockCreateDocumentRecords, + createSingleDocument: mockCreateSingleDocument, + deleteDocument: mockDeleteDocument, + markDocumentAsFailedTimeout: mockMarkDocumentAsFailedTimeout, + processDocumentAsync: mockProcessDocumentAsync, + processDocumentsWithQueue: mockProcessDocumentsWithQueue, + retryDocumentProcessing: mockRetryDocumentProcessing, + updateDocument: mockUpdateDocument, +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + performDeleteKnowledgeDocument, + performMarkKnowledgeDocumentTimedOut, + performRetryKnowledgeDocumentProcessing, + performUpdateKnowledgeDocument, + performUploadKnowledgeDocument, + performUploadKnowledgeDocuments, +} from '@/lib/knowledge/orchestration/documents' + +const KB = { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' } +const FILE = { + filename: 'report.pdf', + fileUrl: 'https://storage/report.pdf', + fileSize: 1024, + mimeType: 'application/pdf', +} +const ACTOR = { userId: 'user-1', source: 'agent' as const, requestId: 'req-1' } + +describe('performUploadKnowledgeDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateSingleDocument.mockResolvedValue({ id: 'doc-1', filename: 'report.pdf' }) + mockProcessDocumentsWithQueue.mockResolvedValue(undefined) + mockProcessDocumentAsync.mockResolvedValue(undefined) + }) + + it('audits an agent upload, which the copilot path never did', async () => { + const outcome = await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: 'user-1', + resourceId: 'doc-1', + resourceName: 'report.pdf', + metadata: expect.objectContaining({ source: 'agent', knowledgeBaseId: 'kb-1' }), + }) + ) + }) + + it('records the document owner the caller names, not the acting user', async () => { + // A workspace API key bills and owns as the workspace account, while the + // acting user stays the audit actor. + await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + uploadedBy: 'workspace-owner', + }) + + expect(mockCreateSingleDocument).toHaveBeenCalledWith(FILE, 'kb-1', 'req-1', 'workspace-owner') + }) + + it('starts no indexing unless the caller asks for it', async () => { + await performUploadKnowledgeDocument({ ...ACTOR, knowledgeBase: KB, document: FILE }) + + expect(mockProcessDocumentsWithQueue).not.toHaveBeenCalled() + expect(mockProcessDocumentAsync).not.toHaveBeenCalled() + }) + + it.each([ + { startProcessing: 'queue' as const, expected: mockProcessDocumentsWithQueue }, + { startProcessing: 'async' as const, expected: mockProcessDocumentAsync }, + ])('hands the record to the $startProcessing pipeline', async ({ startProcessing, expected }) => { + await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + startProcessing, + }) + + expect(expected).toHaveBeenCalled() + }) + + it('classifies a storage-quota rejection as too large, by class not message', async () => { + mockCreateSingleDocument.mockRejectedValue( + new OrchestrationError('payload_too_large', 'Storage limit exceeded. Used: 5.10GB') + ) + + const outcome = await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'payload_too_large' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) + + it('classifies a foreign file reference as forbidden', async () => { + mockCreateSingleDocument.mockRejectedValue( + new OrchestrationError('forbidden', 'Document file is not owned by this knowledge base') + ) + + expect( + (await performUploadKnowledgeDocument({ ...ACTOR, knowledgeBase: KB, document: FILE })) + .errorCode + ).toBe('forbidden') + }) +}) + +describe('performUploadKnowledgeDocuments', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateDocumentRecords.mockResolvedValue([ + { documentId: 'doc-1', filename: 'a.pdf' }, + { documentId: 'doc-2', filename: 'b.pdf' }, + ]) + mockProcessDocumentsWithQueue.mockResolvedValue(undefined) + }) + + it('admits the whole batch in one call and queues it', async () => { + const outcome = await performUploadKnowledgeDocuments({ + ...ACTOR, + knowledgeBase: KB, + documents: [FILE, { ...FILE, filename: 'b.pdf' }], + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockCreateDocumentRecords).toHaveBeenCalledTimes(1) + expect(mockProcessDocumentsWithQueue).toHaveBeenCalledTimes(1) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ resourceName: '2 document(s)' }) + ) + }) + + it('rejects an empty batch before touching the service', async () => { + const outcome = await performUploadKnowledgeDocuments({ + ...ACTOR, + knowledgeBase: KB, + documents: [], + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockCreateDocumentRecords).not.toHaveBeenCalled() + }) +}) + +describe('performUpdateKnowledgeDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + mockUpdateDocument.mockResolvedValue({ id: 'doc-1', filename: 'renamed.pdf' }) + }) + + it('rejects an update that names nothing before touching the service', async () => { + const outcome = await performUpdateKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: { id: 'doc-1', filename: 'report.pdf' }, + updates: { filename: undefined, enabled: undefined }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockUpdateDocument).not.toHaveBeenCalled() + }) + + it('names the changed fields in the audit metadata', async () => { + await performUpdateKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: { id: 'doc-1', filename: 'report.pdf' }, + updates: { filename: 'renamed.pdf', enabled: false }, + }) + + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + resourceName: 'renamed.pdf', + metadata: expect.objectContaining({ + updatedFields: ['filename', 'enabled'], + enabled: false, + }), + }) + ) + }) +}) + +describe('performDeleteKnowledgeDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDeleteDocument.mockResolvedValue({ success: true, message: 'ok' }) + }) + + it('audits the deletion against the acting user', async () => { + const outcome = await performDeleteKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: { id: 'doc-1', filename: 'report.pdf', fileSize: 10, mimeType: 'application/pdf' }, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockDeleteDocument).toHaveBeenCalledWith('doc-1', 'req-1') + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ actorId: 'user-1', resourceId: 'doc-1' }) + ) + expect(mockCaptureServerEvent).toHaveBeenCalled() + }) + + it('emits no telemetry when the delete fails', async () => { + mockDeleteDocument.mockRejectedValue(new Error('deadlock detected')) + + const outcome = await performDeleteKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: { id: 'doc-1', filename: 'report.pdf' }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) +}) + +describe('document processing state changes', () => { + beforeEach(() => vi.clearAllMocks()) + + it('refuses to time out a document that is not processing', async () => { + const outcome = await performMarkKnowledgeDocumentTimedOut({ + document: { id: 'doc-1', processingStatus: 'completed', processingStartedAt: new Date() }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockMarkDocumentAsFailedTimeout).not.toHaveBeenCalled() + }) + + it('refuses to time out a document with no processing start time', async () => { + const outcome = await performMarkKnowledgeDocumentTimedOut({ + document: { id: 'doc-1', processingStatus: 'processing', processingStartedAt: null }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + }) + + it('surfaces a too-soon timeout as caller-fixable, not a fault', async () => { + mockMarkDocumentAsFailedTimeout.mockRejectedValue( + new Error('Document has not been processing long enough to be considered dead') + ) + + const outcome = await performMarkKnowledgeDocumentTimedOut({ + document: { id: 'doc-1', processingStatus: 'processing', processingStartedAt: new Date() }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + }) + + it('refuses to retry a document that has not failed', async () => { + const outcome = await performRetryKnowledgeDocumentProcessing({ + knowledgeBaseId: 'kb-1', + document: { ...FILE, id: 'doc-1', processingStatus: 'completed' }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockRetryDocumentProcessing).not.toHaveBeenCalled() + }) + + it('re-queues a failed document and never audits it', async () => { + mockRetryDocumentProcessing.mockResolvedValue({ + success: true, + status: 'pending', + message: 'Document retry processing started', + }) + + const outcome = await performRetryKnowledgeDocumentProcessing({ + knowledgeBaseId: 'kb-1', + document: { ...FILE, id: 'doc-1', processingStatus: 'failed' }, + requestId: 'req-1', + }) + + expect(outcome).toMatchObject({ success: true, status: 'pending' }) + // No document state the user chose changes, so there is nothing to record. + expect(mockRecordAudit).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/documents.ts b/apps/sim/lib/knowledge/orchestration/documents.ts new file mode 100644 index 00000000000..d0111d53289 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/documents.ts @@ -0,0 +1,486 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { PlatformEvents } from '@/lib/core/telemetry' +import { generateRequestId } from '@/lib/core/utils/request' +import { + createDocumentRecords, + createSingleDocument, + type DocumentData, + deleteDocument, + markDocumentAsFailedTimeout, + type ProcessingOptions, + processDocumentAsync, + processDocumentsWithQueue, + retryDocumentProcessing, + updateDocument, +} from '@/lib/knowledge/documents/service' +import { + auditActorFields, + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { captureServerEvent } from '@/lib/posthog/server' + +const logger = createLogger('KnowledgeDocumentOrchestration') + +/** The knowledge base a document operation targets, already authorized by the caller. */ +export interface KnowledgeBaseTarget { + id: string + name?: string | null + workspaceId: string | null +} + +export interface KnowledgeDocumentInput { + filename: string + fileUrl: string + fileSize: number + mimeType: string + documentTagsData?: string + tag1?: string + tag2?: string + tag3?: string + tag4?: string + tag5?: string + tag6?: string + tag7?: string +} + +/** + * How the created record is handed to the indexing pipeline. + * + * `queue` runs the shared bounded-concurrency queue; `async` starts one + * detached processing run. Omitted starts nothing — the internal single-document + * route deliberately only creates the row and leaves indexing to its caller. + */ +export type KnowledgeDocumentProcessing = 'queue' | 'async' + +export type CreatedKnowledgeDocument = Awaited> + +export interface PerformUploadKnowledgeDocumentParams extends KnowledgeOperationContext { + knowledgeBase: KnowledgeBaseTarget + document: KnowledgeDocumentInput + startProcessing?: KnowledgeDocumentProcessing + processingOptions?: ProcessingOptions + billingAttribution?: BillingAttributionSnapshot + /** Row owner recorded on the document; defaults to the acting user. */ + uploadedBy?: string | null +} + +export type PerformUploadKnowledgeDocumentResult = KnowledgeOrchestrationResult<{ + document: CreatedKnowledgeDocument +}> + +function auditUpload( + params: KnowledgeOperationContext & { knowledgeBase: KnowledgeBaseTarget }, + entry: { resourceId: string; resourceName: string; description: string; metadata: object } +) { + recordAudit({ + workspaceId: params.knowledgeBase.workspaceId, + ...auditActorFields(params), + action: AuditAction.DOCUMENT_UPLOADED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: entry.resourceId, + resourceName: entry.resourceName, + description: entry.description, + metadata: { + source: params.source, + knowledgeBaseId: params.knowledgeBase.id, + knowledgeBaseName: params.knowledgeBase.name, + ...entry.metadata, + }, + ...(params.request ? { request: params.request } : {}), + }) +} + +function captureUpload( + params: KnowledgeOperationContext & { knowledgeBase: KnowledgeBaseTarget }, + documentCount: number, + uploadType: 'single' | 'bulk' +) { + const workspaceId = params.knowledgeBase.workspaceId + captureServerEvent( + params.userId, + 'knowledge_base_document_uploaded', + { + knowledge_base_id: params.knowledgeBase.id, + workspace_id: workspaceId ?? '', + document_count: documentCount, + upload_type: uploadType, + }, + { + ...(workspaceId ? { groups: { workspace: workspaceId } } : {}), + setOnce: { first_document_uploaded_at: new Date().toISOString() }, + } + ) +} + +/** + * Adds one already-resolved file to a knowledge base. + * + * Each surface acquires the file differently — a multipart body uploaded to + * workspace storage, a virtual-filesystem reference resolved to a presigned URL, + * a client-supplied URL — so acquisition stays with the caller and this takes + * the resolved `{filename, fileUrl, fileSize, mimeType}`. Everything downstream + * of that (the record, the indexing hand-off, telemetry, and the audit) is + * identical for all of them, which is why the copilot path used to index + * documents that appear nowhere in the audit log. + */ +export async function performUploadKnowledgeDocument( + params: PerformUploadKnowledgeDocumentParams +): Promise { + const { knowledgeBase, document, startProcessing, processingOptions, billingAttribution } = params + const requestId = params.requestId ?? generateRequestId() + + let created: CreatedKnowledgeDocument + try { + created = await createSingleDocument( + document, + knowledgeBase.id, + requestId, + params.uploadedBy ?? params.userId + ) + } catch (error) { + return classifyKnowledgeFailure( + error, + requestId, + `Upload document "${document.filename}" to knowledge base ${knowledgeBase.id}` + ) + } + + const documentData: DocumentData = { + documentId: created.id, + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + } + + if (startProcessing === 'queue') { + processDocumentsWithQueue( + [documentData], + knowledgeBase.id, + processingOptions ?? {}, + requestId, + billingAttribution + ).catch((error: unknown) => { + logger.error(`[${requestId}] Document processing pipeline failed`, { error }) + }) + } else if (startProcessing === 'async') { + processDocumentAsync( + knowledgeBase.id, + created.id, + document, + processingOptions ?? {}, + billingAttribution + ).catch((error: unknown) => { + logger.error(`[${requestId}] Background document processing failed`, { + documentId: created.id, + error: toError(error).message, + }) + }) + } + + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: knowledgeBase.id, + documentsCount: 1, + uploadType: 'single', + mimeType: document.mimeType, + fileSize: document.fileSize, + }) + captureUpload(params, 1, 'single') + + auditUpload(params, { + resourceId: created.id, + resourceName: document.filename, + description: `Uploaded document "${document.filename}" to knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { + fileName: document.filename, + fileType: document.mimeType, + fileSize: document.fileSize, + }, + }) + + return { success: true, document: created } +} + +export interface PerformUploadKnowledgeDocumentsParams extends KnowledgeOperationContext { + knowledgeBase: KnowledgeBaseTarget + documents: KnowledgeDocumentInput[] + processingOptions?: ProcessingOptions + billingAttribution?: BillingAttributionSnapshot + uploadedBy?: string | null +} + +export type PerformUploadKnowledgeDocumentsResult = KnowledgeOrchestrationResult<{ + documents: DocumentData[] +}> + +/** + * Adds many files to a knowledge base in one storage admission and hands the + * whole set to the bounded-concurrency processing queue. + * + * Kept separate from the single-document path because `createDocumentRecords` + * admits the batch's bytes as one unit; running it per document would let a set + * that exceeds the quota commit its first half. + */ +export async function performUploadKnowledgeDocuments( + params: PerformUploadKnowledgeDocumentsParams +): Promise { + const { knowledgeBase, documents, processingOptions, billingAttribution } = params + const requestId = params.requestId ?? generateRequestId() + + if (documents.length === 0) { + return fail('No documents specified', 'validation') + } + + let created: DocumentData[] + try { + created = await createDocumentRecords( + documents, + knowledgeBase.id, + requestId, + params.uploadedBy ?? params.userId + ) + } catch (error) { + return classifyKnowledgeFailure( + error, + requestId, + `Upload ${documents.length} document(s) to knowledge base ${knowledgeBase.id}` + ) + } + + logger.info(`[${requestId}] Starting controlled async processing of ${created.length} documents`) + + processDocumentsWithQueue( + created, + knowledgeBase.id, + processingOptions ?? {}, + requestId, + billingAttribution + ).catch((error: unknown) => { + logger.error(`[${requestId}] Critical error in document processing pipeline`, { error }) + }) + + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: knowledgeBase.id, + documentsCount: created.length, + uploadType: 'bulk', + recipe: processingOptions?.recipe, + }) + captureUpload(params, created.length, 'bulk') + + auditUpload(params, { + resourceId: knowledgeBase.id, + resourceName: `${created.length} document(s)`, + description: `Uploaded ${created.length} document(s) to knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { fileCount: created.length }, + }) + + return { success: true, documents: created } +} + +export interface PerformUpdateKnowledgeDocumentParams extends KnowledgeOperationContext { + knowledgeBase: KnowledgeBaseTarget + document: { id: string; filename: string } + updates: Parameters[1] +} + +export type PerformUpdateKnowledgeDocumentResult = KnowledgeOrchestrationResult<{ + document: Awaited> +}> + +/** Renames a document, toggles it, or edits its tags, and records the change. */ +export async function performUpdateKnowledgeDocument( + params: PerformUpdateKnowledgeDocumentParams +): Promise { + const { knowledgeBase, document, updates, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + const updatedFields = Object.keys(updates).filter( + (key) => updates[key as keyof typeof updates] !== undefined + ) + if (updatedFields.length === 0) { + return fail('No updates specified', 'validation') + } + + let updated: Awaited> + try { + updated = await updateDocument(document.id, updates, requestId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Update document ${document.id}`) + } + + const filename = updates.filename ?? document.filename + + recordAudit({ + workspaceId: knowledgeBase.workspaceId, + ...auditActorFields(params), + action: AuditAction.DOCUMENT_UPDATED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: document.id, + resourceName: filename, + description: `Updated document "${filename}" in knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { + source, + knowledgeBaseId: knowledgeBase.id, + knowledgeBaseName: knowledgeBase.name, + fileName: filename, + updatedFields, + ...(updates.enabled !== undefined && { enabled: updates.enabled }), + }, + ...(request ? { request } : {}), + }) + + return { success: true, document: updated } +} + +export interface PerformDeleteKnowledgeDocumentParams extends KnowledgeOperationContext { + knowledgeBase: KnowledgeBaseTarget + document: { id: string; filename: string; fileSize?: number; mimeType?: string } +} + +export type PerformDeleteKnowledgeDocumentResult = KnowledgeOrchestrationResult + +/** Deletes a document and its embeddings, and records the deletion. */ +export async function performDeleteKnowledgeDocument( + params: PerformDeleteKnowledgeDocumentParams +): Promise { + const { knowledgeBase, document, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + try { + await deleteDocument(document.id, requestId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Delete document ${document.id}`) + } + + logger.info( + `[${requestId}] Deleted document ${document.id} from knowledge base ${knowledgeBase.id}` + ) + + recordAudit({ + workspaceId: knowledgeBase.workspaceId, + ...auditActorFields(params), + action: AuditAction.DOCUMENT_DELETED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: document.id, + resourceName: document.filename, + description: `Deleted document "${document.filename}" from knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { + source, + knowledgeBaseId: knowledgeBase.id, + knowledgeBaseName: knowledgeBase.name, + fileName: document.filename, + fileSize: document.fileSize, + mimeType: document.mimeType, + }, + ...(request ? { request } : {}), + }) + + const workspaceId = knowledgeBase.workspaceId + captureServerEvent( + params.userId, + 'knowledge_base_document_deleted', + { knowledge_base_id: knowledgeBase.id, workspace_id: workspaceId ?? '' }, + workspaceId ? { groups: { workspace: workspaceId } } : undefined + ) + + return { success: true } +} + +export interface PerformMarkKnowledgeDocumentTimedOutParams { + document: { + id: string + processingStatus: string + processingStartedAt?: Date | null + } + requestId?: string +} + +export type PerformKnowledgeDocumentProcessingResult = KnowledgeOrchestrationResult<{ + status: string + message: string +}> + +/** + * Marks a document whose processing run died as failed. Not audited: the state + * change is the system conceding a run it lost, not a user editing a document. + */ +export async function performMarkKnowledgeDocumentTimedOut( + params: PerformMarkKnowledgeDocumentTimedOutParams +): Promise { + const { document } = params + const requestId = params.requestId ?? generateRequestId() + + if (document.processingStatus !== 'processing') { + return fail( + `Document is not in processing state (current: ${document.processingStatus})`, + 'validation' + ) + } + if (!document.processingStartedAt) { + return fail('Document has no processing start time', 'validation') + } + + try { + await markDocumentAsFailedTimeout(document.id, document.processingStartedAt, requestId) + } catch (error) { + // The service rejects a document that has not been processing long enough + // to be presumed dead; that is a caller-fixable "try again later", not a fault. + if (!(error instanceof OrchestrationError)) { + return fail(toError(error).message, 'validation') + } + return classifyKnowledgeFailure(error, requestId, `Time out document ${document.id}`) + } + + return { success: true, status: 'failed', message: 'Document marked as failed due to timeout' } +} + +export interface PerformRetryKnowledgeDocumentParams { + knowledgeBaseId: string + document: { + id: string + filename: string + fileUrl: string + fileSize: number + mimeType: string + processingStatus: string + } + billingAttribution?: BillingAttributionSnapshot + requestId?: string +} + +/** Re-queues a failed document for indexing. Not audited: no document state a user chose changes. */ +export async function performRetryKnowledgeDocumentProcessing( + params: PerformRetryKnowledgeDocumentParams +): Promise { + const { knowledgeBaseId, document, billingAttribution } = params + const requestId = params.requestId ?? generateRequestId() + + if (document.processingStatus !== 'failed') { + return fail('Document is not in failed state', 'validation') + } + + try { + const result = await retryDocumentProcessing( + knowledgeBaseId, + document.id, + { + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + }, + requestId, + billingAttribution + ) + return { success: true, status: result.status, message: result.message } + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Retry document ${document.id}`) + } +} diff --git a/apps/sim/lib/knowledge/orchestration/index.ts b/apps/sim/lib/knowledge/orchestration/index.ts index dc44ac10ee9..2f9d033a355 100644 --- a/apps/sim/lib/knowledge/orchestration/index.ts +++ b/apps/sim/lib/knowledge/orchestration/index.ts @@ -1,88 +1,34 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { db } from '@sim/db' -import { knowledgeBase } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { eq } from 'drizzle-orm' -import { generateRequestId } from '@/lib/core/utils/request' -import { KnowledgeBaseConflictError, restoreKnowledgeBase } from '@/lib/knowledge/service' - -const logger = createLogger('KnowledgeBaseOrchestration') - -export type KnowledgeOrchestrationErrorCode = 'not_found' | 'conflict' | 'internal' - -export interface RestorableKnowledgeBase { - id: string - name: string - workspaceId: string | null - userId: string -} - -export interface PerformRestoreKnowledgeBaseParams { - knowledgeBaseId: string - userId: string - requestId?: string -} - -export interface PerformRestoreKnowledgeBaseResult { - success: boolean - error?: string - errorCode?: KnowledgeOrchestrationErrorCode - knowledgeBase?: RestorableKnowledgeBase -} - -export async function getRestorableKnowledgeBase( - knowledgeBaseId: string -): Promise { - const [kb] = await db - .select({ - id: knowledgeBase.id, - name: knowledgeBase.name, - workspaceId: knowledgeBase.workspaceId, - userId: knowledgeBase.userId, - }) - .from(knowledgeBase) - .where(eq(knowledgeBase.id, knowledgeBaseId)) - .limit(1) - - return kb ?? null -} - -export async function performRestoreKnowledgeBase( - params: PerformRestoreKnowledgeBaseParams -): Promise { - const { knowledgeBaseId, userId } = params - const requestId = params.requestId ?? generateRequestId() - - const kb = await getRestorableKnowledgeBase(knowledgeBaseId) - if (!kb) { - return { success: false, error: 'Knowledge base not found', errorCode: 'not_found' } - } - - try { - await restoreKnowledgeBase(knowledgeBaseId, requestId) - - logger.info(`[${requestId}] Restored knowledge base ${knowledgeBaseId}`) - - recordAudit({ - workspaceId: kb.workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_RESTORED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: knowledgeBaseId, - resourceName: kb.name, - description: `Restored knowledge base "${kb.name}"`, - metadata: { - knowledgeBaseName: kb.name, - }, - }) - - return { success: true, knowledgeBase: kb } - } catch (error) { - logger.error(`[${requestId}] Failed to restore knowledge base ${knowledgeBaseId}`, { error }) - if (error instanceof KnowledgeBaseConflictError) { - return { success: false, error: error.message, errorCode: 'conflict' } - } - return { success: false, error: toError(error).message, errorCode: 'internal' } - } -} +export { + type ConnectorKnowledgeBase, + type ConnectorWithoutSecret, + getKnowledgeConnector, + type KnowledgeConnectorRow, + performCreateKnowledgeConnector, + performDeleteKnowledgeConnector, + performSyncKnowledgeConnector, + performUpdateKnowledgeConnector, + type SourceConfigRejection, +} from './connectors' +export { + type CreatedKnowledgeDocument, + type KnowledgeBaseTarget, + type KnowledgeDocumentInput, + performDeleteKnowledgeDocument, + performMarkKnowledgeDocumentTimedOut, + performRetryKnowledgeDocumentProcessing, + performUpdateKnowledgeDocument, + performUploadKnowledgeDocument, + performUploadKnowledgeDocuments, +} from './documents' +export { + type PerformKnowledgeBaseResult, + performCreateKnowledgeBase, + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from './knowledge-bases' +export { + getRestorableKnowledgeBase, + performRestoreKnowledgeBase, + type RestorableKnowledgeBase, +} from './restore' +export type { KnowledgeActor, KnowledgeOperationSource } from './shared' diff --git a/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts b/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts new file mode 100644 index 00000000000..6aed845a9be --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts @@ -0,0 +1,256 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCaptureServerEvent, + mockCreateKnowledgeBase, + mockDeleteKnowledgeBase, + mockRecordAudit, + mockUpdateKnowledgeBase, +} = vi.hoisted(() => ({ + mockCaptureServerEvent: vi.fn(), + mockCreateKnowledgeBase: vi.fn(), + mockDeleteKnowledgeBase: vi.fn(), + mockRecordAudit: vi.fn(), + mockUpdateKnowledgeBase: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { + KNOWLEDGE_BASE_CREATED: 'knowledge_base.created', + KNOWLEDGE_BASE_UPDATED: 'knowledge_base.updated', + KNOWLEDGE_BASE_DELETED: 'knowledge_base.deleted', + }, + AuditResourceType: { KNOWLEDGE_BASE: 'knowledge_base' }, + recordAudit: mockRecordAudit, +})) +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { knowledgeBaseCreated: vi.fn(), knowledgeBaseDeleted: vi.fn() }, +})) +vi.mock('@/lib/knowledge/embeddings', () => ({ + EMBEDDING_DIMENSIONS: 1536, + getConfiguredEmbeddingModel: () => 'text-embedding-3-small', +})) +vi.mock('@/lib/knowledge/service', () => ({ + createKnowledgeBase: mockCreateKnowledgeBase, + deleteKnowledgeBase: mockDeleteKnowledgeBase, + updateKnowledgeBase: mockUpdateKnowledgeBase, +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { DEFAULT_CHUNKING_CONFIG } from '@/lib/knowledge/constants' +import { + performCreateKnowledgeBase, + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration/knowledge-bases' + +const CREATED = { id: 'kb-1', name: 'Docs', description: null, workspaceId: 'ws-1' } + +describe('performCreateKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateKnowledgeBase.mockResolvedValue(CREATED) + }) + + it('applies one chunking default for every caller', async () => { + await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'agent', + workspaceId: 'ws-1', + name: 'Docs', + }) + + // The agent used to default minSize to 1 against the API's 100, so the same + // document chunked differently depending on who created the knowledge base. + expect(mockCreateKnowledgeBase).toHaveBeenCalledWith( + expect.objectContaining({ chunkingConfig: { ...DEFAULT_CHUNKING_CONFIG } }), + expect.any(String) + ) + }) + + it('lets a caller override individual chunking fields', async () => { + await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'api', + workspaceId: 'ws-1', + name: 'Docs', + chunkingConfig: { maxSize: 512 }, + }) + + expect(mockCreateKnowledgeBase).toHaveBeenCalledWith( + expect.objectContaining({ + chunkingConfig: { ...DEFAULT_CHUNKING_CONFIG, maxSize: 512 }, + }), + expect.any(String) + ) + }) + + it('audits an agent-created knowledge base, which the copilot path never did', async () => { + await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'agent', + workspaceId: 'ws-1', + name: 'Docs', + }) + + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: 'user-1', + resourceId: 'kb-1', + workspaceId: 'ws-1', + metadata: expect.objectContaining({ source: 'agent' }), + }) + ) + }) + + it('carries request provenance into the audit row', async () => { + const request = new Request('https://sim.ai', { headers: { 'user-agent': 'curl/8' } }) + + await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'ui', + workspaceId: 'ws-1', + name: 'Docs', + request, + }) + + expect(mockRecordAudit).toHaveBeenCalledWith(expect.objectContaining({ request })) + }) + + it('classifies a duplicate name as a conflict, not bad input', async () => { + mockCreateKnowledgeBase.mockRejectedValue( + new OrchestrationError('conflict', 'A knowledge base named "Docs" already exists') + ) + + const outcome = await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'ui', + workspaceId: 'ws-1', + name: 'Docs', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) + + it('keeps an unclassified failure internal and records nothing', async () => { + mockCreateKnowledgeBase.mockRejectedValue(new Error('connection terminated')) + + const outcome = await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'ui', + workspaceId: 'ws-1', + name: 'Docs', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) +}) + +describe('performUpdateKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + mockUpdateKnowledgeBase.mockResolvedValue({ ...CREATED, name: 'Renamed' }) + }) + + it('rejects an update that names nothing before touching the service', async () => { + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: 'kb-1', + workspaceId: 'ws-1', + userId: 'user-1', + source: 'api', + updates: { name: undefined, description: undefined }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockUpdateKnowledgeBase).not.toHaveBeenCalled() + }) + + it('always forwards the actor, so a workspace move is authorized not rejected', async () => { + await performUpdateKnowledgeBase({ + knowledgeBaseId: 'kb-1', + workspaceId: 'ws-1', + userId: 'user-1', + source: 'api', + updates: { workspaceId: 'ws-2' }, + }) + + // The v1 and v2 routes used to omit `actorUserId`, which the service rejects + // outright on a workspace change. + expect(mockUpdateKnowledgeBase).toHaveBeenCalledWith( + 'kb-1', + { workspaceId: 'ws-2' }, + expect.any(String), + { actorUserId: 'user-1' } + ) + }) + + it('files the audit against the destination workspace on a move', async () => { + await performUpdateKnowledgeBase({ + knowledgeBaseId: 'kb-1', + workspaceId: 'ws-1', + userId: 'user-1', + source: 'ui', + updates: { workspaceId: 'ws-2' }, + }) + + expect(mockRecordAudit).toHaveBeenCalledWith(expect.objectContaining({ workspaceId: 'ws-2' })) + }) + + it('classifies a rejected folder as bad input', async () => { + mockUpdateKnowledgeBase.mockRejectedValue( + new OrchestrationError('validation', 'Folder not found in this workspace') + ) + + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: 'kb-1', + workspaceId: 'ws-1', + userId: 'user-1', + source: 'ui', + updates: { folderId: 'folder-elsewhere' }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + }) +}) + +describe('performDeleteKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDeleteKnowledgeBase.mockResolvedValue(undefined) + }) + + it('audits the archive against the acting user', async () => { + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' }, + userId: 'user-1', + source: 'agent', + requestId: 'req-1', + }) + + expect(outcome.success).toBe(true) + expect(mockDeleteKnowledgeBase).toHaveBeenCalledWith('kb-1', 'req-1') + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ actorId: 'user-1', resourceId: 'kb-1' }) + ) + }) + + it('records nothing when the archive fails', async () => { + mockDeleteKnowledgeBase.mockRejectedValue(new Error('deadlock detected')) + + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' }, + userId: 'user-1', + source: 'ui', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts b/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts new file mode 100644 index 00000000000..b808f820e97 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts @@ -0,0 +1,236 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { createLogger } from '@sim/logger' +import { PlatformEvents } from '@/lib/core/telemetry' +import { generateRequestId } from '@/lib/core/utils/request' +import { DEFAULT_CHUNKING_CONFIG } from '@/lib/knowledge/constants' +import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' +import { + auditActorFields, + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { + createKnowledgeBase, + deleteKnowledgeBase, + updateKnowledgeBase, +} from '@/lib/knowledge/service' +import type { ChunkingConfig, KnowledgeBaseWithCounts } from '@/lib/knowledge/types' +import { captureServerEvent } from '@/lib/posthog/server' + +const logger = createLogger('KnowledgeBaseOrchestration') + +export type PerformKnowledgeBaseResult = KnowledgeOrchestrationResult<{ + knowledgeBase: KnowledgeBaseWithCounts +}> + +export interface PerformCreateKnowledgeBaseParams extends KnowledgeOperationContext { + workspaceId: string + name: string + description?: string + /** Folder in the workspace's `knowledge_base` tree; `null`/omitted is the root. */ + folderId?: string | null + /** Omitted fields fall back to {@link DEFAULT_CHUNKING_CONFIG}. */ + chunkingConfig?: Partial +} + +/** + * Creates a knowledge base on behalf of an actor, as the single implementation + * behind the UI route, the v1 and v2 public APIs, and the copilot agent tool. + * + * The chunking default lives here rather than at each boundary: when every + * caller carried its own literal the agent's `minSize` was 1 against the API's + * 100, so identical input produced differently-chunked knowledge bases + * depending on who created it. The audit likewise lives here rather than in the + * three HTTP routes that each had their own copy — which is why an + * agent-created knowledge base used to leave no audit trail at all. + * + * The caller owns authentication; `createKnowledgeBase` still enforces the + * workspace write permission itself, and that failure comes back as `forbidden`. + */ +export async function performCreateKnowledgeBase( + params: PerformCreateKnowledgeBaseParams +): Promise { + const { workspaceId, name, description, folderId, request, source } = params + const requestId = params.requestId ?? generateRequestId() + const chunkingConfig: ChunkingConfig = { ...DEFAULT_CHUNKING_CONFIG, ...params.chunkingConfig } + const embeddingModel = getConfiguredEmbeddingModel() + + let created: KnowledgeBaseWithCounts + try { + created = await createKnowledgeBase( + { + name, + description, + workspaceId, + folderId, + userId: params.userId, + embeddingModel, + embeddingDimension: EMBEDDING_DIMENSIONS, + chunkingConfig, + }, + requestId + ) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create knowledge base "${name}"`) + } + + logger.info(`[${requestId}] Created knowledge base ${created.id} for user ${params.userId}`) + + PlatformEvents.knowledgeBaseCreated({ + knowledgeBaseId: created.id, + name: created.name, + workspaceId, + }) + + captureServerEvent( + params.userId, + 'knowledge_base_created', + { knowledge_base_id: created.id, workspace_id: workspaceId, name: created.name }, + { + groups: { workspace: workspaceId }, + setOnce: { first_kb_created_at: new Date().toISOString() }, + } + ) + + recordAudit({ + workspaceId, + ...auditActorFields(params), + action: AuditAction.KNOWLEDGE_BASE_CREATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: created.id, + resourceName: created.name, + description: `Created knowledge base "${created.name}"`, + metadata: { + source, + name: created.name, + description: created.description, + embeddingModel, + embeddingDimension: EMBEDDING_DIMENSIONS, + chunkingStrategy: chunkingConfig.strategy, + chunkMaxSize: chunkingConfig.maxSize, + chunkMinSize: chunkingConfig.minSize, + chunkOverlap: chunkingConfig.overlap, + }, + ...(request ? { request } : {}), + }) + + return { success: true, knowledgeBase: created } +} + +export interface PerformUpdateKnowledgeBaseParams extends KnowledgeOperationContext { + knowledgeBaseId: string + /** Workspace the knowledge base currently belongs to, for the audit row. */ + workspaceId: string | null + updates: { + name?: string + description?: string + /** Moves the knowledge base between workspaces; omitted leaves it in place. */ + workspaceId?: string | null + folderId?: string | null + chunkingConfig?: ChunkingConfig + } +} + +/** + * Applies a knowledge base update and records it against the actor. + * + * `actorUserId` is always forwarded, so a workspace move is authorized against + * the caller rather than rejected for a missing actor — the v1 and v2 routes + * previously omitted it. + */ +export async function performUpdateKnowledgeBase( + params: PerformUpdateKnowledgeBaseParams +): Promise { + const { knowledgeBaseId, updates, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + const updatedFields = Object.keys(updates).filter( + (key) => updates[key as keyof typeof updates] !== undefined + ) + if (updatedFields.length === 0) { + return fail('No updates specified', 'validation') + } + + let updated: KnowledgeBaseWithCounts + try { + updated = await updateKnowledgeBase(knowledgeBaseId, updates, requestId, { + actorUserId: params.userId, + }) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Update knowledge base ${knowledgeBaseId}`) + } + + logger.info(`[${requestId}] Updated knowledge base ${knowledgeBaseId}`) + + recordAudit({ + // The destination workspace when this update moved it, so the audit row + // lands where the knowledge base now lives. + workspaceId: updates.workspaceId !== undefined ? updates.workspaceId : params.workspaceId, + ...auditActorFields(params), + action: AuditAction.KNOWLEDGE_BASE_UPDATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: knowledgeBaseId, + resourceName: updated.name, + description: `Updated knowledge base "${updated.name}"`, + metadata: { + source, + updatedFields, + ...(updates.name && { newName: updates.name }), + ...(updates.description !== undefined && { description: updates.description }), + ...(updates.chunkingConfig && { + chunkMaxSize: updates.chunkingConfig.maxSize, + chunkMinSize: updates.chunkingConfig.minSize, + chunkOverlap: updates.chunkingConfig.overlap, + }), + }, + ...(request ? { request } : {}), + }) + + return { success: true, knowledgeBase: updated } +} + +export interface PerformDeleteKnowledgeBaseParams extends KnowledgeOperationContext { + knowledgeBase: { id: string; name: string; workspaceId: string | null } +} + +export type PerformDeleteKnowledgeBaseResult = KnowledgeOrchestrationResult + +/** + * Archives a knowledge base and its documents and connectors. + * + * The folder cascade and other internal callers keep calling + * `deleteKnowledgeBase` directly and stay silent by construction — auditing + * follows from a user performing this operation, not from the write running. + */ +export async function performDeleteKnowledgeBase( + params: PerformDeleteKnowledgeBaseParams +): Promise { + const { knowledgeBase, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + try { + await deleteKnowledgeBase(knowledgeBase.id, requestId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Delete knowledge base ${knowledgeBase.id}`) + } + + logger.info(`[${requestId}] Deleted knowledge base ${knowledgeBase.id}`) + + PlatformEvents.knowledgeBaseDeleted({ knowledgeBaseId: knowledgeBase.id }) + + recordAudit({ + workspaceId: knowledgeBase.workspaceId, + ...auditActorFields(params), + action: AuditAction.KNOWLEDGE_BASE_DELETED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: knowledgeBase.id, + resourceName: knowledgeBase.name, + description: `Deleted knowledge base "${knowledgeBase.name}"`, + metadata: { source, knowledgeBaseName: knowledgeBase.name }, + ...(request ? { request } : {}), + }) + + return { success: true } +} diff --git a/apps/sim/lib/knowledge/orchestration/restore.test.ts b/apps/sim/lib/knowledge/orchestration/restore.test.ts new file mode 100644 index 00000000000..558df668d1a --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/restore.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockRecordAudit, mockRestoreKnowledgeBase } = vi.hoisted(() => ({ + mockRecordAudit: vi.fn(), + mockRestoreKnowledgeBase: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { KNOWLEDGE_BASE_RESTORED: 'knowledge_base.restored' }, + AuditResourceType: { KNOWLEDGE_BASE: 'knowledge_base' }, + recordAudit: mockRecordAudit, +})) +vi.mock('@/lib/knowledge/service', () => ({ restoreKnowledgeBase: mockRestoreKnowledgeBase })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { performRestoreKnowledgeBase } from '@/lib/knowledge/orchestration/restore' + +const ARCHIVED = { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1', userId: 'owner' } + +describe('performRestoreKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + afterAll(resetDbChainMock) + + it('audits the restore against the acting user', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ARCHIVED]) + mockRestoreKnowledgeBase.mockResolvedValue(undefined) + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: 'kb-1', + userId: 'user-1', + source: 'ui', + requestId: 'req-1', + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ actorId: 'user-1', resourceId: 'kb-1', workspaceId: 'ws-1' }) + ) + }) + + it('reports a knowledge base that is not archived as a conflict', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ARCHIVED]) + mockRestoreKnowledgeBase.mockRejectedValue( + new OrchestrationError('conflict', 'Knowledge base is not archived') + ) + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: 'kb-1', + userId: 'user-1', + source: 'ui', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + }) + + it('reports bad input as validation, which the narrow local alias could not express', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ARCHIVED]) + mockRestoreKnowledgeBase.mockRejectedValue( + new OrchestrationError('validation', 'Folder not found in this workspace') + ) + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: 'kb-1', + userId: 'user-1', + source: 'ui', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + }) + + it('reports a knowledge base that does not exist as not found', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: 'kb-1', + userId: 'user-1', + source: 'ui', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'not_found' }) + expect(mockRestoreKnowledgeBase).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/restore.ts b/apps/sim/lib/knowledge/orchestration/restore.ts new file mode 100644 index 00000000000..c162dc160b7 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/restore.ts @@ -0,0 +1,88 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { db } from '@sim/db' +import { knowledgeBase } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' +import { generateRequestId } from '@/lib/core/utils/request' +import { + auditActorFields, + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { restoreKnowledgeBase } from '@/lib/knowledge/service' + +const logger = createLogger('KnowledgeBaseRestoreOrchestration') + +export interface RestorableKnowledgeBase { + id: string + name: string + workspaceId: string | null + userId: string +} + +export interface PerformRestoreKnowledgeBaseParams extends KnowledgeOperationContext { + knowledgeBaseId: string +} + +export type PerformRestoreKnowledgeBaseResult = KnowledgeOrchestrationResult<{ + knowledgeBase: RestorableKnowledgeBase +}> + +/** + * Loads an archived knowledge base's identity so the caller can authorize the + * restore. Reads regardless of `deletedAt` — an archived row is exactly what a + * restore targets. + */ +export async function getRestorableKnowledgeBase( + knowledgeBaseId: string +): Promise { + const [kb] = await db + .select({ + id: knowledgeBase.id, + name: knowledgeBase.name, + workspaceId: knowledgeBase.workspaceId, + userId: knowledgeBase.userId, + }) + .from(knowledgeBase) + .where(eq(knowledgeBase.id, knowledgeBaseId)) + .limit(1) + + return kb ?? null +} + +/** Un-archives a knowledge base and its documents and connectors. */ +export async function performRestoreKnowledgeBase( + params: PerformRestoreKnowledgeBaseParams +): Promise { + const { knowledgeBaseId, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + const kb = await getRestorableKnowledgeBase(knowledgeBaseId) + if (!kb) { + return fail('Knowledge base not found', 'not_found') + } + + try { + await restoreKnowledgeBase(knowledgeBaseId, requestId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Restore knowledge base ${knowledgeBaseId}`) + } + + logger.info(`[${requestId}] Restored knowledge base ${knowledgeBaseId}`) + + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.KNOWLEDGE_BASE_RESTORED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: knowledgeBaseId, + resourceName: kb.name, + description: `Restored knowledge base "${kb.name}"`, + metadata: { source, knowledgeBaseName: kb.name }, + ...(request ? { request } : {}), + }) + + return { success: true, knowledgeBase: kb } +} diff --git a/apps/sim/lib/knowledge/orchestration/shared.ts b/apps/sim/lib/knowledge/orchestration/shared.ts new file mode 100644 index 00000000000..2c2581deee7 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/shared.ts @@ -0,0 +1,84 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { + asOrchestrationError, + type OrchestrationErrorCode, + type OrchestrationRequestContext, +} from '@/lib/core/orchestration/types' + +const logger = createLogger('KnowledgeOrchestration') + +/** + * Which surface an operation came in through. Recorded in the audit metadata so + * the audit list still distinguishes a knowledge base the UI created from one an + * API key or the agent created, now that all three share one description. + */ +export type KnowledgeOperationSource = 'ui' | 'api' | 'agent' + +/** The acting user, plus the labels the audit row displays when it has them. */ +export interface KnowledgeActor { + userId: string + actorName?: string | null + actorEmail?: string | null + source: KnowledgeOperationSource +} + +/** Fields every knowledge orchestration function accepts. */ +export interface KnowledgeOperationContext extends KnowledgeActor { + requestId?: string + /** Forwarded to the audit record for IP / user-agent capture. */ + request?: OrchestrationRequestContext +} + +export interface KnowledgeOrchestrationFailure { + success: false + error: string + errorCode: OrchestrationErrorCode +} + +/** + * Every knowledge orchestration function returns this shape. A discriminated + * union rather than an all-optional record, so `if (!outcome.success) return …` + * narrows the success branch and callers reach the payload without asserting it + * is there. + */ +export type KnowledgeOrchestrationResult = + | ({ success: true } & TData) + | KnowledgeOrchestrationFailure + +export function fail( + error: string, + errorCode: OrchestrationErrorCode +): KnowledgeOrchestrationFailure { + return { success: false, error, errorCode } +} + +/** + * Maps a thrown failure to its transport-neutral class. + * + * Every caller-fixable knowledge failure is an {@link OrchestrationError} + * subclass, so the class decides the status rather than the message wording. + * Anything unclassified stays a generic 500, which is what an unexpected fault + * should be, and is logged here because no layer above will see the cause. + */ +export function classifyKnowledgeFailure( + error: unknown, + requestId: string, + operation: string +): KnowledgeOrchestrationFailure { + const classified = asOrchestrationError(error) + if (classified) { + return fail(classified.message, classified.code) + } + logger.error(`[${requestId}] ${operation} failed`, { error }) + return fail(toError(error).message, 'internal') +} + +/** The audit fields carried from the actor, omitting labels the caller lacks. */ +export function auditActorFields(actor: KnowledgeActor) { + return { + actorId: actor.userId, + ...(actor.actorName !== undefined ? { actorName: actor.actorName } : {}), + ...(actor.actorEmail !== undefined ? { actorEmail: actor.actorEmail } : {}), + } +} diff --git a/apps/sim/lib/knowledge/service.test.ts b/apps/sim/lib/knowledge/service.test.ts index 8b9ca673f2d..ce59bc0087b 100644 --- a/apps/sim/lib/knowledge/service.test.ts +++ b/apps/sim/lib/knowledge/service.test.ts @@ -71,7 +71,7 @@ describe('updateKnowledgeBase — workspace transfer authorization', () => { await expect( updateKnowledgeBase('kb-1', { workspaceId: null }, 'req-1', { actorUserId: 'attacker' }) ).rejects.toMatchObject({ - code: 'KNOWLEDGE_BASE_FORBIDDEN', + code: 'forbidden', message: 'Only the knowledge base owner can remove it from a workspace', }) expect(permissionsMockFns.mockGetUserEntityPermissions).not.toHaveBeenCalled() @@ -95,7 +95,7 @@ describe('updateKnowledgeBase — workspace transfer authorization', () => { actorUserId: 'attacker', }) ).rejects.toMatchObject({ - code: 'KNOWLEDGE_BASE_FORBIDDEN', + code: 'forbidden', message: 'User does not have permission on the target workspace', }) expect(permissionsMockFns.mockGetUserEntityPermissions).toHaveBeenCalledWith( diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index cbac1283b63..e4e8cc3707b 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -20,6 +20,7 @@ import { resolveStorageBillingContext, type StorageBillingContext, } from '@/lib/billing/storage' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRestoreName } from '@/lib/core/utils/restore-name' import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries' import type { @@ -31,22 +32,39 @@ import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' const logger = createLogger('KnowledgeBaseService') -export class KnowledgeBaseConflictError extends Error { - readonly code = 'KNOWLEDGE_BASE_EXISTS' as const +/** + * Every caller-fixable knowledge-base failure is an {@link OrchestrationError}, + * so `lib/knowledge/orchestration` classifies it by class and each surface maps + * that one class to its own status. Message text is then free to change without + * silently moving a 409 to a 400. + */ +export class KnowledgeBaseConflictError extends OrchestrationError { constructor(name: string) { - super(`A knowledge base named "${name}" already exists in this workspace`) + super('conflict', `A knowledge base named "${name}" already exists in this workspace`) + this.name = 'KnowledgeBaseConflictError' } } -export class KnowledgeBasePermissionError extends Error { - readonly code = 'KNOWLEDGE_BASE_FORBIDDEN' as const +export class KnowledgeBasePermissionError extends OrchestrationError { + constructor(message: string) { + super('forbidden', message) + this.name = 'KnowledgeBasePermissionError' + } } /** Raised when a caller files a knowledge base under a folder it may not use. */ -export class KnowledgeBaseFolderError extends Error { - readonly code = 'KNOWLEDGE_BASE_FOLDER_INVALID' as const +export class KnowledgeBaseFolderError extends OrchestrationError { constructor() { - super('Folder not found in this workspace') + super('validation', 'Folder not found in this workspace') + this.name = 'KnowledgeBaseFolderError' + } +} + +/** Raised when a knowledge base the caller named does not exist (or is archived). */ +export class KnowledgeBaseNotFoundError extends OrchestrationError { + constructor(knowledgeBaseId: string) { + super('not_found', `Knowledge base ${knowledgeBaseId} not found`) + this.name = 'KnowledgeBaseNotFoundError' } } @@ -341,7 +359,7 @@ export async function updateKnowledgeBase( .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .limit(1) if (!snapshot) { - throw new Error(`Knowledge base ${knowledgeBaseId} not found`) + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } effectiveWorkspaceId = snapshot.workspaceId } @@ -365,7 +383,7 @@ export async function updateKnowledgeBase( .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .limit(1) if (!kbSnapshot) { - throw new Error(`Knowledge base ${knowledgeBaseId} not found`) + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } const sourceWorkspaceId = kbSnapshot.workspaceId ?? null const destinationWorkspaceId = updates.workspaceId ?? null @@ -450,7 +468,7 @@ export async function updateKnowledgeBase( .limit(1) if (!currentKb) { - throw new Error(`Knowledge base ${knowledgeBaseId} not found`) + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } if (storageMove && (currentKb.workspaceId ?? null) !== storageMove.sourceWorkspaceId) { @@ -666,7 +684,7 @@ export async function updateKnowledgeBase( .limit(1) if (updatedKb.length === 0) { - throw new Error(`Knowledge base ${knowledgeBaseId} not found`) + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } logger.info(`[${requestId}] Updated knowledge base: ${knowledgeBaseId}`) @@ -809,18 +827,21 @@ export async function restoreKnowledgeBase( .limit(1) if (!kb) { - throw new Error('Knowledge base not found') + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } if (!kb.deletedAt) { - throw new Error('Knowledge base is not archived') + throw new OrchestrationError('conflict', 'Knowledge base is not archived') } if (kb.workspaceId) { const { getWorkspaceWithOwner } = await import('@/lib/workspaces/permissions/utils') const ws = await getWorkspaceWithOwner(kb.workspaceId) if (!ws || ws.archivedAt) { - throw new Error('Cannot restore knowledge base into an archived workspace') + throw new OrchestrationError( + 'conflict', + 'Cannot restore knowledge base into an archived workspace' + ) } } diff --git a/apps/sim/lib/resources/orchestration/restore-resource.ts b/apps/sim/lib/resources/orchestration/restore-resource.ts index 75baa13aaf4..ec7055f9c5f 100644 --- a/apps/sim/lib/resources/orchestration/restore-resource.ts +++ b/apps/sim/lib/resources/orchestration/restore-resource.ts @@ -148,10 +148,11 @@ export async function performRestoreResource( const result = await performRestoreKnowledgeBase({ knowledgeBaseId: id, userId, + source: 'agent', requestId, }) - if (!result.success || !result.knowledgeBase) { - return { success: false, error: result.error || 'Failed to restore knowledge base' } + if (!result.success) { + return { success: false, error: result.error } } logger.info('Knowledge base restored via restore_resource', { knowledgeBaseId: id })