From bc451afde1af0a0d0cf2ed8d59aca63bfb9f9ddd Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 31 Jul 2026 23:43:44 -0700 Subject: [PATCH 1/2] refactor(knowledge): make lib/knowledge/orchestration the single implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Knowledge base create was implemented four times — the internal route, v1, v2, and the copilot tool — and the orchestration around the shared write had drifted. Extract it the same way lib/table/orchestration was: services write, orchestration decides which writes run, guards them, audits them, and returns a transport-neutral failure. Behavior converged, not preserved: - One chunking default (DEFAULT_CHUNKING_CONFIG). The agent defaulted minSize to 1 against the API's 100, so identical input produced differently-chunked knowledge bases depending on who created it. The agent path now chunks at 100. - Every successful mutation is audited inside the orchestration function. The copilot tool called recordAudit zero times, so agent-created knowledge bases, document uploads, updates and deletes left no audit trail at all. - Failures classify by class, not by message text. The knowledge service errors are OrchestrationError subclasses and storage-quota rejections throw a shared StorageLimitExceededError, replacing four separate message greps for "already exists" / "does not have permission" / "storage limit". delete_connector reported the opposite of what happened. It reached the route through an internal HTTP self-call that sent no query string, so the route's keep-documents default always applied while the agent told the user the documents had been removed. The self-call is gone — all four connector operations run in-process — and the orchestration returns the real counts. Also: - OrchestrationErrorCode gains 'payload_too_large' (413 / PAYLOAD_TOO_LARGE). Without it, dropping the storage-limit message match would have regressed the documented 413 on knowledge base create and document upload to a 500. - messageForOrchestrationError renders a route's own wording for an unclassified fault, so a driver's message no longer reaches the client on a 500. - v1 and v2 knowledge base update now forward actorUserId, which the service requires for a workspace move; both omitted it. - The connector DELETE route reads deleteDocuments through parseRequest. Its contract declared z.boolean(), which would have rejected the string a query param actually is. - Drop the 409 from POST /api/v2/knowledge/{id}/documents in the OpenAPI spec. Nothing on the upload path throws a conflict; it was only ever reachable by the message match this change removes. Behavior change worth noting: a v1/v2 PUT carrying only the workspaceId scope field and no actual updates now returns 400 rather than 200 with the unchanged knowledge base. Deliberately deferred: document update remains internal-only. Extracting performUpdateKnowledgeDocument makes exposing it on v1/v2 a contract and a route away, but that is a new public surface rather than part of this consolidation. --- apps/docs/openapi-v2-knowledge.json | 3 - .../connectors/[connectorId]/route.test.ts | 9 +- .../[id]/connectors/[connectorId]/route.ts | 493 ++++-------- .../[connectorId]/sync/route.test.ts | 10 +- .../connectors/[connectorId]/sync/route.ts | 145 ++-- .../knowledge/[id]/connectors/route.test.ts | 5 +- .../api/knowledge/[id]/connectors/route.ts | 319 ++------ .../[id]/documents/[documentId]/route.ts | 198 ++--- .../knowledge/[id]/documents/route.test.ts | 4 +- .../app/api/knowledge/[id]/documents/route.ts | 199 ++--- .../app/api/knowledge/[id]/restore/route.ts | 15 +- apps/sim/app/api/knowledge/[id]/route.ts | 231 +++--- apps/sim/app/api/knowledge/route.ts | 166 ++-- .../[id]/documents/[documentId]/route.ts | 34 +- .../v1/knowledge/[id]/documents/route.test.ts | 3 + .../api/v1/knowledge/[id]/documents/route.ts | 61 +- apps/sim/app/api/v1/knowledge/[id]/route.ts | 68 +- apps/sim/app/api/v1/knowledge/route.ts | 45 +- .../[id]/documents/[documentId]/route.ts | 35 +- .../api/v2/knowledge/[id]/documents/route.ts | 67 +- apps/sim/app/api/v2/knowledge/[id]/route.ts | 75 +- apps/sim/app/api/v2/knowledge/route.ts | 61 +- apps/sim/app/api/v2/lib/response.ts | 1 + apps/sim/lib/api/contracts/knowledge/base.ts | 11 +- .../lib/api/contracts/knowledge/connectors.ts | 3 +- .../lib/api/contracts/v1/knowledge/index.ts | 17 +- apps/sim/lib/billing/storage/index.ts | 1 + apps/sim/lib/billing/storage/limits.ts | 15 + apps/sim/lib/billing/storage/tracking.ts | 5 +- .../server/knowledge/knowledge-base.test.ts | 138 ++-- .../tools/server/knowledge/knowledge-base.ts | 384 +++++----- apps/sim/lib/core/orchestration/types.ts | 19 + apps/sim/lib/knowledge/constants.ts | 14 + apps/sim/lib/knowledge/documents/service.ts | 20 +- apps/sim/lib/knowledge/folders.test.ts | 2 +- .../orchestration/connectors.test.ts | 258 +++++++ .../lib/knowledge/orchestration/connectors.ts | 708 ++++++++++++++++++ .../knowledge/orchestration/documents.test.ts | 329 ++++++++ .../lib/knowledge/orchestration/documents.ts | 486 ++++++++++++ apps/sim/lib/knowledge/orchestration/index.ts | 121 +-- .../orchestration/knowledge-bases.test.ts | 256 +++++++ .../orchestration/knowledge-bases.ts | 236 ++++++ .../knowledge/orchestration/restore.test.ts | 91 +++ .../lib/knowledge/orchestration/restore.ts | 88 +++ .../sim/lib/knowledge/orchestration/shared.ts | 84 +++ apps/sim/lib/knowledge/service.test.ts | 4 +- apps/sim/lib/knowledge/service.ts | 51 +- .../orchestration/restore-resource.ts | 5 +- 48 files changed, 3665 insertions(+), 1928 deletions(-) create mode 100644 apps/sim/lib/knowledge/orchestration/connectors.test.ts create mode 100644 apps/sim/lib/knowledge/orchestration/connectors.ts create mode 100644 apps/sim/lib/knowledge/orchestration/documents.test.ts create mode 100644 apps/sim/lib/knowledge/orchestration/documents.ts create mode 100644 apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts create mode 100644 apps/sim/lib/knowledge/orchestration/knowledge-bases.ts create mode 100644 apps/sim/lib/knowledge/orchestration/restore.test.ts create mode 100644 apps/sim/lib/knowledge/orchestration/restore.ts create mode 100644 apps/sim/lib/knowledge/orchestration/shared.ts diff --git a/apps/docs/openapi-v2-knowledge.json b/apps/docs/openapi-v2-knowledge.json index eeac6c843b5..806bbbccd83 100644 --- a/apps/docs/openapi-v2-knowledge.json +++ b/apps/docs/openapi-v2-knowledge.json @@ -709,9 +709,6 @@ "404": { "$ref": "#/components/responses/NotFound" }, - "409": { - "$ref": "#/components/responses/Conflict" - }, "413": { "description": "The uploaded file exceeds the 100 MB limit, or the workspace storage limit has been reached.", "content": { diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts index bf347078d73..ce255768db1 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts @@ -151,7 +151,10 @@ describe('Knowledge Connector By ID API Route', () => { success: true, userId: 'user-1', }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) + mockCheckWriteAccess.mockResolvedValue({ + hasAccess: true, + knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, + }) dbChainMockFns.limit.mockResolvedValueOnce([]) const req = createMockRequest('PATCH', { sourceConfig: { project: 'NEW' } }) @@ -174,7 +177,8 @@ describe('Knowledge Connector By ID API Route', () => { mockHasWorkspaceLiveSyncAccess.mockResolvedValue(true) const updatedConnector = { id: 'conn-456', status: 'paused', syncIntervalMinutes: 5 } - dbChainMockFns.limit.mockResolvedValueOnce([updatedConnector]) + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', connectorType: 'jira' }]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedConnector]) const req = createMockRequest('PATCH', { status: 'paused', syncIntervalMinutes: 5 }) const response = await PATCH(req, { params: mockParams }) @@ -196,6 +200,7 @@ describe('Knowledge Connector By ID API Route', () => { knowledgeBase: { workspaceId: 'ws-free', name: 'Free KB' }, }) mockHasWorkspaceLiveSyncAccess.mockResolvedValue(false) + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', connectorType: 'jira' }]) const req = createMockRequest('PATCH', { syncIntervalMinutes: 5 }) const response = await PATCH(req, { params: mockParams }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts index 3ff1d479cd0..8bd4be8b472 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts @@ -1,20 +1,28 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' -import { document, embedding, knowledgeConnector, knowledgeConnectorSyncLog } from '@sim/db/schema' +import { knowledgeConnectorSyncLog } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { and, desc, eq, inArray, isNull, sql } from 'drizzle-orm' +import { desc, eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' -import { updateKnowledgeConnectorContract } from '@/lib/api/contracts/knowledge' +import { + deleteKnowledgeConnectorContract, + updateKnowledgeConnectorContract, +} from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' import { decryptApiKey } from '@/lib/api-key/crypto' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' -import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service' -import { cleanupUnusedTagDefinitions } from '@/lib/knowledge/tags/service' -import { captureServerEvent } from '@/lib/posthog/server' +import { + getKnowledgeConnector, + type KnowledgeConnectorRow, + performDeleteKnowledgeConnector, + performUpdateKnowledgeConnector, +} from '@/lib/knowledge/orchestration' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' @@ -42,20 +50,8 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) } - const connectorRows = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - if (connectorRows.length === 0) { + const connector = await getKnowledgeConnector(knowledgeBaseId, connectorId) + if (!connector) { return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) } @@ -66,7 +62,7 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou .orderBy(desc(knowledgeConnectorSyncLog.startedAt)) .limit(10) - const { encryptedApiKey: _, ...connectorData } = connectorRows[0] + const { encryptedApiKey: _, ...connectorData } = connector return NextResponse.json({ success: true, data: { @@ -81,357 +77,156 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou }) /** - * PATCH /api/knowledge/[id]/connectors/[connectorId] - Update a connector + * Validates a replacement `sourceConfig` against the live source, resolving the + * connector's own token first. Returns a rejection message, or `null` to accept. + * + * Stays with the route rather than moving into orchestration because resolving + * the token needs the requesting identity: workspace credentials are shared and + * token reads are scoped to `account.userId`, so the credential's own account + * owner is used — not the knowledge base owner, and not the acting user when a + * service account mints its own token. */ -export const PATCH = withRouteHandler(async (request: NextRequest, context: RouteParams) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await context.params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) +function makeSourceConfigValidator( + actingUserId: string, + workspaceId: string | null, + connectorId: string +) { + return async ( + connector: KnowledgeConnectorRow, + sourceConfig: Record + ): Promise => { + const connectorConfig = CONNECTOR_REGISTRY[connector.connectorType] + if (!connectorConfig) { + return `Unknown connector type: ${connector.connectorType}` } - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const parsed = await parseRequest(updateKnowledgeConnectorContract, request, context) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - if ( - body.syncIntervalMinutes !== undefined && - body.syncIntervalMinutes > 0 && - body.syncIntervalMinutes < 60 - ) { - const workspaceId = writeCheck.knowledgeBase.workspaceId - if (!workspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace billing context' }, - { status: 409 } - ) - } - const canUseLiveSync = await hasWorkspaceLiveSyncAccess(workspaceId) - if (!canUseLiveSync) { - return NextResponse.json( - { error: 'Live sync requires a Max or Enterprise plan' }, - { status: 403 } - ) + let accessToken: string | null = null + if (connectorConfig.auth.mode === 'apiKey') { + if (!connector.encryptedApiKey) { + return 'API key not found. Please reconfigure the connector.' } - } - - if (body.sourceConfig !== undefined) { - const existingRows = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - if (existingRows.length === 0) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } - - const existing = existingRows[0] - const connectorConfig = CONNECTOR_REGISTRY[existing.connectorType] - - if (!connectorConfig) { - return NextResponse.json( - { error: `Unknown connector type: ${existing.connectorType}` }, - { status: 400 } - ) + accessToken = (await decryptApiKey(connector.encryptedApiKey)).decrypted + } else { + if (!connector.credentialId) { + return 'OAuth credential not found. Please reconfigure the connector.' } - - let accessToken: string | null = null - if (connectorConfig.auth.mode === 'apiKey') { - if (!existing.encryptedApiKey) { - return NextResponse.json( - { error: 'API key not found. Please reconfigure the connector.' }, - { status: 400 } - ) - } - accessToken = (await decryptApiKey(existing.encryptedApiKey)).decrypted - } else { - if (!existing.credentialId) { - return NextResponse.json( - { error: 'OAuth credential not found. Please reconfigure the connector.' }, - { status: 400 } - ) - } - const connectorWorkspaceId = writeCheck.knowledgeBase.workspaceId - if (!connectorWorkspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace context' }, - { status: 409 } - ) - } - /** - * Resolve the credential's own account owner, not the knowledge base owner: - * workspace credentials are shared, and token reads are scoped to - * `account.userId`. - */ - const identity = await resolveCredentialTokenIdentity( - existing.credentialId, - connectorWorkspaceId - ) - if (!identity) { - return NextResponse.json( - { error: 'Credential is no longer usable in this workspace. Please reconnect it.' }, - { status: 400 } - ) - } - accessToken = await refreshAccessTokenIfNeeded( - existing.credentialId, - // Service accounts mint their own token and ignore the acting user. - identity.kind === 'oauth' ? identity.userId : auth.userId, - `patch-${connectorId}` - ) - } - - if (!accessToken) { - return NextResponse.json( - { error: 'Failed to refresh access token. Please reconnect your account.' }, - { status: 401 } - ) + if (!workspaceId) { + return 'Knowledge base is missing workspace context' } - - const validation = await connectorConfig.validateConfig(accessToken, body.sourceConfig) - if (!validation.valid) { - return NextResponse.json( - { error: validation.error || 'Invalid source configuration' }, - { status: 400 } - ) + const identity = await resolveCredentialTokenIdentity(connector.credentialId, workspaceId) + if (!identity) { + return 'Credential is no longer usable in this workspace. Please reconnect it.' } + accessToken = await refreshAccessTokenIfNeeded( + connector.credentialId, + // Service accounts mint their own token and ignore the acting user. + identity.kind === 'oauth' ? identity.userId : actingUserId, + `patch-${connectorId}` + ) } - const updates: Record = { updatedAt: new Date() } - if (body.sourceConfig !== undefined) { - updates.sourceConfig = body.sourceConfig - } - if (body.syncIntervalMinutes !== undefined) { - updates.syncIntervalMinutes = body.syncIntervalMinutes - if (body.syncIntervalMinutes > 0) { - updates.nextSyncAt = new Date(Date.now() + body.syncIntervalMinutes * 60 * 1000) - } else { - updates.nextSyncAt = null - } - } - if (body.status !== undefined) { - updates.status = body.status - if (body.status === 'active') { - updates.consecutiveFailures = 0 - updates.lastSyncError = null - if (updates.nextSyncAt === undefined) { - updates.nextSyncAt = new Date() - } - } + if (!accessToken) { + return 'Failed to refresh access token. Please reconnect your account.' } - await db - .update(knowledgeConnector) - .set(updates) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) + const validation = await connectorConfig.validateConfig(accessToken, sourceConfig) + return validation.valid ? null : validation.error || 'Invalid source configuration' + } +} - const updated = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) +/** + * PATCH /api/knowledge/[id]/connectors/[connectorId] - Update a connector + */ +export const PATCH = withRouteHandler(async (request: NextRequest, context: RouteParams) => { + const requestId = generateRequestId() + const { id: knowledgeBaseId, connectorId } = await context.params - const { encryptedApiKey: __, ...updatedData } = updated[0] + const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_UPDATED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: updatedData.connectorType, - description: `Updated connector for knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - connectorType: updatedData.connectorType, - updatedFields: Object.keys(parsed.data), - ...(body.syncIntervalMinutes !== undefined && { - syncIntervalMinutes: body.syncIntervalMinutes, - }), - ...(body.status !== undefined && { newStatus: body.status }), - }, - request, - }) + const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) + if (!writeCheck.hasAccess) { + const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 + return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) + } - return NextResponse.json({ success: true, data: updatedData }) - } catch (error) { - logger.error(`[${requestId}] Error updating connector`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) + const parsed = await parseRequest(updateKnowledgeConnectorContract, request, context) + if (!parsed.success) return parsed.response + + const outcome = await performUpdateKnowledgeConnector({ + knowledgeBase: { + id: knowledgeBaseId, + name: writeCheck.knowledgeBase.name, + workspaceId: writeCheck.knowledgeBase.workspaceId ?? null, + }, + connectorId, + updates: parsed.data.body, + validateSourceConfig: makeSourceConfigValidator( + auth.userId, + writeCheck.knowledgeBase.workspaceId ?? null, + connectorId + ), + userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Internal server error') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) } + + return NextResponse.json({ success: true, data: outcome.connector }) }) /** * DELETE /api/knowledge/[id]/connectors/[connectorId] - Hard-delete a connector */ -export const DELETE = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => { +export const DELETE = withRouteHandler(async (request: NextRequest, context: RouteParams) => { const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const existingConnector = await db - .select({ id: knowledgeConnector.id, connectorType: knowledgeConnector.connectorType }) - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - if (existingConnector.length === 0) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } - - const { searchParams } = new URL(request.url) - const deleteDocuments = searchParams.get('deleteDocuments') === 'true' - - const { deletedDocs, docCount } = await db.transaction(async (tx) => { - await tx.execute(sql`SELECT 1 FROM knowledge_connector WHERE id = ${connectorId} FOR UPDATE`) - - // Includes pending-removal (tombstoned) docs — the connector is being - // deleted, so there's no future sync left to confirm or resurrect them. - const docs = await tx - .select({ id: document.id, fileUrl: document.fileUrl }) - .from(document) - .where(and(eq(document.connectorId, connectorId), isNull(document.archivedAt))) - - const documentIds = docs.map((doc) => doc.id) - if (deleteDocuments) { - if (documentIds.length > 0) { - await tx.delete(embedding).where(inArray(embedding.documentId, documentIds)) - await tx.delete(document).where(inArray(document.id, documentIds)) - } - } else if (documentIds.length > 0) { - // Kept documents become normal standalone KB entries once their connector - // is gone — resurrect any pending-removal ones rather than leaving them - // invisible tombstones with no future sync left to ever confirm or - // resurrect them. - await tx.update(document).set({ deletedAt: null }).where(inArray(document.id, documentIds)) - } - - const deletedConnectors = await tx - .delete(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .returning({ id: knowledgeConnector.id }) - - if (deletedConnectors.length === 0) { - throw new Error('Connector not found') - } - - return { deletedDocs: deleteDocuments ? docs : [], docCount: docs.length } - }) - - const kbWorkspaceId = writeCheck.knowledgeBase?.workspaceId ?? null + const { id: knowledgeBaseId, connectorId } = await context.params - if (deleteDocuments) { - await Promise.all([ - deletedDocs.length > 0 - ? deleteDocumentStorageFiles( - deletedDocs.map((doc) => ({ ...doc, workspaceId: kbWorkspaceId })), - requestId - ) - : Promise.resolve(), - cleanupUnusedTagDefinitions(knowledgeBaseId, requestId).catch((error) => { - logger.warn(`[${requestId}] Failed to cleanup tag definitions`, error) - }), - ]) - } + const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - logger.info( - `[${requestId}] Deleted connector ${connectorId}${deleteDocuments ? ` and ${docCount} documents` : `, kept ${docCount} documents`}` - ) + const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) + if (!writeCheck.hasAccess) { + const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 + return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) + } - captureServerEvent( - auth.userId, - 'knowledge_base_connector_removed', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId ?? '', - connector_type: existingConnector[0].connectorType, - documents_deleted: deleteDocuments ? docCount : 0, - }, - kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : undefined + const parsed = await parseRequest(deleteKnowledgeConnectorContract, request, context) + if (!parsed.success) return parsed.response + + const outcome = await performDeleteKnowledgeConnector({ + knowledgeBase: { + id: knowledgeBaseId, + name: writeCheck.knowledgeBase.name, + workspaceId: writeCheck.knowledgeBase.workspaceId ?? null, + }, + connectorId, + deleteDocuments: parsed.data.query.deleteDocuments, + userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Internal server error') }, + { status: statusForOrchestrationError(outcome.errorCode) } ) - - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_DELETED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: existingConnector[0].connectorType, - description: `Deleted connector from knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - connectorType: existingConnector[0].connectorType, - deleteDocuments, - documentsDeleted: deleteDocuments ? docCount : 0, - documentsKept: deleteDocuments ? 0 : docCount, - }, - request, - }) - - return NextResponse.json({ success: true }) - } catch (error) { - logger.error(`[${requestId}] Error deleting connector`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) } + + return NextResponse.json({ success: true }) }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts index c79c85df58a..b8869013644 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts @@ -62,7 +62,10 @@ describe('Connector Manual Sync API Route', () => { success: true, userId: 'user-1', }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) + mockCheckWriteAccess.mockResolvedValue({ + hasAccess: true, + knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, + }) dbChainMockFns.limit.mockResolvedValueOnce([]) const req = createMockRequest('POST') @@ -76,7 +79,10 @@ describe('Connector Manual Sync API Route', () => { success: true, userId: 'user-1', }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) + mockCheckWriteAccess.mockResolvedValue({ + hasAccess: true, + knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, + }) dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', status: 'syncing' }]) const req = createMockRequest('POST') diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts index 714f554040b..21e6bfdb50e 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts @@ -1,8 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { db } from '@sim/db' -import { knowledgeConnector } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { and, eq, isNull } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { triggerKnowledgeConnectorSyncContract } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' @@ -11,14 +6,15 @@ import { requireBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { dispatchSync } from '@/lib/knowledge/connectors/queue' -import { captureServerEvent } from '@/lib/posthog/server' +import { performSyncKnowledgeConnector } from '@/lib/knowledge/orchestration' import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' -const logger = createLogger('ConnectorManualSyncAPI') - type RouteParams = { params: Promise<{ id: string; connectorId: string }> } /** @@ -31,105 +27,50 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route const { id: knowledgeBaseId, connectorId } = parsed.data.params const { rehydrate } = parsed.data.query - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const connectorRows = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) + const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - if (connectorRows.length === 0) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } + const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) + if (!writeCheck.hasAccess) { + const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 + return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) + } - if (connectorRows[0].status === 'syncing') { - return NextResponse.json({ error: 'Sync already in progress' }, { status: 409 }) - } + const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId ?? null - const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId - if (!kbWorkspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace billing context' }, - { status: 409 } - ) - } - const billingAttribution = + const outcome = await performSyncKnowledgeConnector({ + knowledgeBase: { + id: knowledgeBaseId, + name: writeCheck.knowledgeBase.name, + workspaceId: kbWorkspaceId, + }, + connectorId, + resolveBillingAttribution: async () => auth.authType === AuthType.INTERNAL_JWT ? requireBillingAttributionHeader(request.headers, { - actorUserId: auth.userId, - workspaceId: kbWorkspaceId, - }) - : await resolveBillingAttribution({ - actorUserId: auth.userId, - workspaceId: kbWorkspaceId, + actorUserId: auth.userId as string, + workspaceId: kbWorkspaceId as string, }) - - logger.info( - `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` - ) - - captureServerEvent( - auth.userId, - 'knowledge_base_connector_synced', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId, - connector_type: connectorRows[0].connectorType, - }, - kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : undefined + : resolveBillingAttribution({ + actorUserId: auth.userId as string, + workspaceId: kbWorkspaceId as string, + }), + rehydrate, + userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Internal server error') }, + { status: statusForOrchestrationError(outcome.errorCode) } ) - - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_SYNCED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: connectorRows[0].connectorType, - description: `Triggered manual sync for connector on knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - connectorType: connectorRows[0].connectorType, - connectorStatus: connectorRows[0].status, - syncType: rehydrate ? 'manual-rehydrate' : 'manual', - }, - request, - }) - - dispatchSync(connectorId, { billingAttribution, requestId, rehydrate }).catch((error) => { - logger.error( - `[${requestId}] Failed to dispatch manual sync for connector ${connectorId}`, - error - ) - }) - - return NextResponse.json({ - success: true, - message: 'Sync triggered', - }) - } catch (error) { - logger.error(`[${requestId}] Error triggering manual sync`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) } + + return NextResponse.json({ success: true, message: 'Sync triggered' }) }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts index 6087572fb40..361a8e2ad68 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts @@ -118,7 +118,8 @@ describe('Knowledge Connectors API Route', () => { }) mockHasWorkspaceLiveSyncAccess.mockResolvedValue(true) mockResolveBillingAttribution.mockResolvedValue(BILLING_ATTRIBUTION) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'knowledge-base-1' }]).mockResolvedValueOnce([ + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'knowledge-base-1' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ { id: 'connector-1', knowledgeBaseId: 'knowledge-base-1', @@ -173,6 +174,8 @@ describe('Knowledge Connectors API Route', () => { expect(response.status).toBe(403) expect(mockHasWorkspaceLiveSyncAccess).toHaveBeenCalledWith('workspace-free') + // The payer is resolved lazily, so a request the plan gate rejects never + // pays for the lookup. expect(mockResolveBillingAttribution).not.toHaveBeenCalled() expect(mockDispatchSync).not.toHaveBeenCalled() }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.ts index b7df3198990..df2f246ae1d 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/route.ts @@ -1,28 +1,25 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' -import { knowledgeBase, knowledgeBaseTagDefinitions, knowledgeConnector } from '@sim/db/schema' +import { knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { and, desc, eq, isNull, sql } from 'drizzle-orm' +import { and, desc, eq, isNull } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { createKnowledgeConnectorContract } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' -import { encryptApiKey } from '@/lib/api-key/crypto' import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { requireBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' -import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' +import { + messageForOrchestrationError, + OrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { dispatchSync } from '@/lib/knowledge/connectors/queue' -import { allocateTagSlots } from '@/lib/knowledge/constants' -import { createTagDefinition } from '@/lib/knowledge/tags/service' -import { captureServerEvent } from '@/lib/posthog/server' +import { performCreateKnowledgeConnector } from '@/lib/knowledge/orchestration' import { getCredential } from '@/app/api/auth/oauth/utils' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' -import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' const logger = createLogger('KnowledgeConnectorsAPI') @@ -80,263 +77,71 @@ export const POST = withRouteHandler( const requestId = generateRequestId() const { id: knowledgeBaseId } = await context.params - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json( - { error: status === 404 ? 'Not found' : 'Unauthorized' }, - { status } - ) - } + const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - const parsed = await parseRequest(createKnowledgeConnectorContract, request, context) - if (!parsed.success) return parsed.response + const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) + if (!writeCheck.hasAccess) { + const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 + return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) + } - const { connectorType, credentialId, apiKey, sourceConfig, syncIntervalMinutes } = - parsed.data.body + const parsed = await parseRequest(createKnowledgeConnectorContract, request, context) + if (!parsed.success) return parsed.response - const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId - if (!kbWorkspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace billing context' }, - { status: 409 } - ) - } + const { connectorType, credentialId, apiKey, sourceConfig, syncIntervalMinutes } = + parsed.data.body - if (syncIntervalMinutes > 0 && syncIntervalMinutes < 60) { - const canUseLiveSync = await hasWorkspaceLiveSyncAccess(kbWorkspaceId) - if (!canUseLiveSync) { - return NextResponse.json( - { error: 'Live sync requires a Max or Enterprise plan' }, - { status: 403 } - ) - } - } + const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId + if (!kbWorkspaceId) { + return NextResponse.json( + { error: 'Knowledge base is missing workspace billing context' }, + { status: 409 } + ) + } - const billingAttribution = + const outcome = await performCreateKnowledgeConnector({ + knowledgeBase: { + id: knowledgeBaseId, + name: writeCheck.knowledgeBase.name, + workspaceId: kbWorkspaceId, + }, + connectorType, + credentialId, + apiKey, + sourceConfig, + syncIntervalMinutes, + resolveBillingAttribution: async () => auth.authType === AuthType.INTERNAL_JWT ? requireBillingAttributionHeader(request.headers, { - actorUserId: auth.userId, + actorUserId: auth.userId as string, workspaceId: kbWorkspaceId, }) - : await resolveBillingAttribution({ - actorUserId: auth.userId, + : resolveBillingAttribution({ + actorUserId: auth.userId as string, workspaceId: kbWorkspaceId, - }) - - const connectorConfig = CONNECTOR_REGISTRY[connectorType] - if (!connectorConfig) { - return NextResponse.json( - { error: `Unknown connector type: ${connectorType}` }, - { status: 400 } - ) - } - - let resolvedCredentialId: string | null = null - let resolvedEncryptedApiKey: string | null = null - let accessToken: string - - if (connectorConfig.auth.mode === 'apiKey') { - if (!apiKey) { - return NextResponse.json({ error: 'API key is required' }, { status: 400 }) - } - accessToken = apiKey - } else { - if (!credentialId) { - return NextResponse.json({ error: 'Credential is required' }, { status: 400 }) - } - - const credential = await getCredential(requestId, credentialId, auth.userId) - if (!credential) { - return NextResponse.json({ error: 'Credential not found' }, { status: 400 }) - } - - if (!credential.accessToken) { - return NextResponse.json( - { error: 'Credential has no access token. Please reconnect your account.' }, - { status: 400 } - ) - } - - accessToken = credential.accessToken - resolvedCredentialId = credentialId - } - - const configValidation = await connectorConfig.validateConfig(accessToken, sourceConfig) - if (!configValidation.valid) { - return NextResponse.json( - { error: configValidation.error || 'Invalid source configuration' }, - { status: 400 } - ) - } - - let finalSourceConfig: Record = { ...sourceConfig } - - if (connectorConfig.auth.mode === 'apiKey' && apiKey) { - const { encrypted } = await encryptApiKey(apiKey) - resolvedEncryptedApiKey = encrypted - } - - const tagSlotMapping: Record = {} - let newTagSlots: Record = {} - - if (connectorConfig.tagDefinitions?.length) { - const disabledIds = new Set((sourceConfig.disabledTagIds as string[] | undefined) ?? []) - const enabledDefs = connectorConfig.tagDefinitions.filter((td) => !disabledIds.has(td.id)) - - const existingDefs = await db - .select({ - tagSlot: knowledgeBaseTagDefinitions.tagSlot, - displayName: knowledgeBaseTagDefinitions.displayName, - fieldType: knowledgeBaseTagDefinitions.fieldType, - }) - .from(knowledgeBaseTagDefinitions) - .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)) - - const usedSlots = new Set(existingDefs.map((d) => d.tagSlot)) - const existingByName = new Map( - existingDefs.map((d) => [d.displayName, { tagSlot: d.tagSlot, fieldType: d.fieldType }]) - ) - - const defsNeedingSlots: typeof enabledDefs = [] - for (const td of enabledDefs) { - const existing = existingByName.get(td.displayName) - if (existing && existing.fieldType === td.fieldType) { - tagSlotMapping[td.id] = existing.tagSlot - } else { - defsNeedingSlots.push(td) - } - } - - const { mapping, skipped: skippedTags } = allocateTagSlots(defsNeedingSlots, usedSlots) - Object.assign(tagSlotMapping, mapping) - newTagSlots = mapping - - for (const name of skippedTags) { - logger.warn(`[${requestId}] No available slots for "${name}"`) - } - - if (skippedTags.length > 0 && Object.keys(tagSlotMapping).length === 0) { - return NextResponse.json( - { error: `No available tag slots. Could not assign: ${skippedTags.join(', ')}` }, - { status: 422 } - ) - } - - finalSourceConfig = { ...finalSourceConfig, tagSlotMapping } - } - - const now = new Date() - const connectorId = generateId() - const nextSyncAt = - syncIntervalMinutes > 0 ? new Date(now.getTime() + syncIntervalMinutes * 60 * 1000) : null - - await db.transaction(async (tx) => { - await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`) - - const activeKb = await tx - .select({ id: knowledgeBase.id }) - .from(knowledgeBase) - .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) - .limit(1) - - if (activeKb.length === 0) { - throw new Error('Knowledge base not found') - } - - for (const [semanticId, slot] of Object.entries(newTagSlots)) { - const td = connectorConfig.tagDefinitions!.find((d) => d.id === semanticId)! - await createTagDefinition( - { - knowledgeBaseId, - tagSlot: slot, - displayName: td.displayName, - fieldType: td.fieldType, - }, - requestId, - tx - ) - } - - await tx.insert(knowledgeConnector).values({ - id: connectorId, - knowledgeBaseId, - connectorType, - credentialId: resolvedCredentialId, - encryptedApiKey: resolvedEncryptedApiKey, - sourceConfig: finalSourceConfig, - syncIntervalMinutes, - status: 'active', - nextSyncAt, - createdAt: now, - updatedAt: now, - }) - }) - - logger.info(`[${requestId}] Created connector ${connectorId} for KB ${knowledgeBaseId}`) - - captureServerEvent( - auth.userId, - 'knowledge_base_connector_added', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId, - connector_type: connectorType, - sync_interval_minutes: syncIntervalMinutes, - }, - { - groups: kbWorkspaceId ? { workspace: kbWorkspaceId } : undefined, - setOnce: { first_connector_added_at: new Date().toISOString() }, - } + }), + resolveAccessToken: async (id) => { + const credential = await getCredential(requestId, id, auth.userId as string) + if (!credential) throw new OrchestrationError('validation', 'Credential not found') + return credential.accessToken ?? null + }, + userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Internal server error') }, + { status: statusForOrchestrationError(outcome.errorCode) } ) - - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_CREATED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: connectorType, - description: `Created ${connectorType} connector for knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - connectorType, - syncIntervalMinutes, - authMode: connectorConfig.auth.mode, - }, - request, - }) - - dispatchSync(connectorId, { billingAttribution, requestId }).catch((error) => { - logger.error( - `[${requestId}] Failed to dispatch initial sync for connector ${connectorId}`, - error - ) - }) - - const created = await db - .select() - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, connectorId)) - .limit(1) - - const { encryptedApiKey: _, ...createdData } = created[0] - return NextResponse.json({ success: true, data: createdData }, { status: 201 }) - } catch (error) { - if (error instanceof Error && error.message === 'Knowledge base not found') { - return NextResponse.json({ error: 'Not found' }, { status: 404 }) - } - logger.error(`[${requestId}] Error creating connector`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) } + + return NextResponse.json({ success: true, data: outcome.connector }, { status: 201 }) } ) diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts index 7acdc821391..dab693e462f 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { updateKnowledgeDocumentContract } from '@/lib/api/contracts/knowledge' @@ -8,15 +7,19 @@ import { requireBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' +import { + messageForOrchestrationError, + type OrchestrationErrorCode, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { - deleteDocument, - markDocumentAsFailedTimeout, - retryDocumentProcessing, - updateDocument, -} from '@/lib/knowledge/documents/service' -import { captureServerEvent } from '@/lib/posthog/server' + performDeleteKnowledgeDocument, + performMarkKnowledgeDocumentTimedOut, + performRetryKnowledgeDocumentProcessing, + performUpdateKnowledgeDocument, +} from '@/lib/knowledge/orchestration' import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils' const logger = createLogger('DocumentByIdAPI') @@ -108,58 +111,30 @@ export const PUT = withRouteHandler( ) if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const updateData: any = {} + const { markFailedDueToTimeout, retryProcessing, ...documentUpdates } = parsed.data.body + const doc = accessCheck.document + const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? null - if (validatedData.markFailedDueToTimeout) { - const doc = accessCheck.document - - if (doc.processingStatus !== 'processing') { - return NextResponse.json( - { error: `Document is not in processing state (current: ${doc.processingStatus})` }, - { status: 400 } - ) - } - - if (!doc.processingStartedAt) { - return NextResponse.json( - { error: 'Document has no processing start time' }, - { status: 400 } - ) - } - - try { - await markDocumentAsFailedTimeout(documentId, doc.processingStartedAt, requestId) + const failed = (outcome: { error?: string; errorCode?: OrchestrationErrorCode }) => + NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to update document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) - return NextResponse.json({ - success: true, - data: { - documentId, - status: 'failed', - message: 'Document marked as failed due to timeout', - }, - }) - } catch (error) { - if (error instanceof Error) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - throw error - } - } else if (validatedData.retryProcessing) { - const doc = accessCheck.document + if (markFailedDueToTimeout) { + const outcome = await performMarkKnowledgeDocumentTimedOut({ + document: doc, + requestId, + }) + if (!outcome.success) return failed(outcome) - if (doc.processingStatus !== 'failed') { - return NextResponse.json({ error: 'Document is not in failed state' }, { status: 400 }) - } + return NextResponse.json({ + success: true, + data: { documentId, status: outcome.status, message: outcome.message }, + }) + } - const docData = { - filename: doc.filename, - fileUrl: doc.fileUrl, - fileSize: doc.fileSize, - mimeType: doc.mimeType, - } - const workspaceId = accessCheck.knowledgeBase?.workspaceId + if (retryProcessing) { const billingAttribution = workspaceId ? auth.authType === AuthType.INTERNAL_JWT ? requireBillingAttributionHeader(req.headers, { @@ -172,56 +147,38 @@ export const PUT = withRouteHandler( }) : undefined - const result = await retryDocumentProcessing( + const outcome = await performRetryKnowledgeDocumentProcessing({ knowledgeBaseId, - documentId, - docData, + document: doc, + billingAttribution, requestId, - billingAttribution - ) - - return NextResponse.json({ - success: true, - data: { - documentId, - status: result.status, - message: result.message, - }, - }) - } else { - const updatedDocument = await updateDocument(documentId, validatedData, requestId) - - logger.info( - `[${requestId}] Document updated: ${documentId} in knowledge base ${knowledgeBaseId}` - ) - - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.DOCUMENT_UPDATED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: documentId, - resourceName: validatedData.filename ?? accessCheck.document?.filename, - description: `Updated document "${validatedData.filename ?? accessCheck.document?.filename}" in knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileName: validatedData.filename ?? accessCheck.document?.filename, - updatedFields: Object.keys(validatedData).filter( - (k) => validatedData[k as keyof typeof validatedData] !== undefined - ), - ...(validatedData.enabled !== undefined && { enabled: validatedData.enabled }), - }, - request: req, }) + if (!outcome.success) return failed(outcome) return NextResponse.json({ success: true, - data: updatedDocument, + data: { documentId, status: outcome.status, message: outcome.message }, }) } + + const outcome = await performUpdateKnowledgeDocument({ + knowledgeBase: { + id: knowledgeBaseId, + name: accessCheck.knowledgeBase?.name, + workspaceId, + }, + document: { id: documentId, filename: doc.filename }, + updates: documentUpdates, + userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request: req, + }) + if (!outcome.success) return failed(outcome) + + return NextResponse.json({ success: true, data: outcome.document }) } catch (error) { logger.error(`[${requestId}] Error updating document ${documentId}`, error) return NextResponse.json({ error: 'Failed to update document' }, { status: 500 }) @@ -257,43 +214,30 @@ export const DELETE = withRouteHandler( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - const result = await deleteDocument(documentId, requestId) - - logger.info( - `[${requestId}] Document deleted: ${documentId} from knowledge base ${knowledgeBaseId}` - ) - - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, + const outcome = await performDeleteKnowledgeDocument({ + knowledgeBase: { + id: knowledgeBaseId, + name: accessCheck.knowledgeBase?.name, + workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, + }, + document: accessCheck.document, + userId, actorName: auth.userName, actorEmail: auth.userEmail, - action: AuditAction.DOCUMENT_DELETED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: documentId, - resourceName: accessCheck.document?.filename, - description: `Deleted document "${accessCheck.document?.filename}" from knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileName: accessCheck.document?.filename, - fileSize: accessCheck.document?.fileSize, - mimeType: accessCheck.document?.mimeType, - }, + source: 'ui', + requestId, request: req, }) - - const kbWorkspaceId = accessCheck.knowledgeBase?.workspaceId ?? '' - captureServerEvent( - userId, - 'knowledge_base_document_deleted', - { knowledge_base_id: knowledgeBaseId, workspace_id: kbWorkspaceId }, - kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : undefined - ) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to delete document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, - data: result, + data: { success: true, message: 'Document deleted successfully' }, }) } catch (error) { logger.error(`[${requestId}] Error deleting document`, error) diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/route.test.ts index 971c4a8f28b..84b523c7870 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.test.ts @@ -570,7 +570,9 @@ describe('Knowledge Base Documents API Route', () => { const data = await response.json() expect(response.status).toBe(500) - expect(data.error).toBe('Database error') + // An unclassified fault renders the route's own wording; the driver's + // message is logged, not returned. + expect(data.error).toBe('Failed to create document') }) }) }) diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.ts b/apps/sim/app/api/knowledge/[id]/documents/route.ts index 9025cea9891..a46d08abae4 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' import { getErrorMessage } from '@sim/utils/errors' @@ -19,19 +18,22 @@ import { requireBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { bulkDocumentOperation, bulkDocumentOperationByFilter, - createDocumentRecords, - createSingleDocument, getDocuments, getProcessingConfig, - KnowledgeBaseFileOwnershipError, - processDocumentsWithQueue, } from '@/lib/knowledge/documents/service' import type { TagFilterCondition } from '@/lib/knowledge/documents/tag-filter' -import { captureServerEvent } from '@/lib/posthog/server' +import { + performUploadKnowledgeDocument, + performUploadKnowledgeDocuments, +} from '@/lib/knowledge/orchestration' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' const logger = createLogger('DocumentsAPI') @@ -210,168 +212,77 @@ export const POST = withRouteHandler( ) } - if (body.bulk === true) { - const createdDocuments = await createDocumentRecords( - body.documents, - knowledgeBaseId, - requestId, - userId - ) - - logger.info( - `[${requestId}] Starting controlled async processing of ${createdDocuments.length} documents` - ) - - try { - const { PlatformEvents } = await import('@/lib/core/telemetry') - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId, - documentsCount: createdDocuments.length, - uploadType: 'bulk', - recipe: body.processingOptions?.recipe, - }) - } catch (_e) { - // Silently fail - } - - captureServerEvent( - userId, - 'knowledge_base_document_uploaded', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId ?? '', - document_count: createdDocuments.length, - upload_type: 'bulk', - }, - { - ...(kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : {}), - setOnce: { first_document_uploaded_at: new Date().toISOString() }, - } - ) - - processDocumentsWithQueue( - createdDocuments, - knowledgeBaseId, - body.processingOptions ?? {}, - requestId, - billingAttribution - ).catch((error: unknown) => { - logger.error(`[${requestId}] Critical error in document processing pipeline:`, error) - }) + const knowledgeBase = { + id: knowledgeBaseId, + name: accessCheck.knowledgeBase?.name, + workspaceId: kbWorkspaceId ?? null, + } + const actor = { + userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui' as const, + requestId, + request: req, + } - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: knowledgeBaseId, - resourceName: `${createdDocuments.length} document(s)`, - description: `Uploaded ${createdDocuments.length} document(s) to knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileCount: createdDocuments.length, - }, - request: req, + if (body.bulk === true) { + const outcome = await performUploadKnowledgeDocuments({ + ...actor, + knowledgeBase, + documents: body.documents, + processingOptions: body.processingOptions, + billingAttribution, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to create document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } + const { batchSize, maxConcurrentDocuments } = getProcessingConfig() return NextResponse.json({ success: true, data: { - total: createdDocuments.length, - documentsCreated: createdDocuments.map((doc) => ({ + total: outcome.documents.length, + documentsCreated: outcome.documents.map((doc) => ({ documentId: doc.documentId, filename: doc.filename, status: 'pending', })), processingMethod: 'background', processingConfig: { - maxConcurrentDocuments: getProcessingConfig().maxConcurrentDocuments, - batchSize: getProcessingConfig().batchSize, - totalBatches: Math.ceil(createdDocuments.length / getProcessingConfig().batchSize), + maxConcurrentDocuments, + batchSize, + totalBatches: Math.ceil(outcome.documents.length / batchSize), }, }, }) } const { bulk: _bulk, workflowId: _workflowId, ...singleDocumentData } = body - const newDocument = await createSingleDocument( - singleDocumentData, - knowledgeBaseId, - requestId, - userId - ) - - try { - const { PlatformEvents } = await import('@/lib/core/telemetry') - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId, - documentsCount: 1, - uploadType: 'single', - mimeType: singleDocumentData.mimeType, - fileSize: singleDocumentData.fileSize, - }) - } catch (_e) { - // Silently fail - } - - captureServerEvent( - userId, - 'knowledge_base_document_uploaded', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: kbWorkspaceId ?? '', - document_count: 1, - upload_type: 'single', - }, - { - ...(kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : {}), - setOnce: { first_document_uploaded_at: new Date().toISOString() }, - } - ) - - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: knowledgeBaseId, - resourceName: singleDocumentData.filename, - description: `Uploaded document "${singleDocumentData.filename}" to knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileName: singleDocumentData.filename, - fileType: singleDocumentData.mimeType, - fileSize: singleDocumentData.fileSize, - }, - request: req, - }) - - return NextResponse.json({ - success: true, - data: newDocument, + // Indexing is deliberately not started here: this path only records the + // document, and its caller drives processing separately. + const outcome = await performUploadKnowledgeDocument({ + ...actor, + knowledgeBase, + document: singleDocumentData, + billingAttribution, }) - } catch (error) { - logger.error(`[${requestId}] Error creating document`, error) - - if (error instanceof KnowledgeBaseFileOwnershipError) { + if (!outcome.success) { return NextResponse.json( - { error: 'File URL does not reference a file owned by this knowledge base' }, - { status: 403 } + { error: messageForOrchestrationError(outcome, 'Failed to create document') }, + { status: statusForOrchestrationError(outcome.errorCode) } ) } - const errorMessage = getErrorMessage(error, 'Failed to create document') - const isStorageLimitError = - errorMessage.includes('Storage limit exceeded') || errorMessage.includes('storage limit') - const isMissingKnowledgeBase = errorMessage === 'Knowledge base not found' - + return NextResponse.json({ success: true, data: outcome.document }) + } catch (error) { + logger.error(`[${requestId}] Error creating document`, error) return NextResponse.json( - { error: errorMessage }, - { status: isMissingKnowledgeBase ? 404 : isStorageLimitError ? 413 : 500 } + { error: getErrorMessage(error, 'Failed to create document') }, + { status: 500 } ) } } diff --git a/apps/sim/app/api/knowledge/[id]/restore/route.ts b/apps/sim/app/api/knowledge/[id]/restore/route.ts index 5dee08582a6..a5ed8b85808 100644 --- a/apps/sim/app/api/knowledge/[id]/restore/route.ts +++ b/apps/sim/app/api/knowledge/[id]/restore/route.ts @@ -4,6 +4,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { restoreKnowledgeBaseContract } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { @@ -45,12 +49,17 @@ export const POST = withRouteHandler( const result = await performRestoreKnowledgeBase({ knowledgeBaseId: id, userId: auth.userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', requestId, + request, }) if (!result.success) { - const status = - result.errorCode === 'not_found' ? 404 : result.errorCode === 'conflict' ? 409 : 500 - return NextResponse.json({ error: result.error }, { status }) + return NextResponse.json( + { error: messageForOrchestrationError(result, 'Failed to restore knowledge base') }, + { status: statusForOrchestrationError(result.errorCode) } + ) } logger.info(`[${requestId}] Restored knowledge base ${id}`) diff --git a/apps/sim/app/api/knowledge/[id]/route.ts b/apps/sim/app/api/knowledge/[id]/route.ts index cd47c173ab4..3b91289af20 100644 --- a/apps/sim/app/api/knowledge/[id]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/route.ts @@ -1,20 +1,19 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { updateKnowledgeBaseContract } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { PlatformEvents } from '@/lib/core/telemetry' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { - deleteKnowledgeBase, - getKnowledgeBaseById, - KnowledgeBaseConflictError, - KnowledgeBaseFolderError, - KnowledgeBasePermissionError, - updateKnowledgeBase, -} from '@/lib/knowledge/service' + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration' +import { getKnowledgeBaseById } from '@/lib/knowledge/service' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' const logger = createLogger('KnowledgeBaseByIdAPI') @@ -69,93 +68,56 @@ export const PUT = withRouteHandler( const requestId = generateRequestId() const { id } = await context.params - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized knowledge base update attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId + const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + logger.warn(`[${requestId}] Unauthorized knowledge base update attempt`) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + const userId = auth.userId - const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) + const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${id}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to update unauthorized knowledge base ${id}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + if (!accessCheck.hasAccess) { + if ('notFound' in accessCheck && accessCheck.notFound) { + logger.warn(`[${requestId}] Knowledge base not found: ${id}`) + return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) } - - const parsed = await parseRequest(updateKnowledgeBaseContract, req, context) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - - const updatedKnowledgeBase = await updateKnowledgeBase( - id, - { - name: validatedData.name, - description: validatedData.description, - workspaceId: validatedData.workspaceId, - folderId: validatedData.folderId, - chunkingConfig: validatedData.chunkingConfig, - }, - requestId, - { actorUserId: userId } + logger.warn( + `[${requestId}] User ${userId} attempted to update unauthorized knowledge base ${id}` ) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - logger.info(`[${requestId}] Knowledge base updated: ${id} for user ${userId}`) - - recordAudit({ - workspaceId: accessCheck.knowledgeBase.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.KNOWLEDGE_BASE_UPDATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: validatedData.name ?? updatedKnowledgeBase.name, - description: `Updated knowledge base "${validatedData.name ?? updatedKnowledgeBase.name}"`, - metadata: { - updatedFields: Object.keys(validatedData).filter( - (k) => validatedData[k as keyof typeof validatedData] !== undefined - ), - ...(validatedData.name && { newName: validatedData.name }), - ...(validatedData.description !== undefined && { - description: validatedData.description, - }), - ...(validatedData.chunkingConfig && { - chunkMaxSize: validatedData.chunkingConfig.maxSize, - chunkMinSize: validatedData.chunkingConfig.minSize, - chunkOverlap: validatedData.chunkingConfig.overlap, - }), - }, - request: req, - }) - - return NextResponse.json({ - success: true, - data: updatedKnowledgeBase, - }) - } catch (error) { - if (error instanceof KnowledgeBaseConflictError) { - return NextResponse.json({ error: error.message }, { status: 409 }) - } - if (error instanceof KnowledgeBaseFolderError) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - if (error instanceof KnowledgeBasePermissionError) { - logger.warn(`[${requestId}] Forbidden knowledge base update on ${id}: ${error.message}`) - return NextResponse.json({ error: error.message }, { status: 403 }) - } - - logger.error(`[${requestId}] Error updating knowledge base`, error) - return NextResponse.json({ error: 'Failed to update knowledge base' }, { status: 500 }) + const parsed = await parseRequest(updateKnowledgeBaseContract, req, context) + if (!parsed.success) return parsed.response + + const body = parsed.data.body + + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: id, + workspaceId: accessCheck.knowledgeBase.workspaceId ?? null, + userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + updates: { + name: body.name, + description: body.description, + workspaceId: body.workspaceId, + folderId: body.folderId, + chunkingConfig: body.chunkingConfig, + }, + requestId, + request: req, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to update knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) } + + return NextResponse.json({ success: true, data: outcome.knowledgeBase }) } ) @@ -164,62 +126,49 @@ export const DELETE = withRouteHandler( const requestId = generateRequestId() const { id } = await params - try { - const auth = await checkSessionOrInternalAuth(_request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized knowledge base delete attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${id}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to delete unauthorized knowledge base ${id}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } + const auth = await checkSessionOrInternalAuth(_request, { requireWorkflowId: false }) + if (!auth.success || !auth.userId) { + logger.warn(`[${requestId}] Unauthorized knowledge base delete attempt`) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + const userId = auth.userId - await deleteKnowledgeBase(id, requestId) + const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) - try { - PlatformEvents.knowledgeBaseDeleted({ - knowledgeBaseId: id, - }) - } catch { - // Telemetry should not fail the operation + if (!accessCheck.hasAccess) { + if ('notFound' in accessCheck && accessCheck.notFound) { + logger.warn(`[${requestId}] Knowledge base not found: ${id}`) + return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) } + logger.warn( + `[${requestId}] User ${userId} attempted to delete unauthorized knowledge base ${id}` + ) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - logger.info(`[${requestId}] Knowledge base deleted: ${id} for user ${userId}`) - - recordAudit({ + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { + id, + name: accessCheck.knowledgeBase.name, workspaceId: accessCheck.knowledgeBase.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.KNOWLEDGE_BASE_DELETED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: accessCheck.knowledgeBase.name, - description: `Deleted knowledge base "${accessCheck.knowledgeBase.name || id}"`, - metadata: { - knowledgeBaseName: accessCheck.knowledgeBase.name, - }, - request: _request, - }) - - return NextResponse.json({ - success: true, - data: { message: 'Knowledge base deleted successfully' }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting knowledge base`, error) - return NextResponse.json({ error: 'Failed to delete knowledge base' }, { status: 500 }) + }, + userId, + actorName: auth.userName, + actorEmail: auth.userEmail, + source: 'ui', + requestId, + request: _request, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to delete knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) } + + return NextResponse.json({ + success: true, + data: { message: 'Knowledge base deleted successfully' }, + }) } ) diff --git a/apps/sim/app/api/knowledge/route.ts b/apps/sim/app/api/knowledge/route.ts index b2f9177b49d..09178f9dff1 100644 --- a/apps/sim/app/api/knowledge/route.ts +++ b/apps/sim/app/api/knowledge/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { @@ -7,19 +6,14 @@ import { } from '@/lib/api/contracts/knowledge' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { PlatformEvents } from '@/lib/core/telemetry' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' -import { - createKnowledgeBase, - getKnowledgeBases, - KnowledgeBaseConflictError, - KnowledgeBaseFolderError, - KnowledgeBasePermissionError, - type KnowledgeBaseScope, -} from '@/lib/knowledge/service' -import { captureServerEvent } from '@/lib/posthog/server' +import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' +import { getKnowledgeBases, type KnowledgeBaseScope } from '@/lib/knowledge/service' const logger = createLogger('KnowledgeBaseAPI') @@ -65,113 +59,49 @@ export const GET = withRouteHandler(async (req: NextRequest) => { export const POST = withRouteHandler(async (req: NextRequest) => { const requestId = generateRequestId() - try { - const session = await getSession() - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthorized knowledge base creation attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - createKnowledgeBaseContract, - req, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid knowledge base data`, { errors: error.issues }) - return NextResponse.json( - { error: 'Invalid request data', details: error.issues }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - - try { - const embeddingModel = getConfiguredEmbeddingModel() - - const createData = { - ...validatedData, - userId: session.user.id, - embeddingModel, - embeddingDimension: EMBEDDING_DIMENSIONS, - } - - const newKnowledgeBase = await createKnowledgeBase(createData, requestId) - - try { - PlatformEvents.knowledgeBaseCreated({ - knowledgeBaseId: newKnowledgeBase.id, - name: validatedData.name, - workspaceId: validatedData.workspaceId, - }) - } catch { - // Telemetry should not fail the operation - } - - captureServerEvent( - session.user.id, - 'knowledge_base_created', - { - knowledge_base_id: newKnowledgeBase.id, - workspace_id: validatedData.workspaceId, - name: validatedData.name, - }, - { - groups: { workspace: validatedData.workspaceId }, - setOnce: { first_kb_created_at: new Date().toISOString() }, - } - ) - - logger.info( - `[${requestId}] Knowledge base created: ${newKnowledgeBase.id} for user ${session.user.id}` - ) - - recordAudit({ - workspaceId: validatedData.workspaceId, - actorId: session.user.id, - actorName: session.user.name, - actorEmail: session.user.email, - action: AuditAction.KNOWLEDGE_BASE_CREATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: newKnowledgeBase.id, - resourceName: validatedData.name, - description: `Created knowledge base "${validatedData.name}"`, - metadata: { - name: validatedData.name, - description: validatedData.description, - embeddingModel, - embeddingDimension: EMBEDDING_DIMENSIONS, - chunkingStrategy: validatedData.chunkingConfig.strategy, - chunkMaxSize: validatedData.chunkingConfig.maxSize, - chunkMinSize: validatedData.chunkingConfig.minSize, - chunkOverlap: validatedData.chunkingConfig.overlap, - }, - request: req, - }) + const session = await getSession() + if (!session?.user?.id) { + logger.warn(`[${requestId}] Unauthorized knowledge base creation attempt`) + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - return NextResponse.json({ - success: true, - data: newKnowledgeBase, - }) - } catch (createError) { - if (createError instanceof KnowledgeBaseConflictError) { - return NextResponse.json({ error: createError.message }, { status: 409 }) - } - if (createError instanceof KnowledgeBaseFolderError) { - return NextResponse.json({ error: createError.message }, { status: 400 }) - } - if (createError instanceof KnowledgeBasePermissionError) { - logger.warn(`[${requestId}] Forbidden knowledge base creation: ${createError.message}`) - return NextResponse.json({ error: createError.message }, { status: 403 }) - } - throw createError + const parsed = await parseRequest( + createKnowledgeBaseContract, + req, + {}, + { + validationErrorResponse: (error) => { + logger.warn(`[${requestId}] Invalid knowledge base data`, { errors: error.issues }) + return NextResponse.json( + { error: 'Invalid request data', details: error.issues }, + { status: 400 } + ) + }, } - } catch (error) { - logger.error(`[${requestId}] Error creating knowledge base`, error) - return NextResponse.json({ error: 'Failed to create knowledge base' }, { status: 500 }) + ) + if (!parsed.success) return parsed.response + + const body = parsed.data.body + + const outcome = await performCreateKnowledgeBase({ + userId: session.user.id, + actorName: session.user.name, + actorEmail: session.user.email, + source: 'ui', + workspaceId: body.workspaceId, + name: body.name, + description: body.description, + folderId: body.folderId, + chunkingConfig: body.chunkingConfig, + requestId, + request: req, + }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to create knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) } + + return NextResponse.json({ success: true, data: outcome.knowledgeBase }) }) diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts index 94c4832f265..33ac4611ffe 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' import { document, knowledgeConnector } from '@sim/db/schema' import { and, eq, isNull } from 'drizzle-orm' @@ -8,8 +7,12 @@ import { v1GetKnowledgeDocumentContract, } from '@/lib/api/contracts/v1/knowledge' import { parseRequest } from '@/lib/api/server' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteDocument } from '@/lib/knowledge/documents/service' +import { performDeleteKnowledgeDocument } from '@/lib/knowledge/orchestration' import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, v1ValidationErrorResponse } from '@/app/api/v1/middleware' @@ -152,19 +155,24 @@ export const DELETE = withRouteHandler( return NextResponse.json({ error: 'Document not found' }, { status: 404 }) } - await deleteDocument(documentId, requestId) - - recordAudit({ - workspaceId: parsed.data.query.workspaceId, - actorId: userId, - action: AuditAction.DOCUMENT_DELETED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: documentId, - resourceName: docs[0].filename, - description: `Deleted document "${docs[0].filename}" from knowledge base via API`, - metadata: { knowledgeBaseId }, + const outcome = await performDeleteKnowledgeDocument({ + knowledgeBase: { + id: knowledgeBaseId, + name: result.kb.name, + workspaceId: parsed.data.query.workspaceId, + }, + document: { id: documentId, filename: docs[0].filename }, + userId, + source: 'api', + requestId, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to delete document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts index 18898d704af..5cba10e6338 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts @@ -75,6 +75,9 @@ vi.mock('@/lib/uploads/contexts/workspace', () => ({ vi.mock('@/lib/uploads/utils/validation', () => ({ validateFileType: mockValidateFileType, + // Read at module scope by `lib/uploads/utils/file-utils`, which the route now + // reaches transitively through the knowledge orchestration module. + SUPPORTED_ARCHIVE_EXTENSIONS: [], })) vi.mock('@/lib/knowledge/documents/service', () => ({ diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts index dfd08d4c892..8f77bb467c6 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { type NextRequest, NextResponse } from 'next/server' import { v1ListKnowledgeDocumentsContract, @@ -10,19 +9,19 @@ import { resolveBillingAttribution, resolveSystemBillingAttribution, } from '@/lib/billing/core/billing-attribution' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { isPayloadSizeLimitError, MAX_MULTIPART_OVERHEAD_BYTES, readFormDataWithLimit, } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSingleDocument, - type DocumentData, - getDocuments, - processDocumentsWithQueue, -} from '@/lib/knowledge/documents/service' +import { getDocuments } from '@/lib/knowledge/documents/service' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' +import { performUploadKnowledgeDocument } from '@/lib/knowledge/orchestration' import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { validateFileType } from '@/lib/uploads/utils/validation' import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' @@ -189,47 +188,29 @@ export const POST = withRouteHandler( contentType ) - const newDocument = await createSingleDocument( - { + const outcome = await performUploadKnowledgeDocument({ + knowledgeBase: { id: knowledgeBaseId, name: result.kb.name, workspaceId }, + document: { filename: file.name, fileUrl: uploadedFile.url, fileSize: file.size, mimeType: contentType, }, - knowledgeBaseId, - requestId, - billingActorUserId - ) - - const documentData: DocumentData = { - documentId: newDocument.id, - filename: file.name, - fileUrl: uploadedFile.url, - fileSize: file.size, - mimeType: contentType, - } - - processDocumentsWithQueue( - [documentData], - knowledgeBaseId, - {}, + startProcessing: 'queue', + billingAttribution, + uploadedBy: billingActorUserId, + userId, + source: 'api', requestId, - billingAttribution - ).catch(() => { - // Processing errors are logged internally - }) - - recordAudit({ - workspaceId, - actorId: userId, - action: AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: newDocument.id, - resourceName: file.name, - description: `Uploaded document "${file.name}" to knowledge base via API`, - metadata: { knowledgeBaseId, fileSize: file.size, mimeType: contentType }, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to upload document') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } + const newDocument = outcome.document return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/knowledge/[id]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/route.ts index 8dbb280559f..373d0951636 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { type NextRequest, NextResponse } from 'next/server' import { v1DeleteKnowledgeBaseContract, @@ -6,8 +5,15 @@ import { v1UpdateKnowledgeBaseContract, } from '@/lib/api/contracts/v1/knowledge' import { parseRequest } from '@/lib/api/server' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteKnowledgeBase, updateKnowledgeBase } from '@/lib/knowledge/service' +import { + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration' import { formatKnowledgeBase, handleError, @@ -67,33 +73,26 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle const result = await resolveKnowledgeBase(id, workspaceId, userId, rateLimit, 'write') if (result instanceof NextResponse) return result - const updates: { - name?: string - description?: string - chunkingConfig?: { maxSize: number; minSize: number; overlap: number } - } = {} - if (name !== undefined) updates.name = name - if (description !== undefined) updates.description = description - if (chunkingConfig !== undefined) updates.chunkingConfig = chunkingConfig - - const updatedKb = await updateKnowledgeBase(id, updates, requestId) - - recordAudit({ + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: id, workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_UPDATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: updatedKb.name, - description: `Updated knowledge base "${updatedKb.name}" via API`, - metadata: { updatedFields: Object.keys(updates) }, + userId, + source: 'api', + updates: { name, description, chunkingConfig }, + requestId, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to update knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, data: { - knowledgeBase: formatKnowledgeBase(updatedKb), + knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase), message: 'Knowledge base updated successfully', }, }) @@ -125,18 +124,23 @@ export const DELETE = withRouteHandler( ) if (result instanceof NextResponse) return result - await deleteKnowledgeBase(id, requestId) - - recordAudit({ - workspaceId: parsed.data.query.workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_DELETED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: result.kb.name, - description: `Deleted knowledge base "${result.kb.name}" via API`, + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { + id, + name: result.kb.name, + workspaceId: parsed.data.query.workspaceId, + }, + userId, + source: 'api', + requestId, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to delete knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/knowledge/route.ts b/apps/sim/app/api/v1/knowledge/route.ts index 5b608484025..cacb36ed482 100644 --- a/apps/sim/app/api/v1/knowledge/route.ts +++ b/apps/sim/app/api/v1/knowledge/route.ts @@ -1,13 +1,16 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { type NextRequest, NextResponse } from 'next/server' import { v1CreateKnowledgeBaseContract, v1ListKnowledgeBasesContract, } from '@/lib/api/contracts/v1/knowledge' import { parseRequest } from '@/lib/api/server' +import { + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' -import { createKnowledgeBase, getKnowledgeBases } from '@/lib/knowledge/service' +import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' +import { getKnowledgeBases } from '@/lib/knowledge/service' import { formatKnowledgeBase, handleError } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, @@ -76,35 +79,27 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId, 'write') if (accessError) return accessError - const kb = await createKnowledgeBase( - { - name, - description, - workspaceId, - userId, - embeddingModel: getConfiguredEmbeddingModel(), - embeddingDimension: EMBEDDING_DIMENSIONS, - chunkingConfig: chunkingConfig ?? { maxSize: 1024, minSize: 100, overlap: 200 }, - }, - requestId - ) - - recordAudit({ + const outcome = await performCreateKnowledgeBase({ + userId, + source: 'api', workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_CREATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: kb.id, - resourceName: kb.name, - description: `Created knowledge base "${kb.name}" via API`, - metadata: { chunkingConfig }, + name, + description, + chunkingConfig, + requestId, request, }) + if (!outcome.success) { + return NextResponse.json( + { error: messageForOrchestrationError(outcome, 'Failed to create knowledge base') }, + { status: statusForOrchestrationError(outcome.errorCode) } + ) + } return NextResponse.json({ success: true, data: { - knowledgeBase: formatKnowledgeBase(kb), + knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase), message: 'Knowledge base created successfully', }, }) diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/[documentId]/route.ts index 41560fb7558..ef9318b1dc6 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/[documentId]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' import { document, knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' @@ -13,12 +12,18 @@ import { import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteDocument } from '@/lib/knowledge/documents/service' +import { performDeleteKnowledgeDocument } from '@/lib/knowledge/orchestration' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' import { checkRateLimit, type RateLimitResult } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' -import { v2Data, v2Error, v2RateLimitError, v2ValidationError } from '@/app/api/v2/lib/response' +import { + v2Data, + v2Error, + v2ErrorForOrchestration, + v2RateLimitError, + v2ValidationError, +} from '@/app/api/v2/lib/response' const logger = createLogger('V2KnowledgeDocumentDetailAPI') @@ -193,19 +198,21 @@ export const DELETE = withRouteHandler( const doc = docs[0] if (!doc) return v2Error('NOT_FOUND', 'Document not found') - await deleteDocument(documentId, requestId) - - recordAudit({ - workspaceId: parsed.data.query.workspaceId, - actorId: userId, - action: AuditAction.DOCUMENT_DELETED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: documentId, - resourceName: doc.filename, - description: `Deleted document "${doc.filename}" from knowledge base via API`, - metadata: { knowledgeBaseId }, + const outcome = await performDeleteKnowledgeDocument({ + knowledgeBase: { + id: knowledgeBaseId, + name: result.kb.name, + workspaceId: parsed.data.query.workspaceId, + }, + document: { id: documentId, filename: doc.filename }, + userId, + source: 'api', + requestId, request, }) + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) + } return v2Data({ id: documentId, deleted: true as const }, { rateLimit }) } catch (error) { diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts index 60103508d3d..1f513d5f2ed 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' @@ -20,13 +19,9 @@ import { readFormDataWithLimit, } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSingleDocument, - type DocumentData, - getDocuments, - processDocumentsWithQueue, -} from '@/lib/knowledge/documents/service' +import { getDocuments } from '@/lib/knowledge/documents/service' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' +import { performUploadKnowledgeDocument } from '@/lib/knowledge/orchestration' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { validateFileType } from '@/lib/uploads/utils/validation' @@ -39,6 +34,7 @@ import { v2CursorList, v2Data, v2Error, + v2ErrorForOrchestration, v2RateLimitError, v2ValidationError, } from '@/app/api/v2/lib/response' @@ -248,47 +244,26 @@ export const POST = withRouteHandler( contentType ) - const newDocument = await createSingleDocument( - { + const outcome = await performUploadKnowledgeDocument({ + knowledgeBase: { id: knowledgeBaseId, name: result.kb.name, workspaceId }, + document: { filename: file.name, fileUrl: uploadedFile.url, fileSize: file.size, mimeType: contentType, }, - knowledgeBaseId, - requestId, - billingAttribution.actorUserId - ) - - const documentData: DocumentData = { - documentId: newDocument.id, - filename: file.name, - fileUrl: uploadedFile.url, - fileSize: file.size, - mimeType: contentType, - } - - processDocumentsWithQueue( - [documentData], - knowledgeBaseId, - {}, + startProcessing: 'queue', + billingAttribution, + uploadedBy: billingAttribution.actorUserId, + userId, + source: 'api', requestId, - billingAttribution - ).catch(() => { - // Processing errors are logged internally by the queue. - }) - - recordAudit({ - workspaceId, - actorId: userId, - action: AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: newDocument.id, - resourceName: file.name, - description: `Uploaded document "${file.name}" to knowledge base via API`, - metadata: { knowledgeBaseId, fileSize: file.size, mimeType: contentType }, request, }) + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) + } + const newDocument = outcome.document const document: V2KnowledgeDocumentSummary = { id: newDocument.id, @@ -310,18 +285,6 @@ export const POST = withRouteHandler( return v2Error('PAYLOAD_TOO_LARGE', error.message) } - if (error instanceof Error) { - if ( - error.message.includes('Storage limit exceeded') || - error.message.includes('storage limit') - ) { - return v2Error('PAYLOAD_TOO_LARGE', 'Storage limit exceeded') - } - if (error.message.includes('already exists')) { - return v2Error('CONFLICT', 'Resource already exists') - } - } - logger.error(`[${requestId}] Error uploading document`, { error: getErrorMessage(error, 'Unknown error'), }) diff --git a/apps/sim/app/api/v2/knowledge/[id]/route.ts b/apps/sim/app/api/v2/knowledge/[id]/route.ts index e6ae3849bae..65364a2b808 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' @@ -7,15 +6,24 @@ import { v2GetKnowledgeBaseContract, v2UpdateKnowledgeBaseContract, } from '@/lib/api/contracts/v2/knowledge' -import { isZodError, parseRequest } from '@/lib/api/server' +import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteKnowledgeBase, updateKnowledgeBase } from '@/lib/knowledge/service' +import { + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { formatKnowledgeBase, resolveKnowledgeBase } from '@/app/api/v1/knowledge/utils' import { checkRateLimit, type RateLimitResult } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' -import { v2Data, v2Error, v2RateLimitError, v2ValidationError } from '@/app/api/v2/lib/response' +import { + v2Data, + v2Error, + v2ErrorForOrchestration, + v2RateLimitError, + v2ValidationError, +} from '@/app/api/v2/lib/response' const logger = createLogger('V2KnowledgeDetailAPI') @@ -110,42 +118,21 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle const result = await resolveKnowledgeBaseScoped(id, workspaceId, userId, rateLimit, 'write') if (result instanceof NextResponse) return result - const updates: { - name?: string - description?: string - chunkingConfig?: { maxSize: number; minSize: number; overlap: number } - } = {} - if (name !== undefined) updates.name = name - if (description !== undefined) updates.description = description - if (chunkingConfig !== undefined) updates.chunkingConfig = chunkingConfig - - const updatedKb = await updateKnowledgeBase(id, updates, requestId) - - recordAudit({ + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: id, workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_UPDATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: updatedKb.name, - description: `Updated knowledge base "${updatedKb.name}" via API`, - metadata: { updatedFields: Object.keys(updates) }, + userId, + source: 'api', + updates: { name, description, chunkingConfig }, + requestId, request, }) - - return v2Data({ knowledgeBase: formatKnowledgeBase(updatedKb) }, { rateLimit }) - } catch (error) { - if (isZodError(error)) return v2ValidationError(error) - - if (error instanceof Error) { - if (error.message.includes('does not have permission')) { - return v2Error('FORBIDDEN', 'Access denied') - } - if (error.message.includes('already exists')) { - return v2Error('CONFLICT', 'Resource already exists') - } + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) } + return v2Data({ knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase) }, { rateLimit }) + } catch (error) { logger.error(`[${requestId}] Error updating knowledge base`, { error: getErrorMessage(error, 'Unknown error'), }) @@ -182,18 +169,16 @@ export const DELETE = withRouteHandler( ) if (result instanceof NextResponse) return result - await deleteKnowledgeBase(id, requestId) - - recordAudit({ - workspaceId: parsed.data.query.workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_DELETED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: id, - resourceName: result.kb.name, - description: `Deleted knowledge base "${result.kb.name}" via API`, + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { id, name: result.kb.name, workspaceId: parsed.data.query.workspaceId }, + userId, + source: 'api', + requestId, request, }) + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) + } return v2Data({ id, deleted: true as const }, { rateLimit }) } catch (error) { diff --git a/apps/sim/app/api/v2/knowledge/route.ts b/apps/sim/app/api/v2/knowledge/route.ts index 65aae6b8d5a..64e6445687b 100644 --- a/apps/sim/app/api/v2/knowledge/route.ts +++ b/apps/sim/app/api/v2/knowledge/route.ts @@ -1,4 +1,3 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { NextRequest } from 'next/server' @@ -6,11 +5,11 @@ import { v2CreateKnowledgeBaseContract, v2ListKnowledgeBasesContract, } from '@/lib/api/contracts/v2/knowledge' -import { isZodError, parseRequest } from '@/lib/api/server' +import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' -import { createKnowledgeBase, getKnowledgeBases } from '@/lib/knowledge/service' +import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' +import { getKnowledgeBases } from '@/lib/knowledge/service' import { formatKnowledgeBase } from '@/app/api/v1/knowledge/utils' import { checkRateLimit, resolveWorkspaceAccess } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' @@ -18,6 +17,7 @@ import { v2CursorList, v2Data, v2Error, + v2ErrorForOrchestration, v2RateLimitError, v2ValidationError, v2WorkspaceAccessError, @@ -97,50 +97,25 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const access = await resolveWorkspaceAccess(rateLimit, userId, workspaceId, 'write') if (access) return v2WorkspaceAccessError(access) - const kb = await createKnowledgeBase( - { - name, - description, - workspaceId, - userId, - embeddingModel: getConfiguredEmbeddingModel(), - embeddingDimension: EMBEDDING_DIMENSIONS, - chunkingConfig: chunkingConfig ?? { maxSize: 1024, minSize: 100, overlap: 200 }, - }, - requestId - ) - - recordAudit({ + const outcome = await performCreateKnowledgeBase({ + userId, + source: 'api', workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_CREATED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: kb.id, - resourceName: kb.name, - description: `Created knowledge base "${kb.name}" via API`, - metadata: { chunkingConfig }, + name, + description, + chunkingConfig, + requestId, request, }) - - return v2Data({ knowledgeBase: formatKnowledgeBase(kb) }, { rateLimit, status: 201 }) - } catch (error) { - if (isZodError(error)) return v2ValidationError(error) - - if (error instanceof Error) { - if (error.message.includes('does not have permission')) { - return v2Error('FORBIDDEN', 'Access denied') - } - if ( - error.message.includes('Storage limit exceeded') || - error.message.includes('storage limit') - ) { - return v2Error('PAYLOAD_TOO_LARGE', 'Storage limit exceeded') - } - if (error.message.includes('already exists')) { - return v2Error('CONFLICT', 'Resource already exists') - } + if (!outcome.success) { + return v2ErrorForOrchestration(outcome.errorCode, outcome.error) } + return v2Data( + { knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase) }, + { rateLimit, status: 201 } + ) + } catch (error) { logger.error(`[${requestId}] Error creating knowledge base`, { error: getErrorMessage(error, 'Unknown error'), }) diff --git a/apps/sim/app/api/v2/lib/response.ts b/apps/sim/app/api/v2/lib/response.ts index 45ed1e6fcb3..8225c065d49 100644 --- a/apps/sim/app/api/v2/lib/response.ts +++ b/apps/sim/app/api/v2/lib/response.ts @@ -163,6 +163,7 @@ const V2_CODE_BY_ORCHESTRATION_ERROR: Record @@ -67,11 +70,7 @@ export const createKnowledgeBaseBodySchema = z.object({ folderId: z.string().min(1, 'Folder ID cannot be empty').nullable().optional(), embeddingModel: z.literal('text-embedding-3-small').default('text-embedding-3-small'), embeddingDimension: z.literal(1536).default(1536), - chunkingConfig: chunkingConfigSchema.default({ - maxSize: 1024, - minSize: 100, - overlap: 200, - }), + chunkingConfig: chunkingConfigSchema.default(DEFAULT_CHUNKING_CONFIG), }) export const updateKnowledgeBaseBodySchema = createKnowledgeBaseBodySchema diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index ed147c741a1..cef3d8718d1 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -22,7 +22,8 @@ export const updateConnectorBodySchema = z.object({ }) export const deleteConnectorQuerySchema = z.object({ - deleteDocuments: z.boolean().optional(), + /** Also hard-delete the documents the connector produced; kept by default. */ + deleteDocuments: booleanQueryFlagSchema.optional().default(false), }) export const connectorDocumentsQuerySchema = z.object({ diff --git a/apps/sim/lib/api/contracts/v1/knowledge/index.ts b/apps/sim/lib/api/contracts/v1/knowledge/index.ts index 76abfcf16a0..926786090b4 100644 --- a/apps/sim/lib/api/contracts/v1/knowledge/index.ts +++ b/apps/sim/lib/api/contracts/v1/knowledge/index.ts @@ -7,7 +7,10 @@ import { } from '@/lib/api/contracts/knowledge/shared' import { requiredFieldSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' -import { KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH } from '@/lib/knowledge/constants' +import { + DEFAULT_CHUNKING_CONFIG, + KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH, +} from '@/lib/knowledge/constants' /** * Public API v1 schemas (`/api/v1/knowledge/**`) @@ -25,9 +28,9 @@ import { KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH } from '@/lib/knowledge/constants /** Simpler chunking config used by the public API (no `strategy`). */ export const v1ChunkingConfigSchema = z.object({ - maxSize: z.number().min(100).max(4000).default(1024), - minSize: z.number().min(1).max(2000).default(100), - overlap: z.number().min(0).max(500).default(200), + maxSize: z.number().min(100).max(4000).default(DEFAULT_CHUNKING_CONFIG.maxSize), + minSize: z.number().min(1).max(2000).default(DEFAULT_CHUNKING_CONFIG.minSize), + overlap: z.number().min(0).max(500).default(DEFAULT_CHUNKING_CONFIG.overlap), }) /** GET `/api/v1/knowledge` — list knowledge bases scoped to a workspace. */ @@ -46,11 +49,7 @@ export const v1CreateKnowledgeBaseBodySchema = z.object({ `Description must be ${KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH} characters or less` ) .optional(), - chunkingConfig: v1ChunkingConfigSchema.optional().default({ - maxSize: 1024, - minSize: 100, - overlap: 200, - }), + chunkingConfig: v1ChunkingConfigSchema.optional().default(DEFAULT_CHUNKING_CONFIG), }) /** GET/DELETE `/api/v1/knowledge/[id]` — workspace scope param. */ diff --git a/apps/sim/lib/billing/storage/index.ts b/apps/sim/lib/billing/storage/index.ts index 5496ad64a5d..59e829499df 100644 --- a/apps/sim/lib/billing/storage/index.ts +++ b/apps/sim/lib/billing/storage/index.ts @@ -6,6 +6,7 @@ export { getStorageUsageForBillingContext, getUserStorageLimit, getUserStorageUsage, + StorageLimitExceededError, } from './limits' export { applyStorageUsageDeltasInTx, diff --git a/apps/sim/lib/billing/storage/limits.ts b/apps/sim/lib/billing/storage/limits.ts index 34b01d97b07..96d27a82265 100644 --- a/apps/sim/lib/billing/storage/limits.ts +++ b/apps/sim/lib/billing/storage/limits.ts @@ -21,9 +21,24 @@ import type { StorageBillingContext } from '@/lib/billing/storage/context' import { getLegacyStorageBillingEntity } from '@/lib/billing/storage/entity' import { getEnv } from '@/lib/core/config/env' import { isBillingEnabled } from '@/lib/core/config/env-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' const logger = createLogger('StorageLimits') +/** + * Thrown when accepting a write would push its payer past its storage quota. + * + * An {@link OrchestrationError} so every surface reaches 413 by class. The bare + * `Error` this replaced was classified by searching the message for "storage + * limit", which the UI, v1, and v2 knowledge routes each re-implemented. + */ +export class StorageLimitExceededError extends OrchestrationError { + constructor(message: string) { + super('payload_too_large', message) + this.name = 'StorageLimitExceededError' + } +} + type StorageLimits = ReturnType interface StorageLimitResolutionInput { diff --git a/apps/sim/lib/billing/storage/tracking.ts b/apps/sim/lib/billing/storage/tracking.ts index 1fab48fa652..0de9b03507b 100644 --- a/apps/sim/lib/billing/storage/tracking.ts +++ b/apps/sim/lib/billing/storage/tracking.ts @@ -18,6 +18,7 @@ import { getUserStorageLimit, getUserStorageUsage, isStorageEnforcementEnabled, + StorageLimitExceededError, } from '@/lib/billing/storage/limits' import { getFreeTierLimit, isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils' import type { DbOrTx } from '@/lib/db/types' @@ -362,7 +363,7 @@ export async function applyStorageUsageDeltasInTx( payerDelta.maximumUsage !== undefined && nextUsage > payerDelta.maximumUsage ) { - throw new Error( + throw new StorageLimitExceededError( `Storage limit exceeded. Used: ${(nextUsage / 1024 ** 3).toFixed(2)}GB, Limit: ${(payerDelta.maximumUsage / 1024 ** 3).toFixed(0)}GB` ) } @@ -471,7 +472,7 @@ async function mutateWorkspaceStorageUsage( currentPayerUsage + bytes > maximumUsage ) { const newUsage = currentPayerUsage + bytes - throw new Error( + throw new StorageLimitExceededError( `Storage limit exceeded. Used: ${(newUsage / 1024 ** 3).toFixed(2)}GB, Limit: ${(maximumUsage / 1024 ** 3).toFixed(0)}GB` ) } diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts index 186e785d29c..52a32dd22d9 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts @@ -2,34 +2,29 @@ * @vitest-environment node */ import { knowledgeConnector } from '@sim/db/schema' -import { queueTableRows, resetDbChainMock, resetUrlsMock, urlsMockFns } from '@sim/testing' -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockAssertBillingAttributionSnapshot, mockCheckKnowledgeBaseWriteAccess, - mockFetch, - mockGenerateInternalToken, - mockSerializeBillingAttributionHeader, + mockPerformCreateKnowledgeConnector, + mockPerformDeleteKnowledgeConnector, + mockPerformSyncKnowledgeConnector, } = vi.hoisted(() => ({ mockAssertBillingAttributionSnapshot: vi.fn(), mockCheckKnowledgeBaseWriteAccess: vi.fn(), - mockFetch: vi.fn(), - mockGenerateInternalToken: vi.fn(), - mockSerializeBillingAttributionHeader: vi.fn(), + mockPerformCreateKnowledgeConnector: vi.fn(), + mockPerformDeleteKnowledgeConnector: vi.fn(), + mockPerformSyncKnowledgeConnector: vi.fn(), })) -vi.mock('@/lib/auth/internal', () => ({ - generateInternalToken: mockGenerateInternalToken, -})) vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ checkActorUsageLimits: vi.fn(), })) vi.mock('@/lib/billing/core/billing-attribution', () => ({ - BILLING_ATTRIBUTION_HEADER: 'x-sim-billing-attribution', assertBillingAttributionSnapshot: mockAssertBillingAttributionSnapshot, checkAttributedUsageLimits: vi.fn(), - serializeBillingAttributionHeader: mockSerializeBillingAttributionHeader, })) vi.mock('@/lib/copilot/generated/tool-catalog-v1', () => ({ KnowledgeBase: { id: 'knowledge_base' }, @@ -37,28 +32,24 @@ vi.mock('@/lib/copilot/generated/tool-catalog-v1', () => ({ vi.mock('@/lib/copilot/tools/server/base-tool', () => ({ assertServerToolNotAborted: vi.fn(), })) -beforeAll(() => { - urlsMockFns.mockGetInternalApiBaseUrl.mockReturnValue('http://internal.test') -}) - -afterAll(resetUrlsMock) -vi.mock('@/lib/knowledge/documents/service', () => ({ - createSingleDocument: vi.fn(), - deleteDocument: vi.fn(), - processDocumentAsync: vi.fn(), - updateDocument: vi.fn(), -})) vi.mock('@/lib/knowledge/embeddings', () => ({ - EMBEDDING_DIMENSIONS: 1536, generateSearchEmbedding: vi.fn(), - getConfiguredEmbeddingModel: vi.fn(), recordSearchEmbeddingUsage: vi.fn(), })) +vi.mock('@/lib/knowledge/orchestration', () => ({ + performCreateKnowledgeBase: vi.fn(), + performCreateKnowledgeConnector: mockPerformCreateKnowledgeConnector, + performDeleteKnowledgeBase: vi.fn(), + performDeleteKnowledgeConnector: mockPerformDeleteKnowledgeConnector, + performDeleteKnowledgeDocument: vi.fn(), + performSyncKnowledgeConnector: mockPerformSyncKnowledgeConnector, + performUpdateKnowledgeBase: vi.fn(), + performUpdateKnowledgeConnector: vi.fn(), + performUpdateKnowledgeDocument: vi.fn(), + performUploadKnowledgeDocument: vi.fn(), +})) vi.mock('@/lib/knowledge/service', () => ({ - createKnowledgeBase: vi.fn(), - deleteKnowledgeBase: vi.fn(), getKnowledgeBaseById: vi.fn(), - updateKnowledgeBase: vi.fn(), })) vi.mock('@/lib/knowledge/tags/service', () => ({ createTagDefinition: vi.fn(), @@ -73,6 +64,7 @@ vi.mock('@/lib/uploads', () => ({ StorageService: {} })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ resolveWorkspaceFileReference: vi.fn(), })) +vi.mock('@/app/api/auth/oauth/utils', () => ({ getCredential: vi.fn() })) vi.mock('@/app/api/knowledge/search/utils', () => ({ executeKnowledgeSearch: vi.fn(), })) @@ -97,6 +89,12 @@ const BILLING_ATTRIBUTION = { payerSubscription: null, } +const CONTEXT = { + userId: 'external-admin', + workspaceId: 'workspace-paid', + billingAttribution: BILLING_ATTRIBUTION, +} + describe('knowledge base connector Copilot operations', () => { afterAll(() => { resetDbChainMock() @@ -105,11 +103,8 @@ describe('knowledge base connector Copilot operations', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - vi.stubGlobal('fetch', mockFetch) queueTableRows(knowledgeConnector, [{ knowledgeBaseId: 'knowledge-base-1' }]) mockAssertBillingAttributionSnapshot.mockReturnValue(BILLING_ATTRIBUTION) - mockSerializeBillingAttributionHeader.mockReturnValue('serialized-attribution') - mockGenerateInternalToken.mockResolvedValue('internal-token') mockCheckKnowledgeBaseWriteAccess.mockResolvedValue({ hasAccess: true, knowledgeBase: { @@ -118,21 +113,21 @@ describe('knowledge base connector Copilot operations', () => { name: 'Paid KB', }, }) - mockFetch.mockResolvedValue({ - ok: true, - json: vi.fn().mockResolvedValue({ - success: true, - data: { - id: 'connector-1', - connectorType: 'notion', - status: 'active', - }, - }), + mockPerformCreateKnowledgeConnector.mockResolvedValue({ + success: true, + connector: { id: 'connector-1', connectorType: 'notion', status: 'active' }, + }) + mockPerformSyncKnowledgeConnector.mockResolvedValue({ success: true }) + mockPerformDeleteKnowledgeConnector.mockResolvedValue({ + success: true, + documentsDeleted: 0, + documentsKept: 3, }) }) it.each([ { + operation: 'add_connector', params: { operation: 'add_connector', args: { @@ -141,35 +136,40 @@ describe('knowledge base connector Copilot operations', () => { apiKey: 'api-key', }, }, - expectedPath: '/api/knowledge/knowledge-base-1/connectors', + perform: mockPerformCreateKnowledgeConnector, }, { - params: { - operation: 'sync_connector', - args: { connectorId: 'connector-1' }, - }, - expectedPath: '/api/knowledge/knowledge-base-1/connectors/connector-1/sync', + operation: 'sync_connector', + params: { operation: 'sync_connector', args: { connectorId: 'connector-1' } }, + perform: mockPerformSyncKnowledgeConnector, }, - ])( - 'forwards immutable billing attribution for $params.operation', - async ({ params, expectedPath }) => { - const result = await knowledgeBaseServerTool.execute(params, { - userId: 'external-admin', - workspaceId: 'workspace-paid', - billingAttribution: BILLING_ATTRIBUTION, - }) + ])('forwards immutable billing attribution for $operation', async ({ params, perform }) => { + const result = await knowledgeBaseServerTool.execute(params, CONTEXT) - expect(result.success).toBe(true) - expect(mockFetch).toHaveBeenCalledWith( - `http://internal.test${expectedPath}`, - expect.objectContaining({ - headers: expect.objectContaining({ - Authorization: 'Bearer internal-token', - 'x-sim-billing-attribution': 'serialized-attribution', - }), - }) - ) - expect(mockSerializeBillingAttributionHeader).toHaveBeenCalledWith(BILLING_ATTRIBUTION) - } - ) + expect(result.success).toBe(true) + // The operation runs in-process now. The payer travels as a value on the + // orchestration call rather than as a serialized header on an internal + // HTTP self-call back into this same process. + const call = perform.mock.calls[0][0] + expect(await call.resolveBillingAttribution()).toEqual(BILLING_ATTRIBUTION) + expect(call.source).toBe('agent') + expect(mockAssertBillingAttributionSnapshot).toHaveBeenCalledWith(BILLING_ATTRIBUTION) + }) + + it('reports that a deleted connector kept its documents, because it did', async () => { + const result = await knowledgeBaseServerTool.execute( + { operation: 'delete_connector', args: { connectorId: 'connector-1' } }, + CONTEXT + ) + + // The old wording claimed the documents "have been removed". They never + // were: the tool reached the route over HTTP with no query string, so the + // route's keep-documents default always applied. + expect(result.success).toBe(true) + expect(result.message).toContain('3 document(s) were kept') + expect(result.message).not.toContain('removed') + expect(mockPerformDeleteKnowledgeConnector).toHaveBeenCalledWith( + expect.objectContaining({ connectorId: 'connector-1', source: 'agent' }) + ) + }) }) diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index bf19100c9ec..220e96c5090 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -1,18 +1,16 @@ import { db } from '@sim/db' import { knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { filterUndefined } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import { and, eq, isNull } from 'drizzle-orm' -import { generateInternalToken } from '@/lib/auth/internal' import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor' import { assertBillingAttributionSnapshot, - BILLING_ATTRIBUTION_HEADER, type BillingAttributionSnapshot, checkAttributedUsageLimits, - serializeBillingAttributionHeader, } from '@/lib/billing/core/billing-attribution' import { KnowledgeBase } from '@/lib/copilot/generated/tool-catalog-v1' import { @@ -20,25 +18,20 @@ import { type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' -import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' +import { generateSearchEmbedding, recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings' import { - createSingleDocument, - deleteDocument, - processDocumentAsync, - updateDocument, -} from '@/lib/knowledge/documents/service' -import { - EMBEDDING_DIMENSIONS, - generateSearchEmbedding, - getConfiguredEmbeddingModel, - recordSearchEmbeddingUsage, -} from '@/lib/knowledge/embeddings' -import { - createKnowledgeBase, - deleteKnowledgeBase, - getKnowledgeBaseById, - updateKnowledgeBase, -} from '@/lib/knowledge/service' + performCreateKnowledgeBase, + performCreateKnowledgeConnector, + performDeleteKnowledgeBase, + performDeleteKnowledgeConnector, + performDeleteKnowledgeDocument, + performSyncKnowledgeConnector, + performUpdateKnowledgeBase, + performUpdateKnowledgeConnector, + performUpdateKnowledgeDocument, + performUploadKnowledgeDocument, +} from '@/lib/knowledge/orchestration' +import { getKnowledgeBaseById } from '@/lib/knowledge/service' import { createTagDefinition, deleteTagDefinition, @@ -50,6 +43,7 @@ import { } from '@/lib/knowledge/tags/service' import { StorageService } from '@/lib/uploads' import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { getCredential } from '@/app/api/auth/oauth/utils' import { executeKnowledgeSearch } from '@/app/api/knowledge/search/utils' import { checkDocumentWriteAccess, @@ -109,6 +103,17 @@ export const knowledgeBaseServerTool: BaseServerTool ({ + userId: context.userId as string, + source: 'agent' as const, + requestId, + }) try { switch (operation) { @@ -129,29 +134,18 @@ export const knowledgeBaseServerTool: BaseServerTool { - logger.error('Background document processing failed', { - documentId: doc.id, - error: toError(err).message, - }) + startProcessing: 'async', + billingAttribution, }) + if (!outcome.success) { + failedFiles.push(fileRef) + continue + } - added.push({ documentId: doc.id, filename: fileRecord.name }) - - logger.info('Workspace file added to knowledge base via copilot', { - knowledgeBaseId: args.knowledgeBaseId, - documentId: doc.id, - fileName: fileRecord.name, - userId: context.userId, - }) + added.push({ documentId: outcome.document.id, filename: fileRecord.name }) } const addedNames = added.map((a) => a.filename).join(', ') @@ -461,13 +439,17 @@ export const knowledgeBaseServerTool: BaseServerTool = { - connectorType: args.connectorType, - sourceConfig: args.sourceConfig ?? {}, - syncIntervalMinutes: args.syncIntervalMinutes ?? 1440, - } - - if (args.credentialId) { - createBody.credentialId = args.credentialId - } - if (args.apiKey) { - createBody.apiKey = args.apiKey - } - + const sourceConfig: Record = { ...(args.sourceConfig ?? {}) } if (args.disabledTagIds?.length) { - ;(createBody.sourceConfig as Record).disabledTagIds = - args.disabledTagIds + sourceConfig.disabledTagIds = args.disabledTagIds } + const requestId = generateId().slice(0, 8) assertNotAborted() - const createRes = await connectorApiCall( - context.userId, - `/api/knowledge/${args.knowledgeBaseId}/connectors`, - 'POST', - createBody, - billingAttribution - ) - - if (!createRes.success) { - return { success: false, message: createRes.error ?? 'Failed to create connector' } - } - - const connector = createRes.data - logger.info('Connector created via copilot', { - connectorId: connector.id, + const outcome = await performCreateKnowledgeConnector({ + ...actor(requestId), + knowledgeBase: { + id: args.knowledgeBaseId, + name: writeAccess.knowledgeBase.name, + workspaceId: connectorWorkspaceId, + }, connectorType: args.connectorType, - knowledgeBaseId: args.knowledgeBaseId, - userId: context.userId, + credentialId: args.credentialId, + apiKey: args.apiKey, + sourceConfig, + syncIntervalMinutes: args.syncIntervalMinutes ?? 1440, + resolveBillingAttribution: async () => billingAttribution, + resolveAccessToken: async (credentialId) => + (await getCredential(requestId, credentialId, context.userId as string)) + ?.accessToken ?? null, }) + if (!outcome.success) { + return { success: false, message: outcome.error } + } + const connector = outcome.connector return { success: true, message: `Connector "${args.connectorType}" added to knowledge base. Initial sync started.`, data: { id: connector.id, - connectorType: connector.connectorType ?? connector.connector_type, + connectorType: connector.connectorType, status: connector.status, knowledgeBaseId: args.knowledgeBaseId, }, @@ -962,41 +960,35 @@ export const knowledgeBaseServerTool: BaseServerTool = {} - if (args.sourceConfig !== undefined) updateBody.sourceConfig = args.sourceConfig - if (args.syncIntervalMinutes !== undefined) - updateBody.syncIntervalMinutes = args.syncIntervalMinutes - if (args.connectorStatus !== undefined) updateBody.status = args.connectorStatus - - if (Object.keys(updateBody).length === 0) { - return { - success: false, - message: - 'At least one of sourceConfig, syncIntervalMinutes, or connectorStatus is required', - } + const updates = { + sourceConfig: args.sourceConfig, + syncIntervalMinutes: args.syncIntervalMinutes, + status: args.connectorStatus, } + const requestId = generateId().slice(0, 8) assertNotAborted() - const updateRes = await connectorApiCall( - context.userId, - `/api/knowledge/${kbId}/connectors/${args.connectorId}`, - 'PATCH', - updateBody - ) - - if (!updateRes.success) { - return { success: false, message: updateRes.error ?? 'Failed to update connector' } - } - - logger.info('Connector updated via copilot', { + // No `validateSourceConfig`: the agent has no requesting identity to + // resolve the connector's OAuth token with, so a replacement config is + // stored unvalidated and the next sync reports any problem with it. + const outcome = await performUpdateKnowledgeConnector({ + ...actor(requestId), + knowledgeBase: { + id: kbId, + name: writeAccess.knowledgeBase.name, + workspaceId: writeAccess.knowledgeBase.workspaceId ?? null, + }, connectorId: args.connectorId, - userId: context.userId, + updates, }) + if (!outcome.success) { + return { success: false, message: outcome.error } + } return { success: true, message: 'Connector updated successfully', - data: { id: args.connectorId, ...updateBody }, + data: { id: args.connectorId, ...filterUndefined(updates) }, } } @@ -1015,26 +1007,36 @@ export const knowledgeBaseServerTool: BaseServerTool 0 + ? `Connector deleted successfully. Its ${outcome.documentsKept} document(s) were kept in the knowledge base.` + : 'Connector deleted successfully.', + data: { + id: args.connectorId, + documentsKept: outcome.documentsKept, + documentsDeleted: outcome.documentsDeleted, + }, } } @@ -1064,23 +1066,21 @@ export const knowledgeBaseServerTool: BaseServerTool billingAttribution, }) + if (!outcome.success) { + return { success: false, message: outcome.error } + } return { success: true, @@ -1111,42 +1111,6 @@ export const knowledgeBaseServerTool: BaseServerTool, - billingAttribution?: BillingAttributionSnapshot -): Promise<{ success: boolean; data?: any; error?: string }> { - const token = await generateInternalToken(userId) - const baseUrl = getInternalApiBaseUrl() - - const res = await fetch(`${baseUrl}${path}`, { - method, - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${token}`, - ...(billingAttribution - ? { - [BILLING_ATTRIBUTION_HEADER]: serializeBillingAttributionHeader(billingAttribution), - } - : {}), - }, - ...(body ? { body: JSON.stringify(body) } : {}), - }) - - const json = await res.json().catch(() => ({})) - - if (!res.ok) { - return { - success: false, - error: json.error || `API returned ${res.status}`, - } - } - - return { success: true, data: json.data } -} - async function resolveKnowledgeBaseId(connectorId: string): Promise { const rows = await db .select({ knowledgeBaseId: knowledgeConnector.knowledgeBaseId }) diff --git a/apps/sim/lib/core/orchestration/types.ts b/apps/sim/lib/core/orchestration/types.ts index 59ccf54aad6..c9d416c3bc6 100644 --- a/apps/sim/lib/core/orchestration/types.ts +++ b/apps/sim/lib/core/orchestration/types.ts @@ -4,6 +4,7 @@ export type OrchestrationErrorCode = | 'forbidden' | 'conflict' | 'locked' + | 'payload_too_large' | 'internal' /** @@ -17,9 +18,27 @@ export function statusForOrchestrationError(code: OrchestrationErrorCode | undef if (code === 'not_found') return 404 if (code === 'conflict') return 409 if (code === 'locked') return 423 + if (code === 'payload_too_large') return 413 return 500 } +/** + * The message a JSON route should render for an orchestration failure. + * + * A classified failure is caller-fixable, so its message is written for the + * caller and is safe to return. An unclassified one carries whatever text the + * fault happened to have — a driver's failed SQL, say — so the caller gets the + * route's own generic wording instead. `v2ErrorForOrchestration` applies the + * same rule for the v2 envelope. + */ +export function messageForOrchestrationError( + result: { error?: string; errorCode?: OrchestrationErrorCode }, + fallback: string +): string { + if (!result.errorCode || result.errorCode === 'internal') return fallback + return result.error ?? fallback +} + /** * A domain failure that already knows its own class. * diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index 84b9d8c9830..57ee50321be 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -1,6 +1,20 @@ /** Max character length for a knowledge base description, enforced at every layer (UI, internal API, v1 API). */ export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000 +/** + * Chunking a knowledge base gets when its creator names no configuration. + * + * Applied in `lib/knowledge/orchestration` so the UI, the v1 and v2 APIs, and + * the copilot agent all index identical input identically. Previously each + * caller carried its own literal and the agent's `minSize` was 1, so the same + * document chunked differently depending on who uploaded it. + */ +export const DEFAULT_CHUNKING_CONFIG = { + maxSize: 1024, + minSize: 100, + overlap: 200, +} as const + export const TAG_SLOT_CONFIG = { text: { slots: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6', 'tag7'] as const, diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 1b8651a8424..9ca64bb6d4b 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -32,6 +32,7 @@ import { maybeNotifyStorageLimitForBillingContext, resolveStorageBillingContext, type StorageBillingContext, + StorageLimitExceededError, } from '@/lib/billing/storage' import { checkAndBillOverageThreshold, @@ -41,6 +42,7 @@ import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' import { env, envNumber } from '@/lib/core/config/env' import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/env-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { processDocument } from '@/lib/knowledge/documents/document-processor' import { @@ -85,9 +87,9 @@ const logger = createLogger('DocumentService') * storage object that is not owned by the target knowledge base's workspace. * Routes map this to a 403. */ -export class KnowledgeBaseFileOwnershipError extends Error { +export class KnowledgeBaseFileOwnershipError extends OrchestrationError { constructor(public readonly storageKey: string) { - super('Document file is not owned by this knowledge base') + super('forbidden', 'Document file is not owned by this knowledge base') this.name = 'KnowledgeBaseFileOwnershipError' } } @@ -1052,7 +1054,7 @@ async function resolveDocumentStorageAdmission( .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .limit(1) if (!kb) { - throw new Error('Knowledge base not found') + throw new OrchestrationError('not_found', 'Knowledge base not found') } if (bytes <= 0) { @@ -1064,7 +1066,7 @@ async function resolveDocumentStorageAdmission( const context = await resolveStorageBillingContext(kb.workspaceId) const quotaCheck = await checkStorageQuotaForBillingContext(context, bytes) if (!quotaCheck.allowed) { - throw new Error(quotaCheck.error || 'Storage limit exceeded') + throw new StorageLimitExceededError(quotaCheck.error || 'Storage limit exceeded') } return { workspaceId: kb.workspaceId, @@ -1078,7 +1080,7 @@ async function resolveDocumentStorageAdmission( getHighestPrioritySubscription(billedUserId), ]) if (!quotaCheck.allowed) { - throw new Error(quotaCheck.error || 'Storage limit exceeded') + throw new StorageLimitExceededError(quotaCheck.error || 'Storage limit exceeded') } return { workspaceId: null, @@ -1126,7 +1128,7 @@ export async function createDocumentRecords( .limit(1) if (kb.length === 0) { - throw new Error('Knowledge base not found') + throw new OrchestrationError('not_found', 'Knowledge base not found') } if ( @@ -1176,7 +1178,7 @@ export async function createDocumentRecords( preparedBilling.bytes ) if (!quotaCheck.allowed) { - throw new Error(quotaCheck.error || 'Storage limit exceeded') + throw new StorageLimitExceededError(quotaCheck.error || 'Storage limit exceeded') } } } @@ -1618,7 +1620,7 @@ export async function createSingleDocument( .limit(1) if (kb.length === 0) { - throw new Error('Knowledge base not found') + throw new OrchestrationError('not_found', 'Knowledge base not found') } if ( @@ -1665,7 +1667,7 @@ export async function createSingleDocument( preparedBilling.bytes ) if (!quotaCheck.allowed) { - throw new Error(quotaCheck.error || 'Storage limit exceeded') + throw new StorageLimitExceededError(quotaCheck.error || 'Storage limit exceeded') } } } diff --git a/apps/sim/lib/knowledge/folders.test.ts b/apps/sim/lib/knowledge/folders.test.ts index a81ebba83bf..009827d4a90 100644 --- a/apps/sim/lib/knowledge/folders.test.ts +++ b/apps/sim/lib/knowledge/folders.test.ts @@ -105,7 +105,7 @@ describe('createKnowledgeBase — folder assignment', () => { await expect( createKnowledgeBase({ ...CREATE_INPUT, folderId: 'f-1' }, 'req-1') - ).rejects.toMatchObject({ code: 'KNOWLEDGE_BASE_FORBIDDEN' }) + ).rejects.toMatchObject({ code: 'forbidden' }) expect(mockFindActiveFolder).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts new file mode 100644 index 00000000000..fec4795dc79 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -0,0 +1,258 @@ +/** + * @vitest-environment node + */ +import { document } from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCaptureServerEvent, + mockDispatchSync, + mockHasWorkspaceLiveSyncAccess, + mockRecordAudit, +} = vi.hoisted(() => ({ + mockCaptureServerEvent: vi.fn(), + mockDispatchSync: vi.fn(), + mockHasWorkspaceLiveSyncAccess: vi.fn(), + mockRecordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { + CONNECTOR_CREATED: 'connector.created', + CONNECTOR_UPDATED: 'connector.updated', + CONNECTOR_DELETED: 'connector.deleted', + CONNECTOR_SYNCED: 'connector.synced', + }, + AuditResourceType: { CONNECTOR: 'connector' }, + recordAudit: mockRecordAudit, +})) +vi.mock('@/lib/api-key/crypto', () => ({ encryptApiKey: vi.fn() })) +vi.mock('@/lib/billing/core/subscription', () => ({ + hasWorkspaceLiveSyncAccess: mockHasWorkspaceLiveSyncAccess, +})) +vi.mock('@/lib/knowledge/connectors/queue', () => ({ dispatchSync: mockDispatchSync })) +vi.mock('@/lib/knowledge/documents/service', () => ({ + deleteDocumentStorageFiles: vi.fn().mockResolvedValue(undefined), +})) +vi.mock('@/lib/knowledge/tags/service', () => ({ + cleanupUnusedTagDefinitions: vi.fn().mockResolvedValue(undefined), + createTagDefinition: vi.fn(), +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) + +import { + performDeleteKnowledgeConnector, + performSyncKnowledgeConnector, + performUpdateKnowledgeConnector, +} from '@/lib/knowledge/orchestration/connectors' + +const KB = { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' } +const ACTOR = { userId: 'user-1', source: 'agent' as const, requestId: 'req-1' } +const BILLING = { actorUserId: 'user-1', workspaceId: 'ws-1' } as never +const resolveBillingAttribution = vi.fn().mockResolvedValue(BILLING) + +describe('performDeleteKnowledgeConnector', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + afterAll(resetDbChainMock) + + it('reports the documents it kept, so the caller cannot claim otherwise', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + queueTableRows(document, [ + { id: 'doc-1', fileUrl: '/a.txt' }, + { id: 'doc-2', fileUrl: '/b.txt' }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-1' }]) + + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + }) + + // The default keeps the documents. The copilot tool used to assert they had + // been removed while taking exactly this path. + expect(outcome).toMatchObject({ success: true, documentsKept: 2, documentsDeleted: 0 }) + expect(dbChainMockFns.delete).not.toHaveBeenCalledWith(document) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: expect.objectContaining({ deleteDocuments: false, documentsKept: 2 }), + }) + ) + }) + + it('reports the documents it deleted when asked to delete them', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + queueTableRows(document, [{ id: 'doc-1', fileUrl: '/a.txt' }]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-1' }]) + + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + deleteDocuments: true, + }) + + expect(outcome).toMatchObject({ success: true, documentsDeleted: 1, documentsKept: 0 }) + }) + + it('reports a missing connector as not found', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'not_found' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) +}) + +describe('performUpdateKnowledgeConnector', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + afterAll(resetDbChainMock) + + it('rejects an update that names nothing before reading the connector', async () => { + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: {}, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('classifies a sub-hourly interval on an unentitled workspace as forbidden', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + mockHasWorkspaceLiveSyncAccess.mockResolvedValue(false) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { syncIntervalMinutes: 5 }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'forbidden' }) + expect(mockHasWorkspaceLiveSyncAccess).toHaveBeenCalledWith('ws-1') + }) + + it('leaves a caller-supplied validator to reject a bad source config', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { sourceConfig: { database: 'gone' } }, + validateSourceConfig: async () => 'Database not found', + }) + + expect(outcome).toMatchObject({ + success: false, + errorCode: 'validation', + error: 'Database not found', + }) + }) + + it('clears the failure counters when a paused connector is resumed', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { status: 'active' }, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ consecutiveFailures: 0, lastSyncError: null }) + ) + }) +}) + +describe('performSyncKnowledgeConnector', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockDispatchSync.mockResolvedValue(undefined) + }) + + afterAll(resetDbChainMock) + + it('refuses to stack a sync on one already running', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'syncing' }, + ]) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + resolveBillingAttribution, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + // A rejected request never pays for the payer lookup. + expect(resolveBillingAttribution).not.toHaveBeenCalled() + expect(mockDispatchSync).not.toHaveBeenCalled() + }) + + it('dispatches and records who asked for it', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + resolveBillingAttribution, + rehydrate: true, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockDispatchSync).toHaveBeenCalledWith('conn-1', { + billingAttribution: BILLING, + requestId: 'req-1', + rehydrate: true, + }) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: 'user-1', + metadata: expect.objectContaining({ syncType: 'manual-rehydrate' }), + }) + ) + }) + + it('rejects a knowledge base with no workspace to bill', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: { ...KB, workspaceId: null }, + connectorId: 'conn-1', + resolveBillingAttribution, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts new file mode 100644 index 00000000000..c2ba8c86f1d --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -0,0 +1,708 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeBase, + knowledgeBaseTagDefinitions, + knowledgeConnector, +} from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import { encryptApiKey } from '@/lib/api-key/crypto' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' +import { allocateTagSlots } from '@/lib/knowledge/constants' +import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service' +import { + auditActorFields, + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { cleanupUnusedTagDefinitions, createTagDefinition } from '@/lib/knowledge/tags/service' +import { captureServerEvent } from '@/lib/posthog/server' + +const logger = createLogger('KnowledgeConnectorOrchestration') + +/** + * The connector registry and the sync queue are loaded on demand rather than + * imported at module scope. Both pull in every connector's SDK and the whole + * sync engine, and this module is re-exported from the knowledge orchestration + * barrel — a static edge would drag that graph into the bundle of every route + * that merely creates a knowledge base or uploads a document. + */ +async function loadDispatchSync() { + return (await import('@/lib/knowledge/connectors/queue')).dispatchSync +} + +/** A connector row exactly as stored, including its encrypted API key. */ +export type KnowledgeConnectorRow = typeof knowledgeConnector.$inferSelect +type ConnectorRow = KnowledgeConnectorRow +/** The connector row as it reaches every caller: never carrying the stored API key. */ +export type ConnectorWithoutSecret = Omit + +/** The knowledge base a connector operation targets, already authorized by the caller. */ +export interface ConnectorKnowledgeBase { + id: string + name: string + workspaceId: string | null +} + +function withoutSecret(row: ConnectorRow): ConnectorWithoutSecret { + const { encryptedApiKey: _encryptedApiKey, ...rest } = row + return rest +} + +/** + * Rejects a sub-hourly sync interval on a workspace without the plan for it. + * `0` disables scheduled syncs and is always allowed. + */ +async function assertLiveSyncAllowed( + workspaceId: string, + syncIntervalMinutes: number | undefined +): Promise { + if (syncIntervalMinutes === undefined || syncIntervalMinutes <= 0 || syncIntervalMinutes >= 60) { + return + } + if (!(await hasWorkspaceLiveSyncAccess(workspaceId))) { + throw new OrchestrationError('forbidden', 'Live sync requires a Max or Enterprise plan') + } +} + +export interface PerformCreateKnowledgeConnectorParams extends KnowledgeOperationContext { + knowledgeBase: ConnectorKnowledgeBase + connectorType: string + credentialId?: string + apiKey?: string + sourceConfig: Record + syncIntervalMinutes: number + /** + * Resolves the payer the sync is billed to. A thunk so a request rejected by + * a guard never pays for the lookup, and so the payer is read at the moment + * the sync is dispatched. + */ + resolveBillingAttribution: () => Promise + /** + * Resolves an OAuth credential to its access token. Supplied by the caller + * because credential lookup is scoped to the requesting identity. + */ + resolveAccessToken: (credentialId: string) => Promise +} + +export type PerformConnectorResult = KnowledgeOrchestrationResult<{ + connector: ConnectorWithoutSecret +}> + +/** + * Creates a connector on a knowledge base and dispatches its first sync. + * + * The tag-slot allocation and the connector insert share one transaction under + * the knowledge base's row lock, so a knowledge base archived mid-request can + * never end up with a live connector, and a partial slot allocation cannot + * outlive a failed insert. + */ +export async function performCreateKnowledgeConnector( + params: PerformCreateKnowledgeConnectorParams +): Promise { + const { + knowledgeBase: kb, + connectorType, + credentialId, + apiKey, + sourceConfig, + syncIntervalMinutes, + resolveBillingAttribution, + resolveAccessToken, + request, + source, + } = params + const requestId = params.requestId ?? generateRequestId() + + if (!kb.workspaceId) { + return fail('Knowledge base is missing workspace billing context', 'conflict') + } + const workspaceId = kb.workspaceId + + const { CONNECTOR_REGISTRY } = await import('@/connectors/registry.server') + const connectorConfig = CONNECTOR_REGISTRY[connectorType] + if (!connectorConfig) { + return fail(`Unknown connector type: ${connectorType}`, 'validation') + } + + try { + await assertLiveSyncAllowed(workspaceId, syncIntervalMinutes) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + + let resolvedCredentialId: string | null = null + let resolvedEncryptedApiKey: string | null = null + let accessToken: string + + if (connectorConfig.auth.mode === 'apiKey') { + if (!apiKey) { + return fail('API key is required', 'validation') + } + accessToken = apiKey + } else { + if (!credentialId) { + return fail('Credential is required', 'validation') + } + let token: string | null + try { + token = await resolveAccessToken(credentialId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + if (!token) { + return fail('Credential has no access token. Please reconnect your account.', 'validation') + } + accessToken = token + resolvedCredentialId = credentialId + } + + const configValidation = await connectorConfig.validateConfig(accessToken, sourceConfig) + if (!configValidation.valid) { + return fail(configValidation.error || 'Invalid source configuration', 'validation') + } + + if (connectorConfig.auth.mode === 'apiKey' && apiKey) { + resolvedEncryptedApiKey = (await encryptApiKey(apiKey)).encrypted + } + + let finalSourceConfig: Record = { ...sourceConfig } + const tagSlotMapping: Record = {} + let newTagSlots: Record = {} + + if (connectorConfig.tagDefinitions?.length) { + const disabledIds = new Set((sourceConfig.disabledTagIds as string[] | undefined) ?? []) + const enabledDefs = connectorConfig.tagDefinitions.filter((td) => !disabledIds.has(td.id)) + + const existingDefs = await db + .select({ + tagSlot: knowledgeBaseTagDefinitions.tagSlot, + displayName: knowledgeBaseTagDefinitions.displayName, + fieldType: knowledgeBaseTagDefinitions.fieldType, + }) + .from(knowledgeBaseTagDefinitions) + .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, kb.id)) + + const usedSlots = new Set(existingDefs.map((d) => d.tagSlot)) + const existingByName = new Map( + existingDefs.map((d) => [d.displayName, { tagSlot: d.tagSlot, fieldType: d.fieldType }]) + ) + + const defsNeedingSlots: typeof enabledDefs = [] + for (const td of enabledDefs) { + const existing = existingByName.get(td.displayName) + if (existing && existing.fieldType === td.fieldType) { + tagSlotMapping[td.id] = existing.tagSlot + } else { + defsNeedingSlots.push(td) + } + } + + const { mapping, skipped: skippedTags } = allocateTagSlots(defsNeedingSlots, usedSlots) + Object.assign(tagSlotMapping, mapping) + newTagSlots = mapping + + for (const name of skippedTags) { + logger.warn(`[${requestId}] No available slots for "${name}"`) + } + + if (skippedTags.length > 0 && Object.keys(tagSlotMapping).length === 0) { + return fail( + `No available tag slots. Could not assign: ${skippedTags.join(', ')}`, + 'validation' + ) + } + + finalSourceConfig = { ...finalSourceConfig, tagSlotMapping } + } + + const now = new Date() + const connectorId = generateId() + const nextSyncAt = + syncIntervalMinutes > 0 ? new Date(now.getTime() + syncIntervalMinutes * 60 * 1000) : null + + let created: ConnectorRow + try { + created = await db.transaction(async (tx) => { + await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${kb.id} FOR UPDATE`) + + const activeKb = await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where(and(eq(knowledgeBase.id, kb.id), isNull(knowledgeBase.deletedAt))) + .limit(1) + + if (activeKb.length === 0) { + throw new OrchestrationError('not_found', 'Knowledge base not found') + } + + for (const [semanticId, slot] of Object.entries(newTagSlots)) { + const td = connectorConfig.tagDefinitions?.find((d) => d.id === semanticId) + if (!td) continue + await createTagDefinition( + { + knowledgeBaseId: kb.id, + tagSlot: slot, + displayName: td.displayName, + fieldType: td.fieldType, + }, + requestId, + tx + ) + } + + const [row] = await tx + .insert(knowledgeConnector) + .values({ + id: connectorId, + knowledgeBaseId: kb.id, + connectorType, + credentialId: resolvedCredentialId, + encryptedApiKey: resolvedEncryptedApiKey, + sourceConfig: finalSourceConfig, + syncIntervalMinutes, + status: 'active', + nextSyncAt, + createdAt: now, + updatedAt: now, + }) + .returning() + + return row + }) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + + logger.info(`[${requestId}] Created connector ${connectorId} for KB ${kb.id}`) + + captureServerEvent( + params.userId, + 'knowledge_base_connector_added', + { + knowledge_base_id: kb.id, + workspace_id: workspaceId, + connector_type: connectorType, + sync_interval_minutes: syncIntervalMinutes, + }, + { + groups: { workspace: workspaceId }, + setOnce: { first_connector_added_at: new Date().toISOString() }, + } + ) + + recordAudit({ + workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_CREATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: connectorType, + description: `Created ${connectorType} connector for knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType, + syncIntervalMinutes, + authMode: connectorConfig.auth.mode, + }, + ...(request ? { request } : {}), + }) + + const billingAttribution = await resolveBillingAttribution() + const dispatchSync = await loadDispatchSync() + dispatchSync(connectorId, { billingAttribution, requestId }).catch((error) => { + logger.error( + `[${requestId}] Failed to dispatch initial sync for connector ${connectorId}`, + error + ) + }) + + return { success: true, connector: withoutSecret(created) } +} + +export interface PerformUpdateKnowledgeConnectorParams extends KnowledgeOperationContext { + knowledgeBase: ConnectorKnowledgeBase + connectorId: string + updates: { + sourceConfig?: Record + syncIntervalMinutes?: number + status?: 'active' | 'paused' + } + /** + * Validates a replacement `sourceConfig` against the live source. Supplied by + * the caller because resolving the connector's token needs the requesting + * identity. Returning a message rejects the update. + */ + validateSourceConfig?: ( + connector: KnowledgeConnectorRow, + sourceConfig: Record + ) => Promise +} + +/** Loads an active connector scoped to its knowledge base. */ +export async function getKnowledgeConnector( + knowledgeBaseId: string, + connectorId: string +): Promise { + const [row] = await db + .select() + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .limit(1) + + return row ?? null +} + +/** Applies a connector configuration change and records it against the actor. */ +export async function performUpdateKnowledgeConnector( + params: PerformUpdateKnowledgeConnectorParams +): Promise { + const { knowledgeBase: kb, connectorId, updates, validateSourceConfig, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + const updatedFields = Object.keys(updates).filter( + (key) => updates[key as keyof typeof updates] !== undefined + ) + if (updatedFields.length === 0) { + return fail( + 'At least one of sourceConfig, syncIntervalMinutes, or status is required', + 'validation' + ) + } + + const existing = await getKnowledgeConnector(kb.id, connectorId) + if (!existing) { + return fail('Connector not found', 'not_found') + } + + if (updates.syncIntervalMinutes !== undefined) { + if (!kb.workspaceId && updates.syncIntervalMinutes > 0 && updates.syncIntervalMinutes < 60) { + return fail('Knowledge base is missing workspace billing context', 'conflict') + } + if (kb.workspaceId) { + try { + await assertLiveSyncAllowed(kb.workspaceId, updates.syncIntervalMinutes) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Update connector ${connectorId}`) + } + } + } + + if (updates.sourceConfig !== undefined && validateSourceConfig) { + const rejection = await validateSourceConfig(existing, updates.sourceConfig) + if (rejection) { + return fail(rejection, 'validation') + } + } + + const values: Partial = { updatedAt: new Date() } + if (updates.sourceConfig !== undefined) { + values.sourceConfig = updates.sourceConfig + } + if (updates.syncIntervalMinutes !== undefined) { + values.syncIntervalMinutes = updates.syncIntervalMinutes + values.nextSyncAt = + updates.syncIntervalMinutes > 0 + ? new Date(Date.now() + updates.syncIntervalMinutes * 60 * 1000) + : null + } + if (updates.status !== undefined) { + values.status = updates.status + if (updates.status === 'active') { + values.consecutiveFailures = 0 + values.lastSyncError = null + // Resuming a paused connector syncs immediately unless this same request + // set a schedule, which then owns the next run. + if (values.nextSyncAt === undefined) { + values.nextSyncAt = new Date() + } + } + } + + let updated: ConnectorRow + try { + const [row] = await db + .update(knowledgeConnector) + .set(values) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, kb.id), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() + + if (!row) { + return fail('Connector not found', 'not_found') + } + updated = row + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Update connector ${connectorId}`) + } + + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_UPDATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: updated.connectorType, + description: `Updated connector for knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: updated.connectorType, + updatedFields, + ...(updates.syncIntervalMinutes !== undefined && { + syncIntervalMinutes: updates.syncIntervalMinutes, + }), + ...(updates.status !== undefined && { newStatus: updates.status }), + }, + ...(request ? { request } : {}), + }) + + return { success: true, connector: withoutSecret(updated) } +} + +export interface PerformDeleteKnowledgeConnectorParams extends KnowledgeOperationContext { + knowledgeBase: ConnectorKnowledgeBase + connectorId: string + /** + * Also hard-delete the documents the connector produced. Defaults to keeping + * them, which turns them into ordinary standalone knowledge base entries. + */ + deleteDocuments?: boolean +} + +/** What actually happened to the connector's documents, for the caller to report. */ +export type PerformDeleteKnowledgeConnectorResult = KnowledgeOrchestrationResult<{ + documentsDeleted: number + documentsKept: number +}> + +/** + * Hard-deletes a connector, either removing the documents it produced or + * releasing them as standalone entries. + * + * Returns the counts so callers state what happened rather than assert it. The + * copilot tool used to reach this through an internal HTTP self-call that sent + * no query string, so it always took the keep-documents default while telling + * the user the documents had been removed. + */ +export async function performDeleteKnowledgeConnector( + params: PerformDeleteKnowledgeConnectorParams +): Promise { + const { knowledgeBase: kb, connectorId, request, source } = params + const deleteDocuments = params.deleteDocuments ?? false + const requestId = params.requestId ?? generateRequestId() + + const existing = await getKnowledgeConnector(kb.id, connectorId) + if (!existing) { + return fail('Connector not found', 'not_found') + } + + let deletedDocs: Array<{ id: string; fileUrl: string }> + let docCount: number + try { + ;({ deletedDocs, docCount } = await db.transaction(async (tx) => { + await tx.execute(sql`SELECT 1 FROM knowledge_connector WHERE id = ${connectorId} FOR UPDATE`) + + // Includes pending-removal (tombstoned) docs — the connector is being + // deleted, so there's no future sync left to confirm or resurrect them. + const docs = await tx + .select({ id: document.id, fileUrl: document.fileUrl }) + .from(document) + .where(and(eq(document.connectorId, connectorId), isNull(document.archivedAt))) + + const documentIds = docs.map((doc) => doc.id) + if (deleteDocuments) { + if (documentIds.length > 0) { + await tx.delete(embedding).where(inArray(embedding.documentId, documentIds)) + await tx.delete(document).where(inArray(document.id, documentIds)) + } + } else if (documentIds.length > 0) { + // Kept documents become normal standalone KB entries once their connector + // is gone — resurrect any pending-removal ones rather than leaving them + // invisible tombstones with no future sync left to ever confirm or + // resurrect them. + await tx.update(document).set({ deletedAt: null }).where(inArray(document.id, documentIds)) + } + + const deletedConnectors = await tx + .delete(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, kb.id), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning({ id: knowledgeConnector.id }) + + if (deletedConnectors.length === 0) { + throw new OrchestrationError('not_found', 'Connector not found') + } + + return { deletedDocs: deleteDocuments ? docs : [], docCount: docs.length } + })) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Delete connector ${connectorId}`) + } + + if (deleteDocuments) { + await Promise.all([ + deletedDocs.length > 0 + ? deleteDocumentStorageFiles( + deletedDocs.map((doc) => ({ ...doc, workspaceId: kb.workspaceId })), + requestId + ) + : Promise.resolve(), + cleanupUnusedTagDefinitions(kb.id, requestId).catch((error) => { + logger.warn(`[${requestId}] Failed to cleanup tag definitions`, error) + }), + ]) + } + + logger.info( + `[${requestId}] Deleted connector ${connectorId}${deleteDocuments ? ` and ${docCount} documents` : `, kept ${docCount} documents`}` + ) + + captureServerEvent( + params.userId, + 'knowledge_base_connector_removed', + { + knowledge_base_id: kb.id, + workspace_id: kb.workspaceId ?? '', + connector_type: existing.connectorType, + documents_deleted: deleteDocuments ? docCount : 0, + }, + kb.workspaceId ? { groups: { workspace: kb.workspaceId } } : undefined + ) + + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_DELETED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: existing.connectorType, + description: `Deleted connector from knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: existing.connectorType, + deleteDocuments, + documentsDeleted: deleteDocuments ? docCount : 0, + documentsKept: deleteDocuments ? 0 : docCount, + }, + ...(request ? { request } : {}), + }) + + return { + success: true, + documentsDeleted: deleteDocuments ? docCount : 0, + documentsKept: deleteDocuments ? 0 : docCount, + } +} + +export interface PerformSyncKnowledgeConnectorParams extends KnowledgeOperationContext { + knowledgeBase: ConnectorKnowledgeBase + connectorId: string + /** + * Resolves the payer the sync is billed to. A thunk so a request rejected by + * a guard never pays for the lookup. + */ + resolveBillingAttribution: () => Promise + /** Re-fetch and re-index every already-synced document, not only changed ones. */ + rehydrate?: boolean +} + +export type PerformSyncKnowledgeConnectorResult = KnowledgeOrchestrationResult + +/** Triggers a manual sync for a connector and records who asked for it. */ +export async function performSyncKnowledgeConnector( + params: PerformSyncKnowledgeConnectorParams +): Promise { + const { knowledgeBase: kb, connectorId, resolveBillingAttribution, request, source } = params + const rehydrate = params.rehydrate ?? false + const requestId = params.requestId ?? generateRequestId() + + const connector = await getKnowledgeConnector(kb.id, connectorId) + if (!connector) { + return fail('Connector not found', 'not_found') + } + if (connector.status === 'syncing') { + return fail('Sync already in progress', 'conflict') + } + if (!kb.workspaceId) { + return fail('Knowledge base is missing workspace billing context', 'conflict') + } + const billingAttribution = await resolveBillingAttribution() + + logger.info( + `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` + ) + + captureServerEvent( + params.userId, + 'knowledge_base_connector_synced', + { + knowledge_base_id: kb.id, + workspace_id: kb.workspaceId ?? '', + connector_type: connector.connectorType, + }, + kb.workspaceId ? { groups: { workspace: kb.workspaceId } } : undefined + ) + + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_SYNCED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: connector.connectorType, + description: `Triggered manual sync for connector on knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: connector.connectorType, + connectorStatus: connector.status, + syncType: rehydrate ? 'manual-rehydrate' : 'manual', + }, + ...(request ? { request } : {}), + }) + + const dispatchSync = await loadDispatchSync() + dispatchSync(connectorId, { billingAttribution, requestId, rehydrate }).catch((error) => { + logger.error( + `[${requestId}] Failed to dispatch manual sync for connector ${connectorId}`, + error + ) + }) + + return { success: true } +} diff --git a/apps/sim/lib/knowledge/orchestration/documents.test.ts b/apps/sim/lib/knowledge/orchestration/documents.test.ts new file mode 100644 index 00000000000..bfbe1bdc403 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/documents.test.ts @@ -0,0 +1,329 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCaptureServerEvent, + mockCreateDocumentRecords, + mockCreateSingleDocument, + mockDeleteDocument, + mockMarkDocumentAsFailedTimeout, + mockProcessDocumentAsync, + mockProcessDocumentsWithQueue, + mockRecordAudit, + mockRetryDocumentProcessing, + mockUpdateDocument, +} = vi.hoisted(() => ({ + mockCaptureServerEvent: vi.fn(), + mockCreateDocumentRecords: vi.fn(), + mockCreateSingleDocument: vi.fn(), + mockDeleteDocument: vi.fn(), + mockMarkDocumentAsFailedTimeout: vi.fn(), + mockProcessDocumentAsync: vi.fn(), + mockProcessDocumentsWithQueue: vi.fn(), + mockRecordAudit: vi.fn(), + mockRetryDocumentProcessing: vi.fn(), + mockUpdateDocument: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { + DOCUMENT_UPLOADED: 'document.uploaded', + DOCUMENT_UPDATED: 'document.updated', + DOCUMENT_DELETED: 'document.deleted', + }, + AuditResourceType: { DOCUMENT: 'document' }, + recordAudit: mockRecordAudit, +})) +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { knowledgeBaseDocumentsUploaded: vi.fn() }, +})) +vi.mock('@/lib/knowledge/documents/service', () => ({ + createDocumentRecords: mockCreateDocumentRecords, + createSingleDocument: mockCreateSingleDocument, + deleteDocument: mockDeleteDocument, + markDocumentAsFailedTimeout: mockMarkDocumentAsFailedTimeout, + processDocumentAsync: mockProcessDocumentAsync, + processDocumentsWithQueue: mockProcessDocumentsWithQueue, + retryDocumentProcessing: mockRetryDocumentProcessing, + updateDocument: mockUpdateDocument, +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + performDeleteKnowledgeDocument, + performMarkKnowledgeDocumentTimedOut, + performRetryKnowledgeDocumentProcessing, + performUpdateKnowledgeDocument, + performUploadKnowledgeDocument, + performUploadKnowledgeDocuments, +} from '@/lib/knowledge/orchestration/documents' + +const KB = { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' } +const FILE = { + filename: 'report.pdf', + fileUrl: 'https://storage/report.pdf', + fileSize: 1024, + mimeType: 'application/pdf', +} +const ACTOR = { userId: 'user-1', source: 'agent' as const, requestId: 'req-1' } + +describe('performUploadKnowledgeDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateSingleDocument.mockResolvedValue({ id: 'doc-1', filename: 'report.pdf' }) + mockProcessDocumentsWithQueue.mockResolvedValue(undefined) + mockProcessDocumentAsync.mockResolvedValue(undefined) + }) + + it('audits an agent upload, which the copilot path never did', async () => { + const outcome = await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: 'user-1', + resourceId: 'doc-1', + resourceName: 'report.pdf', + metadata: expect.objectContaining({ source: 'agent', knowledgeBaseId: 'kb-1' }), + }) + ) + }) + + it('records the document owner the caller names, not the acting user', async () => { + // A workspace API key bills and owns as the workspace account, while the + // acting user stays the audit actor. + await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + uploadedBy: 'workspace-owner', + }) + + expect(mockCreateSingleDocument).toHaveBeenCalledWith(FILE, 'kb-1', 'req-1', 'workspace-owner') + }) + + it('starts no indexing unless the caller asks for it', async () => { + await performUploadKnowledgeDocument({ ...ACTOR, knowledgeBase: KB, document: FILE }) + + expect(mockProcessDocumentsWithQueue).not.toHaveBeenCalled() + expect(mockProcessDocumentAsync).not.toHaveBeenCalled() + }) + + it.each([ + { startProcessing: 'queue' as const, expected: mockProcessDocumentsWithQueue }, + { startProcessing: 'async' as const, expected: mockProcessDocumentAsync }, + ])('hands the record to the $startProcessing pipeline', async ({ startProcessing, expected }) => { + await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + startProcessing, + }) + + expect(expected).toHaveBeenCalled() + }) + + it('classifies a storage-quota rejection as too large, by class not message', async () => { + mockCreateSingleDocument.mockRejectedValue( + new OrchestrationError('payload_too_large', 'Storage limit exceeded. Used: 5.10GB') + ) + + const outcome = await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'payload_too_large' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) + + it('classifies a foreign file reference as forbidden', async () => { + mockCreateSingleDocument.mockRejectedValue( + new OrchestrationError('forbidden', 'Document file is not owned by this knowledge base') + ) + + expect( + (await performUploadKnowledgeDocument({ ...ACTOR, knowledgeBase: KB, document: FILE })) + .errorCode + ).toBe('forbidden') + }) +}) + +describe('performUploadKnowledgeDocuments', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateDocumentRecords.mockResolvedValue([ + { documentId: 'doc-1', filename: 'a.pdf' }, + { documentId: 'doc-2', filename: 'b.pdf' }, + ]) + mockProcessDocumentsWithQueue.mockResolvedValue(undefined) + }) + + it('admits the whole batch in one call and queues it', async () => { + const outcome = await performUploadKnowledgeDocuments({ + ...ACTOR, + knowledgeBase: KB, + documents: [FILE, { ...FILE, filename: 'b.pdf' }], + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockCreateDocumentRecords).toHaveBeenCalledTimes(1) + expect(mockProcessDocumentsWithQueue).toHaveBeenCalledTimes(1) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ resourceName: '2 document(s)' }) + ) + }) + + it('rejects an empty batch before touching the service', async () => { + const outcome = await performUploadKnowledgeDocuments({ + ...ACTOR, + knowledgeBase: KB, + documents: [], + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockCreateDocumentRecords).not.toHaveBeenCalled() + }) +}) + +describe('performUpdateKnowledgeDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + mockUpdateDocument.mockResolvedValue({ id: 'doc-1', filename: 'renamed.pdf' }) + }) + + it('rejects an update that names nothing before touching the service', async () => { + const outcome = await performUpdateKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: { id: 'doc-1', filename: 'report.pdf' }, + updates: { filename: undefined, enabled: undefined }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockUpdateDocument).not.toHaveBeenCalled() + }) + + it('names the changed fields in the audit metadata', async () => { + await performUpdateKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: { id: 'doc-1', filename: 'report.pdf' }, + updates: { filename: 'renamed.pdf', enabled: false }, + }) + + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + resourceName: 'renamed.pdf', + metadata: expect.objectContaining({ + updatedFields: ['filename', 'enabled'], + enabled: false, + }), + }) + ) + }) +}) + +describe('performDeleteKnowledgeDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDeleteDocument.mockResolvedValue({ success: true, message: 'ok' }) + }) + + it('audits the deletion against the acting user', async () => { + const outcome = await performDeleteKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: { id: 'doc-1', filename: 'report.pdf', fileSize: 10, mimeType: 'application/pdf' }, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockDeleteDocument).toHaveBeenCalledWith('doc-1', 'req-1') + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ actorId: 'user-1', resourceId: 'doc-1' }) + ) + expect(mockCaptureServerEvent).toHaveBeenCalled() + }) + + it('emits no telemetry when the delete fails', async () => { + mockDeleteDocument.mockRejectedValue(new Error('deadlock detected')) + + const outcome = await performDeleteKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: { id: 'doc-1', filename: 'report.pdf' }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) +}) + +describe('document processing state changes', () => { + beforeEach(() => vi.clearAllMocks()) + + it('refuses to time out a document that is not processing', async () => { + const outcome = await performMarkKnowledgeDocumentTimedOut({ + document: { id: 'doc-1', processingStatus: 'completed', processingStartedAt: new Date() }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockMarkDocumentAsFailedTimeout).not.toHaveBeenCalled() + }) + + it('refuses to time out a document with no processing start time', async () => { + const outcome = await performMarkKnowledgeDocumentTimedOut({ + document: { id: 'doc-1', processingStatus: 'processing', processingStartedAt: null }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + }) + + it('surfaces a too-soon timeout as caller-fixable, not a fault', async () => { + mockMarkDocumentAsFailedTimeout.mockRejectedValue( + new Error('Document has not been processing long enough to be considered dead') + ) + + const outcome = await performMarkKnowledgeDocumentTimedOut({ + document: { id: 'doc-1', processingStatus: 'processing', processingStartedAt: new Date() }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + }) + + it('refuses to retry a document that has not failed', async () => { + const outcome = await performRetryKnowledgeDocumentProcessing({ + knowledgeBaseId: 'kb-1', + document: { ...FILE, id: 'doc-1', processingStatus: 'completed' }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockRetryDocumentProcessing).not.toHaveBeenCalled() + }) + + it('re-queues a failed document and never audits it', async () => { + mockRetryDocumentProcessing.mockResolvedValue({ + success: true, + status: 'pending', + message: 'Document retry processing started', + }) + + const outcome = await performRetryKnowledgeDocumentProcessing({ + knowledgeBaseId: 'kb-1', + document: { ...FILE, id: 'doc-1', processingStatus: 'failed' }, + requestId: 'req-1', + }) + + expect(outcome).toMatchObject({ success: true, status: 'pending' }) + // No document state the user chose changes, so there is nothing to record. + expect(mockRecordAudit).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/documents.ts b/apps/sim/lib/knowledge/orchestration/documents.ts new file mode 100644 index 00000000000..d0111d53289 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/documents.ts @@ -0,0 +1,486 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { PlatformEvents } from '@/lib/core/telemetry' +import { generateRequestId } from '@/lib/core/utils/request' +import { + createDocumentRecords, + createSingleDocument, + type DocumentData, + deleteDocument, + markDocumentAsFailedTimeout, + type ProcessingOptions, + processDocumentAsync, + processDocumentsWithQueue, + retryDocumentProcessing, + updateDocument, +} from '@/lib/knowledge/documents/service' +import { + auditActorFields, + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { captureServerEvent } from '@/lib/posthog/server' + +const logger = createLogger('KnowledgeDocumentOrchestration') + +/** The knowledge base a document operation targets, already authorized by the caller. */ +export interface KnowledgeBaseTarget { + id: string + name?: string | null + workspaceId: string | null +} + +export interface KnowledgeDocumentInput { + filename: string + fileUrl: string + fileSize: number + mimeType: string + documentTagsData?: string + tag1?: string + tag2?: string + tag3?: string + tag4?: string + tag5?: string + tag6?: string + tag7?: string +} + +/** + * How the created record is handed to the indexing pipeline. + * + * `queue` runs the shared bounded-concurrency queue; `async` starts one + * detached processing run. Omitted starts nothing — the internal single-document + * route deliberately only creates the row and leaves indexing to its caller. + */ +export type KnowledgeDocumentProcessing = 'queue' | 'async' + +export type CreatedKnowledgeDocument = Awaited> + +export interface PerformUploadKnowledgeDocumentParams extends KnowledgeOperationContext { + knowledgeBase: KnowledgeBaseTarget + document: KnowledgeDocumentInput + startProcessing?: KnowledgeDocumentProcessing + processingOptions?: ProcessingOptions + billingAttribution?: BillingAttributionSnapshot + /** Row owner recorded on the document; defaults to the acting user. */ + uploadedBy?: string | null +} + +export type PerformUploadKnowledgeDocumentResult = KnowledgeOrchestrationResult<{ + document: CreatedKnowledgeDocument +}> + +function auditUpload( + params: KnowledgeOperationContext & { knowledgeBase: KnowledgeBaseTarget }, + entry: { resourceId: string; resourceName: string; description: string; metadata: object } +) { + recordAudit({ + workspaceId: params.knowledgeBase.workspaceId, + ...auditActorFields(params), + action: AuditAction.DOCUMENT_UPLOADED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: entry.resourceId, + resourceName: entry.resourceName, + description: entry.description, + metadata: { + source: params.source, + knowledgeBaseId: params.knowledgeBase.id, + knowledgeBaseName: params.knowledgeBase.name, + ...entry.metadata, + }, + ...(params.request ? { request: params.request } : {}), + }) +} + +function captureUpload( + params: KnowledgeOperationContext & { knowledgeBase: KnowledgeBaseTarget }, + documentCount: number, + uploadType: 'single' | 'bulk' +) { + const workspaceId = params.knowledgeBase.workspaceId + captureServerEvent( + params.userId, + 'knowledge_base_document_uploaded', + { + knowledge_base_id: params.knowledgeBase.id, + workspace_id: workspaceId ?? '', + document_count: documentCount, + upload_type: uploadType, + }, + { + ...(workspaceId ? { groups: { workspace: workspaceId } } : {}), + setOnce: { first_document_uploaded_at: new Date().toISOString() }, + } + ) +} + +/** + * Adds one already-resolved file to a knowledge base. + * + * Each surface acquires the file differently — a multipart body uploaded to + * workspace storage, a virtual-filesystem reference resolved to a presigned URL, + * a client-supplied URL — so acquisition stays with the caller and this takes + * the resolved `{filename, fileUrl, fileSize, mimeType}`. Everything downstream + * of that (the record, the indexing hand-off, telemetry, and the audit) is + * identical for all of them, which is why the copilot path used to index + * documents that appear nowhere in the audit log. + */ +export async function performUploadKnowledgeDocument( + params: PerformUploadKnowledgeDocumentParams +): Promise { + const { knowledgeBase, document, startProcessing, processingOptions, billingAttribution } = params + const requestId = params.requestId ?? generateRequestId() + + let created: CreatedKnowledgeDocument + try { + created = await createSingleDocument( + document, + knowledgeBase.id, + requestId, + params.uploadedBy ?? params.userId + ) + } catch (error) { + return classifyKnowledgeFailure( + error, + requestId, + `Upload document "${document.filename}" to knowledge base ${knowledgeBase.id}` + ) + } + + const documentData: DocumentData = { + documentId: created.id, + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + } + + if (startProcessing === 'queue') { + processDocumentsWithQueue( + [documentData], + knowledgeBase.id, + processingOptions ?? {}, + requestId, + billingAttribution + ).catch((error: unknown) => { + logger.error(`[${requestId}] Document processing pipeline failed`, { error }) + }) + } else if (startProcessing === 'async') { + processDocumentAsync( + knowledgeBase.id, + created.id, + document, + processingOptions ?? {}, + billingAttribution + ).catch((error: unknown) => { + logger.error(`[${requestId}] Background document processing failed`, { + documentId: created.id, + error: toError(error).message, + }) + }) + } + + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: knowledgeBase.id, + documentsCount: 1, + uploadType: 'single', + mimeType: document.mimeType, + fileSize: document.fileSize, + }) + captureUpload(params, 1, 'single') + + auditUpload(params, { + resourceId: created.id, + resourceName: document.filename, + description: `Uploaded document "${document.filename}" to knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { + fileName: document.filename, + fileType: document.mimeType, + fileSize: document.fileSize, + }, + }) + + return { success: true, document: created } +} + +export interface PerformUploadKnowledgeDocumentsParams extends KnowledgeOperationContext { + knowledgeBase: KnowledgeBaseTarget + documents: KnowledgeDocumentInput[] + processingOptions?: ProcessingOptions + billingAttribution?: BillingAttributionSnapshot + uploadedBy?: string | null +} + +export type PerformUploadKnowledgeDocumentsResult = KnowledgeOrchestrationResult<{ + documents: DocumentData[] +}> + +/** + * Adds many files to a knowledge base in one storage admission and hands the + * whole set to the bounded-concurrency processing queue. + * + * Kept separate from the single-document path because `createDocumentRecords` + * admits the batch's bytes as one unit; running it per document would let a set + * that exceeds the quota commit its first half. + */ +export async function performUploadKnowledgeDocuments( + params: PerformUploadKnowledgeDocumentsParams +): Promise { + const { knowledgeBase, documents, processingOptions, billingAttribution } = params + const requestId = params.requestId ?? generateRequestId() + + if (documents.length === 0) { + return fail('No documents specified', 'validation') + } + + let created: DocumentData[] + try { + created = await createDocumentRecords( + documents, + knowledgeBase.id, + requestId, + params.uploadedBy ?? params.userId + ) + } catch (error) { + return classifyKnowledgeFailure( + error, + requestId, + `Upload ${documents.length} document(s) to knowledge base ${knowledgeBase.id}` + ) + } + + logger.info(`[${requestId}] Starting controlled async processing of ${created.length} documents`) + + processDocumentsWithQueue( + created, + knowledgeBase.id, + processingOptions ?? {}, + requestId, + billingAttribution + ).catch((error: unknown) => { + logger.error(`[${requestId}] Critical error in document processing pipeline`, { error }) + }) + + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: knowledgeBase.id, + documentsCount: created.length, + uploadType: 'bulk', + recipe: processingOptions?.recipe, + }) + captureUpload(params, created.length, 'bulk') + + auditUpload(params, { + resourceId: knowledgeBase.id, + resourceName: `${created.length} document(s)`, + description: `Uploaded ${created.length} document(s) to knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { fileCount: created.length }, + }) + + return { success: true, documents: created } +} + +export interface PerformUpdateKnowledgeDocumentParams extends KnowledgeOperationContext { + knowledgeBase: KnowledgeBaseTarget + document: { id: string; filename: string } + updates: Parameters[1] +} + +export type PerformUpdateKnowledgeDocumentResult = KnowledgeOrchestrationResult<{ + document: Awaited> +}> + +/** Renames a document, toggles it, or edits its tags, and records the change. */ +export async function performUpdateKnowledgeDocument( + params: PerformUpdateKnowledgeDocumentParams +): Promise { + const { knowledgeBase, document, updates, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + const updatedFields = Object.keys(updates).filter( + (key) => updates[key as keyof typeof updates] !== undefined + ) + if (updatedFields.length === 0) { + return fail('No updates specified', 'validation') + } + + let updated: Awaited> + try { + updated = await updateDocument(document.id, updates, requestId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Update document ${document.id}`) + } + + const filename = updates.filename ?? document.filename + + recordAudit({ + workspaceId: knowledgeBase.workspaceId, + ...auditActorFields(params), + action: AuditAction.DOCUMENT_UPDATED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: document.id, + resourceName: filename, + description: `Updated document "${filename}" in knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { + source, + knowledgeBaseId: knowledgeBase.id, + knowledgeBaseName: knowledgeBase.name, + fileName: filename, + updatedFields, + ...(updates.enabled !== undefined && { enabled: updates.enabled }), + }, + ...(request ? { request } : {}), + }) + + return { success: true, document: updated } +} + +export interface PerformDeleteKnowledgeDocumentParams extends KnowledgeOperationContext { + knowledgeBase: KnowledgeBaseTarget + document: { id: string; filename: string; fileSize?: number; mimeType?: string } +} + +export type PerformDeleteKnowledgeDocumentResult = KnowledgeOrchestrationResult + +/** Deletes a document and its embeddings, and records the deletion. */ +export async function performDeleteKnowledgeDocument( + params: PerformDeleteKnowledgeDocumentParams +): Promise { + const { knowledgeBase, document, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + try { + await deleteDocument(document.id, requestId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Delete document ${document.id}`) + } + + logger.info( + `[${requestId}] Deleted document ${document.id} from knowledge base ${knowledgeBase.id}` + ) + + recordAudit({ + workspaceId: knowledgeBase.workspaceId, + ...auditActorFields(params), + action: AuditAction.DOCUMENT_DELETED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: document.id, + resourceName: document.filename, + description: `Deleted document "${document.filename}" from knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { + source, + knowledgeBaseId: knowledgeBase.id, + knowledgeBaseName: knowledgeBase.name, + fileName: document.filename, + fileSize: document.fileSize, + mimeType: document.mimeType, + }, + ...(request ? { request } : {}), + }) + + const workspaceId = knowledgeBase.workspaceId + captureServerEvent( + params.userId, + 'knowledge_base_document_deleted', + { knowledge_base_id: knowledgeBase.id, workspace_id: workspaceId ?? '' }, + workspaceId ? { groups: { workspace: workspaceId } } : undefined + ) + + return { success: true } +} + +export interface PerformMarkKnowledgeDocumentTimedOutParams { + document: { + id: string + processingStatus: string + processingStartedAt?: Date | null + } + requestId?: string +} + +export type PerformKnowledgeDocumentProcessingResult = KnowledgeOrchestrationResult<{ + status: string + message: string +}> + +/** + * Marks a document whose processing run died as failed. Not audited: the state + * change is the system conceding a run it lost, not a user editing a document. + */ +export async function performMarkKnowledgeDocumentTimedOut( + params: PerformMarkKnowledgeDocumentTimedOutParams +): Promise { + const { document } = params + const requestId = params.requestId ?? generateRequestId() + + if (document.processingStatus !== 'processing') { + return fail( + `Document is not in processing state (current: ${document.processingStatus})`, + 'validation' + ) + } + if (!document.processingStartedAt) { + return fail('Document has no processing start time', 'validation') + } + + try { + await markDocumentAsFailedTimeout(document.id, document.processingStartedAt, requestId) + } catch (error) { + // The service rejects a document that has not been processing long enough + // to be presumed dead; that is a caller-fixable "try again later", not a fault. + if (!(error instanceof OrchestrationError)) { + return fail(toError(error).message, 'validation') + } + return classifyKnowledgeFailure(error, requestId, `Time out document ${document.id}`) + } + + return { success: true, status: 'failed', message: 'Document marked as failed due to timeout' } +} + +export interface PerformRetryKnowledgeDocumentParams { + knowledgeBaseId: string + document: { + id: string + filename: string + fileUrl: string + fileSize: number + mimeType: string + processingStatus: string + } + billingAttribution?: BillingAttributionSnapshot + requestId?: string +} + +/** Re-queues a failed document for indexing. Not audited: no document state a user chose changes. */ +export async function performRetryKnowledgeDocumentProcessing( + params: PerformRetryKnowledgeDocumentParams +): Promise { + const { knowledgeBaseId, document, billingAttribution } = params + const requestId = params.requestId ?? generateRequestId() + + if (document.processingStatus !== 'failed') { + return fail('Document is not in failed state', 'validation') + } + + try { + const result = await retryDocumentProcessing( + knowledgeBaseId, + document.id, + { + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + }, + requestId, + billingAttribution + ) + return { success: true, status: result.status, message: result.message } + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Retry document ${document.id}`) + } +} diff --git a/apps/sim/lib/knowledge/orchestration/index.ts b/apps/sim/lib/knowledge/orchestration/index.ts index dc44ac10ee9..afef6127d04 100644 --- a/apps/sim/lib/knowledge/orchestration/index.ts +++ b/apps/sim/lib/knowledge/orchestration/index.ts @@ -1,88 +1,33 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { db } from '@sim/db' -import { knowledgeBase } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { eq } from 'drizzle-orm' -import { generateRequestId } from '@/lib/core/utils/request' -import { KnowledgeBaseConflictError, restoreKnowledgeBase } from '@/lib/knowledge/service' - -const logger = createLogger('KnowledgeBaseOrchestration') - -export type KnowledgeOrchestrationErrorCode = 'not_found' | 'conflict' | 'internal' - -export interface RestorableKnowledgeBase { - id: string - name: string - workspaceId: string | null - userId: string -} - -export interface PerformRestoreKnowledgeBaseParams { - knowledgeBaseId: string - userId: string - requestId?: string -} - -export interface PerformRestoreKnowledgeBaseResult { - success: boolean - error?: string - errorCode?: KnowledgeOrchestrationErrorCode - knowledgeBase?: RestorableKnowledgeBase -} - -export async function getRestorableKnowledgeBase( - knowledgeBaseId: string -): Promise { - const [kb] = await db - .select({ - id: knowledgeBase.id, - name: knowledgeBase.name, - workspaceId: knowledgeBase.workspaceId, - userId: knowledgeBase.userId, - }) - .from(knowledgeBase) - .where(eq(knowledgeBase.id, knowledgeBaseId)) - .limit(1) - - return kb ?? null -} - -export async function performRestoreKnowledgeBase( - params: PerformRestoreKnowledgeBaseParams -): Promise { - const { knowledgeBaseId, userId } = params - const requestId = params.requestId ?? generateRequestId() - - const kb = await getRestorableKnowledgeBase(knowledgeBaseId) - if (!kb) { - return { success: false, error: 'Knowledge base not found', errorCode: 'not_found' } - } - - try { - await restoreKnowledgeBase(knowledgeBaseId, requestId) - - logger.info(`[${requestId}] Restored knowledge base ${knowledgeBaseId}`) - - recordAudit({ - workspaceId: kb.workspaceId, - actorId: userId, - action: AuditAction.KNOWLEDGE_BASE_RESTORED, - resourceType: AuditResourceType.KNOWLEDGE_BASE, - resourceId: knowledgeBaseId, - resourceName: kb.name, - description: `Restored knowledge base "${kb.name}"`, - metadata: { - knowledgeBaseName: kb.name, - }, - }) - - return { success: true, knowledgeBase: kb } - } catch (error) { - logger.error(`[${requestId}] Failed to restore knowledge base ${knowledgeBaseId}`, { error }) - if (error instanceof KnowledgeBaseConflictError) { - return { success: false, error: error.message, errorCode: 'conflict' } - } - return { success: false, error: toError(error).message, errorCode: 'internal' } - } -} +export { + type ConnectorKnowledgeBase, + type ConnectorWithoutSecret, + getKnowledgeConnector, + type KnowledgeConnectorRow, + performCreateKnowledgeConnector, + performDeleteKnowledgeConnector, + performSyncKnowledgeConnector, + performUpdateKnowledgeConnector, +} from './connectors' +export { + type CreatedKnowledgeDocument, + type KnowledgeBaseTarget, + type KnowledgeDocumentInput, + performDeleteKnowledgeDocument, + performMarkKnowledgeDocumentTimedOut, + performRetryKnowledgeDocumentProcessing, + performUpdateKnowledgeDocument, + performUploadKnowledgeDocument, + performUploadKnowledgeDocuments, +} from './documents' +export { + type PerformKnowledgeBaseResult, + performCreateKnowledgeBase, + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from './knowledge-bases' +export { + getRestorableKnowledgeBase, + performRestoreKnowledgeBase, + type RestorableKnowledgeBase, +} from './restore' +export type { KnowledgeActor, KnowledgeOperationSource } from './shared' diff --git a/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts b/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts new file mode 100644 index 00000000000..6aed845a9be --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts @@ -0,0 +1,256 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCaptureServerEvent, + mockCreateKnowledgeBase, + mockDeleteKnowledgeBase, + mockRecordAudit, + mockUpdateKnowledgeBase, +} = vi.hoisted(() => ({ + mockCaptureServerEvent: vi.fn(), + mockCreateKnowledgeBase: vi.fn(), + mockDeleteKnowledgeBase: vi.fn(), + mockRecordAudit: vi.fn(), + mockUpdateKnowledgeBase: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { + KNOWLEDGE_BASE_CREATED: 'knowledge_base.created', + KNOWLEDGE_BASE_UPDATED: 'knowledge_base.updated', + KNOWLEDGE_BASE_DELETED: 'knowledge_base.deleted', + }, + AuditResourceType: { KNOWLEDGE_BASE: 'knowledge_base' }, + recordAudit: mockRecordAudit, +})) +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { knowledgeBaseCreated: vi.fn(), knowledgeBaseDeleted: vi.fn() }, +})) +vi.mock('@/lib/knowledge/embeddings', () => ({ + EMBEDDING_DIMENSIONS: 1536, + getConfiguredEmbeddingModel: () => 'text-embedding-3-small', +})) +vi.mock('@/lib/knowledge/service', () => ({ + createKnowledgeBase: mockCreateKnowledgeBase, + deleteKnowledgeBase: mockDeleteKnowledgeBase, + updateKnowledgeBase: mockUpdateKnowledgeBase, +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { DEFAULT_CHUNKING_CONFIG } from '@/lib/knowledge/constants' +import { + performCreateKnowledgeBase, + performDeleteKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration/knowledge-bases' + +const CREATED = { id: 'kb-1', name: 'Docs', description: null, workspaceId: 'ws-1' } + +describe('performCreateKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateKnowledgeBase.mockResolvedValue(CREATED) + }) + + it('applies one chunking default for every caller', async () => { + await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'agent', + workspaceId: 'ws-1', + name: 'Docs', + }) + + // The agent used to default minSize to 1 against the API's 100, so the same + // document chunked differently depending on who created the knowledge base. + expect(mockCreateKnowledgeBase).toHaveBeenCalledWith( + expect.objectContaining({ chunkingConfig: { ...DEFAULT_CHUNKING_CONFIG } }), + expect.any(String) + ) + }) + + it('lets a caller override individual chunking fields', async () => { + await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'api', + workspaceId: 'ws-1', + name: 'Docs', + chunkingConfig: { maxSize: 512 }, + }) + + expect(mockCreateKnowledgeBase).toHaveBeenCalledWith( + expect.objectContaining({ + chunkingConfig: { ...DEFAULT_CHUNKING_CONFIG, maxSize: 512 }, + }), + expect.any(String) + ) + }) + + it('audits an agent-created knowledge base, which the copilot path never did', async () => { + await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'agent', + workspaceId: 'ws-1', + name: 'Docs', + }) + + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: 'user-1', + resourceId: 'kb-1', + workspaceId: 'ws-1', + metadata: expect.objectContaining({ source: 'agent' }), + }) + ) + }) + + it('carries request provenance into the audit row', async () => { + const request = new Request('https://sim.ai', { headers: { 'user-agent': 'curl/8' } }) + + await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'ui', + workspaceId: 'ws-1', + name: 'Docs', + request, + }) + + expect(mockRecordAudit).toHaveBeenCalledWith(expect.objectContaining({ request })) + }) + + it('classifies a duplicate name as a conflict, not bad input', async () => { + mockCreateKnowledgeBase.mockRejectedValue( + new OrchestrationError('conflict', 'A knowledge base named "Docs" already exists') + ) + + const outcome = await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'ui', + workspaceId: 'ws-1', + name: 'Docs', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) + + it('keeps an unclassified failure internal and records nothing', async () => { + mockCreateKnowledgeBase.mockRejectedValue(new Error('connection terminated')) + + const outcome = await performCreateKnowledgeBase({ + userId: 'user-1', + source: 'ui', + workspaceId: 'ws-1', + name: 'Docs', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) +}) + +describe('performUpdateKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + mockUpdateKnowledgeBase.mockResolvedValue({ ...CREATED, name: 'Renamed' }) + }) + + it('rejects an update that names nothing before touching the service', async () => { + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: 'kb-1', + workspaceId: 'ws-1', + userId: 'user-1', + source: 'api', + updates: { name: undefined, description: undefined }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockUpdateKnowledgeBase).not.toHaveBeenCalled() + }) + + it('always forwards the actor, so a workspace move is authorized not rejected', async () => { + await performUpdateKnowledgeBase({ + knowledgeBaseId: 'kb-1', + workspaceId: 'ws-1', + userId: 'user-1', + source: 'api', + updates: { workspaceId: 'ws-2' }, + }) + + // The v1 and v2 routes used to omit `actorUserId`, which the service rejects + // outright on a workspace change. + expect(mockUpdateKnowledgeBase).toHaveBeenCalledWith( + 'kb-1', + { workspaceId: 'ws-2' }, + expect.any(String), + { actorUserId: 'user-1' } + ) + }) + + it('files the audit against the destination workspace on a move', async () => { + await performUpdateKnowledgeBase({ + knowledgeBaseId: 'kb-1', + workspaceId: 'ws-1', + userId: 'user-1', + source: 'ui', + updates: { workspaceId: 'ws-2' }, + }) + + expect(mockRecordAudit).toHaveBeenCalledWith(expect.objectContaining({ workspaceId: 'ws-2' })) + }) + + it('classifies a rejected folder as bad input', async () => { + mockUpdateKnowledgeBase.mockRejectedValue( + new OrchestrationError('validation', 'Folder not found in this workspace') + ) + + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: 'kb-1', + workspaceId: 'ws-1', + userId: 'user-1', + source: 'ui', + updates: { folderId: 'folder-elsewhere' }, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + }) +}) + +describe('performDeleteKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDeleteKnowledgeBase.mockResolvedValue(undefined) + }) + + it('audits the archive against the acting user', async () => { + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' }, + userId: 'user-1', + source: 'agent', + requestId: 'req-1', + }) + + expect(outcome.success).toBe(true) + expect(mockDeleteKnowledgeBase).toHaveBeenCalledWith('kb-1', 'req-1') + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ actorId: 'user-1', resourceId: 'kb-1' }) + ) + }) + + it('records nothing when the archive fails', async () => { + mockDeleteKnowledgeBase.mockRejectedValue(new Error('deadlock detected')) + + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' }, + userId: 'user-1', + source: 'ui', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts b/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts new file mode 100644 index 00000000000..b808f820e97 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts @@ -0,0 +1,236 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { createLogger } from '@sim/logger' +import { PlatformEvents } from '@/lib/core/telemetry' +import { generateRequestId } from '@/lib/core/utils/request' +import { DEFAULT_CHUNKING_CONFIG } from '@/lib/knowledge/constants' +import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' +import { + auditActorFields, + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { + createKnowledgeBase, + deleteKnowledgeBase, + updateKnowledgeBase, +} from '@/lib/knowledge/service' +import type { ChunkingConfig, KnowledgeBaseWithCounts } from '@/lib/knowledge/types' +import { captureServerEvent } from '@/lib/posthog/server' + +const logger = createLogger('KnowledgeBaseOrchestration') + +export type PerformKnowledgeBaseResult = KnowledgeOrchestrationResult<{ + knowledgeBase: KnowledgeBaseWithCounts +}> + +export interface PerformCreateKnowledgeBaseParams extends KnowledgeOperationContext { + workspaceId: string + name: string + description?: string + /** Folder in the workspace's `knowledge_base` tree; `null`/omitted is the root. */ + folderId?: string | null + /** Omitted fields fall back to {@link DEFAULT_CHUNKING_CONFIG}. */ + chunkingConfig?: Partial +} + +/** + * Creates a knowledge base on behalf of an actor, as the single implementation + * behind the UI route, the v1 and v2 public APIs, and the copilot agent tool. + * + * The chunking default lives here rather than at each boundary: when every + * caller carried its own literal the agent's `minSize` was 1 against the API's + * 100, so identical input produced differently-chunked knowledge bases + * depending on who created it. The audit likewise lives here rather than in the + * three HTTP routes that each had their own copy — which is why an + * agent-created knowledge base used to leave no audit trail at all. + * + * The caller owns authentication; `createKnowledgeBase` still enforces the + * workspace write permission itself, and that failure comes back as `forbidden`. + */ +export async function performCreateKnowledgeBase( + params: PerformCreateKnowledgeBaseParams +): Promise { + const { workspaceId, name, description, folderId, request, source } = params + const requestId = params.requestId ?? generateRequestId() + const chunkingConfig: ChunkingConfig = { ...DEFAULT_CHUNKING_CONFIG, ...params.chunkingConfig } + const embeddingModel = getConfiguredEmbeddingModel() + + let created: KnowledgeBaseWithCounts + try { + created = await createKnowledgeBase( + { + name, + description, + workspaceId, + folderId, + userId: params.userId, + embeddingModel, + embeddingDimension: EMBEDDING_DIMENSIONS, + chunkingConfig, + }, + requestId + ) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create knowledge base "${name}"`) + } + + logger.info(`[${requestId}] Created knowledge base ${created.id} for user ${params.userId}`) + + PlatformEvents.knowledgeBaseCreated({ + knowledgeBaseId: created.id, + name: created.name, + workspaceId, + }) + + captureServerEvent( + params.userId, + 'knowledge_base_created', + { knowledge_base_id: created.id, workspace_id: workspaceId, name: created.name }, + { + groups: { workspace: workspaceId }, + setOnce: { first_kb_created_at: new Date().toISOString() }, + } + ) + + recordAudit({ + workspaceId, + ...auditActorFields(params), + action: AuditAction.KNOWLEDGE_BASE_CREATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: created.id, + resourceName: created.name, + description: `Created knowledge base "${created.name}"`, + metadata: { + source, + name: created.name, + description: created.description, + embeddingModel, + embeddingDimension: EMBEDDING_DIMENSIONS, + chunkingStrategy: chunkingConfig.strategy, + chunkMaxSize: chunkingConfig.maxSize, + chunkMinSize: chunkingConfig.minSize, + chunkOverlap: chunkingConfig.overlap, + }, + ...(request ? { request } : {}), + }) + + return { success: true, knowledgeBase: created } +} + +export interface PerformUpdateKnowledgeBaseParams extends KnowledgeOperationContext { + knowledgeBaseId: string + /** Workspace the knowledge base currently belongs to, for the audit row. */ + workspaceId: string | null + updates: { + name?: string + description?: string + /** Moves the knowledge base between workspaces; omitted leaves it in place. */ + workspaceId?: string | null + folderId?: string | null + chunkingConfig?: ChunkingConfig + } +} + +/** + * Applies a knowledge base update and records it against the actor. + * + * `actorUserId` is always forwarded, so a workspace move is authorized against + * the caller rather than rejected for a missing actor — the v1 and v2 routes + * previously omitted it. + */ +export async function performUpdateKnowledgeBase( + params: PerformUpdateKnowledgeBaseParams +): Promise { + const { knowledgeBaseId, updates, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + const updatedFields = Object.keys(updates).filter( + (key) => updates[key as keyof typeof updates] !== undefined + ) + if (updatedFields.length === 0) { + return fail('No updates specified', 'validation') + } + + let updated: KnowledgeBaseWithCounts + try { + updated = await updateKnowledgeBase(knowledgeBaseId, updates, requestId, { + actorUserId: params.userId, + }) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Update knowledge base ${knowledgeBaseId}`) + } + + logger.info(`[${requestId}] Updated knowledge base ${knowledgeBaseId}`) + + recordAudit({ + // The destination workspace when this update moved it, so the audit row + // lands where the knowledge base now lives. + workspaceId: updates.workspaceId !== undefined ? updates.workspaceId : params.workspaceId, + ...auditActorFields(params), + action: AuditAction.KNOWLEDGE_BASE_UPDATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: knowledgeBaseId, + resourceName: updated.name, + description: `Updated knowledge base "${updated.name}"`, + metadata: { + source, + updatedFields, + ...(updates.name && { newName: updates.name }), + ...(updates.description !== undefined && { description: updates.description }), + ...(updates.chunkingConfig && { + chunkMaxSize: updates.chunkingConfig.maxSize, + chunkMinSize: updates.chunkingConfig.minSize, + chunkOverlap: updates.chunkingConfig.overlap, + }), + }, + ...(request ? { request } : {}), + }) + + return { success: true, knowledgeBase: updated } +} + +export interface PerformDeleteKnowledgeBaseParams extends KnowledgeOperationContext { + knowledgeBase: { id: string; name: string; workspaceId: string | null } +} + +export type PerformDeleteKnowledgeBaseResult = KnowledgeOrchestrationResult + +/** + * Archives a knowledge base and its documents and connectors. + * + * The folder cascade and other internal callers keep calling + * `deleteKnowledgeBase` directly and stay silent by construction — auditing + * follows from a user performing this operation, not from the write running. + */ +export async function performDeleteKnowledgeBase( + params: PerformDeleteKnowledgeBaseParams +): Promise { + const { knowledgeBase, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + try { + await deleteKnowledgeBase(knowledgeBase.id, requestId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Delete knowledge base ${knowledgeBase.id}`) + } + + logger.info(`[${requestId}] Deleted knowledge base ${knowledgeBase.id}`) + + PlatformEvents.knowledgeBaseDeleted({ knowledgeBaseId: knowledgeBase.id }) + + recordAudit({ + workspaceId: knowledgeBase.workspaceId, + ...auditActorFields(params), + action: AuditAction.KNOWLEDGE_BASE_DELETED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: knowledgeBase.id, + resourceName: knowledgeBase.name, + description: `Deleted knowledge base "${knowledgeBase.name}"`, + metadata: { source, knowledgeBaseName: knowledgeBase.name }, + ...(request ? { request } : {}), + }) + + return { success: true } +} diff --git a/apps/sim/lib/knowledge/orchestration/restore.test.ts b/apps/sim/lib/knowledge/orchestration/restore.test.ts new file mode 100644 index 00000000000..558df668d1a --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/restore.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockRecordAudit, mockRestoreKnowledgeBase } = vi.hoisted(() => ({ + mockRecordAudit: vi.fn(), + mockRestoreKnowledgeBase: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { KNOWLEDGE_BASE_RESTORED: 'knowledge_base.restored' }, + AuditResourceType: { KNOWLEDGE_BASE: 'knowledge_base' }, + recordAudit: mockRecordAudit, +})) +vi.mock('@/lib/knowledge/service', () => ({ restoreKnowledgeBase: mockRestoreKnowledgeBase })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { performRestoreKnowledgeBase } from '@/lib/knowledge/orchestration/restore' + +const ARCHIVED = { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1', userId: 'owner' } + +describe('performRestoreKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + afterAll(resetDbChainMock) + + it('audits the restore against the acting user', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ARCHIVED]) + mockRestoreKnowledgeBase.mockResolvedValue(undefined) + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: 'kb-1', + userId: 'user-1', + source: 'ui', + requestId: 'req-1', + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ actorId: 'user-1', resourceId: 'kb-1', workspaceId: 'ws-1' }) + ) + }) + + it('reports a knowledge base that is not archived as a conflict', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ARCHIVED]) + mockRestoreKnowledgeBase.mockRejectedValue( + new OrchestrationError('conflict', 'Knowledge base is not archived') + ) + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: 'kb-1', + userId: 'user-1', + source: 'ui', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + }) + + it('reports bad input as validation, which the narrow local alias could not express', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ARCHIVED]) + mockRestoreKnowledgeBase.mockRejectedValue( + new OrchestrationError('validation', 'Folder not found in this workspace') + ) + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: 'kb-1', + userId: 'user-1', + source: 'ui', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + }) + + it('reports a knowledge base that does not exist as not found', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: 'kb-1', + userId: 'user-1', + source: 'ui', + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'not_found' }) + expect(mockRestoreKnowledgeBase).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/restore.ts b/apps/sim/lib/knowledge/orchestration/restore.ts new file mode 100644 index 00000000000..c162dc160b7 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/restore.ts @@ -0,0 +1,88 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { db } from '@sim/db' +import { knowledgeBase } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' +import { generateRequestId } from '@/lib/core/utils/request' +import { + auditActorFields, + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { restoreKnowledgeBase } from '@/lib/knowledge/service' + +const logger = createLogger('KnowledgeBaseRestoreOrchestration') + +export interface RestorableKnowledgeBase { + id: string + name: string + workspaceId: string | null + userId: string +} + +export interface PerformRestoreKnowledgeBaseParams extends KnowledgeOperationContext { + knowledgeBaseId: string +} + +export type PerformRestoreKnowledgeBaseResult = KnowledgeOrchestrationResult<{ + knowledgeBase: RestorableKnowledgeBase +}> + +/** + * Loads an archived knowledge base's identity so the caller can authorize the + * restore. Reads regardless of `deletedAt` — an archived row is exactly what a + * restore targets. + */ +export async function getRestorableKnowledgeBase( + knowledgeBaseId: string +): Promise { + const [kb] = await db + .select({ + id: knowledgeBase.id, + name: knowledgeBase.name, + workspaceId: knowledgeBase.workspaceId, + userId: knowledgeBase.userId, + }) + .from(knowledgeBase) + .where(eq(knowledgeBase.id, knowledgeBaseId)) + .limit(1) + + return kb ?? null +} + +/** Un-archives a knowledge base and its documents and connectors. */ +export async function performRestoreKnowledgeBase( + params: PerformRestoreKnowledgeBaseParams +): Promise { + const { knowledgeBaseId, request, source } = params + const requestId = params.requestId ?? generateRequestId() + + const kb = await getRestorableKnowledgeBase(knowledgeBaseId) + if (!kb) { + return fail('Knowledge base not found', 'not_found') + } + + try { + await restoreKnowledgeBase(knowledgeBaseId, requestId) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Restore knowledge base ${knowledgeBaseId}`) + } + + logger.info(`[${requestId}] Restored knowledge base ${knowledgeBaseId}`) + + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.KNOWLEDGE_BASE_RESTORED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: knowledgeBaseId, + resourceName: kb.name, + description: `Restored knowledge base "${kb.name}"`, + metadata: { source, knowledgeBaseName: kb.name }, + ...(request ? { request } : {}), + }) + + return { success: true, knowledgeBase: kb } +} diff --git a/apps/sim/lib/knowledge/orchestration/shared.ts b/apps/sim/lib/knowledge/orchestration/shared.ts new file mode 100644 index 00000000000..2c2581deee7 --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/shared.ts @@ -0,0 +1,84 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { + asOrchestrationError, + type OrchestrationErrorCode, + type OrchestrationRequestContext, +} from '@/lib/core/orchestration/types' + +const logger = createLogger('KnowledgeOrchestration') + +/** + * Which surface an operation came in through. Recorded in the audit metadata so + * the audit list still distinguishes a knowledge base the UI created from one an + * API key or the agent created, now that all three share one description. + */ +export type KnowledgeOperationSource = 'ui' | 'api' | 'agent' + +/** The acting user, plus the labels the audit row displays when it has them. */ +export interface KnowledgeActor { + userId: string + actorName?: string | null + actorEmail?: string | null + source: KnowledgeOperationSource +} + +/** Fields every knowledge orchestration function accepts. */ +export interface KnowledgeOperationContext extends KnowledgeActor { + requestId?: string + /** Forwarded to the audit record for IP / user-agent capture. */ + request?: OrchestrationRequestContext +} + +export interface KnowledgeOrchestrationFailure { + success: false + error: string + errorCode: OrchestrationErrorCode +} + +/** + * Every knowledge orchestration function returns this shape. A discriminated + * union rather than an all-optional record, so `if (!outcome.success) return …` + * narrows the success branch and callers reach the payload without asserting it + * is there. + */ +export type KnowledgeOrchestrationResult = + | ({ success: true } & TData) + | KnowledgeOrchestrationFailure + +export function fail( + error: string, + errorCode: OrchestrationErrorCode +): KnowledgeOrchestrationFailure { + return { success: false, error, errorCode } +} + +/** + * Maps a thrown failure to its transport-neutral class. + * + * Every caller-fixable knowledge failure is an {@link OrchestrationError} + * subclass, so the class decides the status rather than the message wording. + * Anything unclassified stays a generic 500, which is what an unexpected fault + * should be, and is logged here because no layer above will see the cause. + */ +export function classifyKnowledgeFailure( + error: unknown, + requestId: string, + operation: string +): KnowledgeOrchestrationFailure { + const classified = asOrchestrationError(error) + if (classified) { + return fail(classified.message, classified.code) + } + logger.error(`[${requestId}] ${operation} failed`, { error }) + return fail(toError(error).message, 'internal') +} + +/** The audit fields carried from the actor, omitting labels the caller lacks. */ +export function auditActorFields(actor: KnowledgeActor) { + return { + actorId: actor.userId, + ...(actor.actorName !== undefined ? { actorName: actor.actorName } : {}), + ...(actor.actorEmail !== undefined ? { actorEmail: actor.actorEmail } : {}), + } +} diff --git a/apps/sim/lib/knowledge/service.test.ts b/apps/sim/lib/knowledge/service.test.ts index 8b9ca673f2d..ce59bc0087b 100644 --- a/apps/sim/lib/knowledge/service.test.ts +++ b/apps/sim/lib/knowledge/service.test.ts @@ -71,7 +71,7 @@ describe('updateKnowledgeBase — workspace transfer authorization', () => { await expect( updateKnowledgeBase('kb-1', { workspaceId: null }, 'req-1', { actorUserId: 'attacker' }) ).rejects.toMatchObject({ - code: 'KNOWLEDGE_BASE_FORBIDDEN', + code: 'forbidden', message: 'Only the knowledge base owner can remove it from a workspace', }) expect(permissionsMockFns.mockGetUserEntityPermissions).not.toHaveBeenCalled() @@ -95,7 +95,7 @@ describe('updateKnowledgeBase — workspace transfer authorization', () => { actorUserId: 'attacker', }) ).rejects.toMatchObject({ - code: 'KNOWLEDGE_BASE_FORBIDDEN', + code: 'forbidden', message: 'User does not have permission on the target workspace', }) expect(permissionsMockFns.mockGetUserEntityPermissions).toHaveBeenCalledWith( diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index cbac1283b63..e4e8cc3707b 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -20,6 +20,7 @@ import { resolveStorageBillingContext, type StorageBillingContext, } from '@/lib/billing/storage' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRestoreName } from '@/lib/core/utils/restore-name' import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries' import type { @@ -31,22 +32,39 @@ import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' const logger = createLogger('KnowledgeBaseService') -export class KnowledgeBaseConflictError extends Error { - readonly code = 'KNOWLEDGE_BASE_EXISTS' as const +/** + * Every caller-fixable knowledge-base failure is an {@link OrchestrationError}, + * so `lib/knowledge/orchestration` classifies it by class and each surface maps + * that one class to its own status. Message text is then free to change without + * silently moving a 409 to a 400. + */ +export class KnowledgeBaseConflictError extends OrchestrationError { constructor(name: string) { - super(`A knowledge base named "${name}" already exists in this workspace`) + super('conflict', `A knowledge base named "${name}" already exists in this workspace`) + this.name = 'KnowledgeBaseConflictError' } } -export class KnowledgeBasePermissionError extends Error { - readonly code = 'KNOWLEDGE_BASE_FORBIDDEN' as const +export class KnowledgeBasePermissionError extends OrchestrationError { + constructor(message: string) { + super('forbidden', message) + this.name = 'KnowledgeBasePermissionError' + } } /** Raised when a caller files a knowledge base under a folder it may not use. */ -export class KnowledgeBaseFolderError extends Error { - readonly code = 'KNOWLEDGE_BASE_FOLDER_INVALID' as const +export class KnowledgeBaseFolderError extends OrchestrationError { constructor() { - super('Folder not found in this workspace') + super('validation', 'Folder not found in this workspace') + this.name = 'KnowledgeBaseFolderError' + } +} + +/** Raised when a knowledge base the caller named does not exist (or is archived). */ +export class KnowledgeBaseNotFoundError extends OrchestrationError { + constructor(knowledgeBaseId: string) { + super('not_found', `Knowledge base ${knowledgeBaseId} not found`) + this.name = 'KnowledgeBaseNotFoundError' } } @@ -341,7 +359,7 @@ export async function updateKnowledgeBase( .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .limit(1) if (!snapshot) { - throw new Error(`Knowledge base ${knowledgeBaseId} not found`) + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } effectiveWorkspaceId = snapshot.workspaceId } @@ -365,7 +383,7 @@ export async function updateKnowledgeBase( .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .limit(1) if (!kbSnapshot) { - throw new Error(`Knowledge base ${knowledgeBaseId} not found`) + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } const sourceWorkspaceId = kbSnapshot.workspaceId ?? null const destinationWorkspaceId = updates.workspaceId ?? null @@ -450,7 +468,7 @@ export async function updateKnowledgeBase( .limit(1) if (!currentKb) { - throw new Error(`Knowledge base ${knowledgeBaseId} not found`) + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } if (storageMove && (currentKb.workspaceId ?? null) !== storageMove.sourceWorkspaceId) { @@ -666,7 +684,7 @@ export async function updateKnowledgeBase( .limit(1) if (updatedKb.length === 0) { - throw new Error(`Knowledge base ${knowledgeBaseId} not found`) + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } logger.info(`[${requestId}] Updated knowledge base: ${knowledgeBaseId}`) @@ -809,18 +827,21 @@ export async function restoreKnowledgeBase( .limit(1) if (!kb) { - throw new Error('Knowledge base not found') + throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } if (!kb.deletedAt) { - throw new Error('Knowledge base is not archived') + throw new OrchestrationError('conflict', 'Knowledge base is not archived') } if (kb.workspaceId) { const { getWorkspaceWithOwner } = await import('@/lib/workspaces/permissions/utils') const ws = await getWorkspaceWithOwner(kb.workspaceId) if (!ws || ws.archivedAt) { - throw new Error('Cannot restore knowledge base into an archived workspace') + throw new OrchestrationError( + 'conflict', + 'Cannot restore knowledge base into an archived workspace' + ) } } diff --git a/apps/sim/lib/resources/orchestration/restore-resource.ts b/apps/sim/lib/resources/orchestration/restore-resource.ts index 75baa13aaf4..ec7055f9c5f 100644 --- a/apps/sim/lib/resources/orchestration/restore-resource.ts +++ b/apps/sim/lib/resources/orchestration/restore-resource.ts @@ -148,10 +148,11 @@ export async function performRestoreResource( const result = await performRestoreKnowledgeBase({ knowledgeBaseId: id, userId, + source: 'agent', requestId, }) - if (!result.success || !result.knowledgeBase) { - return { success: false, error: result.error || 'Failed to restore knowledge base' } + if (!result.success) { + return { success: false, error: result.error } } logger.info('Knowledge base restored via restore_resource', { knowledgeBaseId: id }) From 1ba0215ee74291778c8f34b625f58f8da3b137cf Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Fri, 31 Jul 2026 23:57:59 -0700 Subject: [PATCH 2/2] fix(knowledge): make connector create atomic and stop flattening failures Review round 1 on #6154. - Resolve the billing payer before the connector is committed, not after. A malformed attribution header rejected post-commit left a live connector behind a 500, and a retry created a duplicate plus duplicate sync work. Manual sync resolves before writing its audit for the same reason. - Let the source-config validator carry its own failure class. Collapsing every rejection to `validation` flattened the connector PATCH route's 401 (stale stored credential) and 409 (missing workspace context) into a 400. - Add `unauthorized` to OrchestrationErrorCode. It is the class that 401 was already expressing on this route, and the v2 vocabulary already had UNAUTHORIZED; only the shared union was missing it. - Report a knowledge base that exists but failed to archive as failed, with the reason, rather than as not found. The copilot delete loop folded every non-not-found failure into `notFound`, telling the user it was never there. - Route copilot failures through the same message helper the HTTP surfaces use, so an unclassified fault's raw text (a driver's failed SQL) no longer reaches the agent verbatim while the UI and public APIs get the generic wording. --- .../[id]/connectors/[connectorId]/route.ts | 40 ++++++++-- apps/sim/app/api/v2/lib/response.ts | 1 + .../server/knowledge/knowledge-base.test.ts | 55 ++++++++++++- .../tools/server/knowledge/knowledge-base.ts | 80 ++++++++++++++++--- apps/sim/lib/core/orchestration/types.ts | 7 ++ .../orchestration/connectors.test.ts | 42 +++++++++- .../lib/knowledge/orchestration/connectors.ts | 39 +++++++-- apps/sim/lib/knowledge/orchestration/index.ts | 1 + 8 files changed, 235 insertions(+), 30 deletions(-) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts index 8bd4be8b472..d63513af694 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts @@ -22,6 +22,7 @@ import { type KnowledgeConnectorRow, performDeleteKnowledgeConnector, performUpdateKnowledgeConnector, + type SourceConfigRejection, } from '@/lib/knowledge/orchestration' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' @@ -94,28 +95,43 @@ function makeSourceConfigValidator( return async ( connector: KnowledgeConnectorRow, sourceConfig: Record - ): Promise => { + ): Promise => { const connectorConfig = CONNECTOR_REGISTRY[connector.connectorType] if (!connectorConfig) { - return `Unknown connector type: ${connector.connectorType}` + return { + message: `Unknown connector type: ${connector.connectorType}`, + errorCode: 'validation', + } } let accessToken: string | null = null if (connectorConfig.auth.mode === 'apiKey') { if (!connector.encryptedApiKey) { - return 'API key not found. Please reconfigure the connector.' + return { + message: 'API key not found. Please reconfigure the connector.', + errorCode: 'validation', + } } accessToken = (await decryptApiKey(connector.encryptedApiKey)).decrypted } else { if (!connector.credentialId) { - return 'OAuth credential not found. Please reconfigure the connector.' + return { + message: 'OAuth credential not found. Please reconfigure the connector.', + errorCode: 'validation', + } } if (!workspaceId) { - return 'Knowledge base is missing workspace context' + return { + message: 'Knowledge base is missing workspace context', + errorCode: 'conflict', + } } const identity = await resolveCredentialTokenIdentity(connector.credentialId, workspaceId) if (!identity) { - return 'Credential is no longer usable in this workspace. Please reconnect it.' + return { + message: 'Credential is no longer usable in this workspace. Please reconnect it.', + errorCode: 'validation', + } } accessToken = await refreshAccessTokenIfNeeded( connector.credentialId, @@ -126,11 +142,19 @@ function makeSourceConfigValidator( } if (!accessToken) { - return 'Failed to refresh access token. Please reconnect your account.' + // A stale stored credential, not an unauthenticated caller — but the route + // has always answered 401 here, so keep that rather than silently + // reclassifying it as part of this refactor. + return { + message: 'Failed to refresh access token. Please reconnect your account.', + errorCode: 'unauthorized', + } } const validation = await connectorConfig.validateConfig(accessToken, sourceConfig) - return validation.valid ? null : validation.error || 'Invalid source configuration' + return validation.valid + ? null + : { message: validation.error || 'Invalid source configuration', errorCode: 'validation' } } } diff --git a/apps/sim/app/api/v2/lib/response.ts b/apps/sim/app/api/v2/lib/response.ts index 8225c065d49..3bdc2b90b91 100644 --- a/apps/sim/app/api/v2/lib/response.ts +++ b/apps/sim/app/api/v2/lib/response.ts @@ -159,6 +159,7 @@ export function decodeCursor>(cursor: string): T | n const V2_CODE_BY_ORCHESTRATION_ERROR: Record = { validation: 'BAD_REQUEST', + unauthorized: 'UNAUTHORIZED', forbidden: 'FORBIDDEN', not_found: 'NOT_FOUND', conflict: 'CONFLICT', diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts index 52a32dd22d9..20980bc6a95 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts @@ -8,13 +8,17 @@ import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockAssertBillingAttributionSnapshot, mockCheckKnowledgeBaseWriteAccess, + mockGetKnowledgeBaseById, mockPerformCreateKnowledgeConnector, + mockPerformDeleteKnowledgeBase, mockPerformDeleteKnowledgeConnector, mockPerformSyncKnowledgeConnector, } = vi.hoisted(() => ({ mockAssertBillingAttributionSnapshot: vi.fn(), mockCheckKnowledgeBaseWriteAccess: vi.fn(), + mockGetKnowledgeBaseById: vi.fn(), mockPerformCreateKnowledgeConnector: vi.fn(), + mockPerformDeleteKnowledgeBase: vi.fn(), mockPerformDeleteKnowledgeConnector: vi.fn(), mockPerformSyncKnowledgeConnector: vi.fn(), })) @@ -38,8 +42,8 @@ vi.mock('@/lib/knowledge/embeddings', () => ({ })) vi.mock('@/lib/knowledge/orchestration', () => ({ performCreateKnowledgeBase: vi.fn(), + performDeleteKnowledgeBase: mockPerformDeleteKnowledgeBase, performCreateKnowledgeConnector: mockPerformCreateKnowledgeConnector, - performDeleteKnowledgeBase: vi.fn(), performDeleteKnowledgeConnector: mockPerformDeleteKnowledgeConnector, performDeleteKnowledgeDocument: vi.fn(), performSyncKnowledgeConnector: mockPerformSyncKnowledgeConnector, @@ -49,7 +53,7 @@ vi.mock('@/lib/knowledge/orchestration', () => ({ performUploadKnowledgeDocument: vi.fn(), })) vi.mock('@/lib/knowledge/service', () => ({ - getKnowledgeBaseById: vi.fn(), + getKnowledgeBaseById: mockGetKnowledgeBaseById, })) vi.mock('@/lib/knowledge/tags/service', () => ({ createTagDefinition: vi.fn(), @@ -156,6 +160,53 @@ describe('knowledge base connector Copilot operations', () => { expect(mockAssertBillingAttributionSnapshot).toHaveBeenCalledWith(BILLING_ATTRIBUTION) }) + it('reports a failed knowledge base delete as failed, not as missing', async () => { + mockGetKnowledgeBaseById.mockResolvedValue({ + id: 'knowledge-base-1', + name: 'Paid KB', + workspaceId: 'workspace-paid', + }) + mockPerformDeleteKnowledgeBase.mockResolvedValue({ + success: false, + error: 'Knowledge base is locked', + errorCode: 'conflict', + }) + + const result = await knowledgeBaseServerTool.execute( + { operation: 'delete', args: { knowledgeBaseId: 'knowledge-base-1' } }, + CONTEXT + ) + + // A knowledge base that exists but could not be archived is neither deleted + // nor missing — folding it into notFound told the user it was never there. + expect(result.data.notFound).toEqual([]) + expect(result.data.failed).toEqual([ + { id: 'knowledge-base-1', name: 'Paid KB', reason: 'Knowledge base is locked' }, + ]) + expect(result.message).toContain('Knowledge base is locked') + }) + + it('never relays an unclassified fault to the agent verbatim', async () => { + mockGetKnowledgeBaseById.mockResolvedValue({ + id: 'knowledge-base-1', + name: 'Paid KB', + workspaceId: 'workspace-paid', + }) + mockPerformDeleteKnowledgeBase.mockResolvedValue({ + success: false, + error: 'select "id" from "knowledge_base" — connection terminated', + errorCode: 'internal', + }) + + const result = await knowledgeBaseServerTool.execute( + { operation: 'delete', args: { knowledgeBaseId: 'knowledge-base-1' } }, + CONTEXT + ) + + expect(result.data.failed[0].reason).toBe('Failed to delete knowledge base') + expect(result.message).not.toContain('connection terminated') + }) + it('reports that a deleted connector kept its documents, because it did', async () => { const result = await knowledgeBaseServerTool.execute( { operation: 'delete_connector', args: { connectorId: 'connector-1' } }, diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index 220e96c5090..a7f41a21768 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -18,6 +18,10 @@ import { type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' +import { + messageForOrchestrationError, + type OrchestrationErrorCode, +} from '@/lib/core/orchestration/types' import { generateSearchEmbedding, recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings' import { performCreateKnowledgeBase, @@ -67,6 +71,20 @@ function requireKnowledgeBillingAttribution( return attribution } +/** + * The message the agent — and therefore the user — is shown for a failed + * operation. Mirrors `messageForOrchestrationError` on the HTTP surfaces: a + * classified failure is caller-fixable and safe to relay, an unclassified one + * carries whatever text the fault happened to have (a driver's failed SQL, say) + * and is replaced by the operation's own wording. + */ +function agentFacingError( + outcome: { error?: string; errorCode?: OrchestrationErrorCode }, + fallback: string +): string { + return messageForOrchestrationError(outcome, fallback) +} + type KnowledgeBaseArgs = { operation: string args?: Record @@ -142,7 +160,10 @@ export const knowledgeBaseServerTool: BaseServerTool = [] const notFound: string[] = [] + // A knowledge base that exists but could not be archived is neither + // deleted nor missing. Folding it into `notFound` told the user it was + // never there instead of why the delete failed. + const failed: Array<{ id: string; name: string; reason: string }> = [] for (const kbId of kbIds) { const writeAccess = await checkKnowledgeBaseWriteAccess(kbId, context.userId) @@ -501,19 +529,33 @@ export const knowledgeBaseServerTool: BaseServerTool 0 ? `Deleted: ${deleted.map((d) => d.name).join(', ')}` : null, + failed.length > 0 + ? `Failed: ${failed.map((f) => `${f.name} (${f.reason})`).join(', ')}` + : null, + ] + .filter(Boolean) + .join('. ') + return { success: deleted.length > 0, - message: - deleted.length > 0 - ? `Deleted: ${deleted.map((d) => d.name).join(', ')}` - : 'No knowledge bases found', - data: { deleted, notFound }, + message: deleteSummary || 'No knowledge bases found', + data: { deleted, notFound, failed }, } } @@ -611,7 +653,10 @@ export const knowledgeBaseServerTool: BaseServerTool billingAttribution, }) if (!outcome.success) { - return { success: false, message: outcome.error } + return { + success: false, + message: agentFacingError(outcome, 'Failed to sync connector'), + } } return { diff --git a/apps/sim/lib/core/orchestration/types.ts b/apps/sim/lib/core/orchestration/types.ts index c9d416c3bc6..eb42af6bf65 100644 --- a/apps/sim/lib/core/orchestration/types.ts +++ b/apps/sim/lib/core/orchestration/types.ts @@ -1,5 +1,11 @@ export type OrchestrationErrorCode = | 'validation' + /** + * The credentials this operation depends on are no longer usable — a stored + * third-party token that will not refresh, not an unauthenticated caller. + * Distinct from `forbidden`, which is the caller lacking permission. + */ + | 'unauthorized' | 'not_found' | 'forbidden' | 'conflict' @@ -14,6 +20,7 @@ export type OrchestrationErrorCode = */ export function statusForOrchestrationError(code: OrchestrationErrorCode | undefined): number { if (code === 'validation') return 400 + if (code === 'unauthorized') return 401 if (code === 'forbidden') return 403 if (code === 'not_found') return 404 if (code === 'conflict') return 409 diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index fec4795dc79..68f7a10f242 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -157,7 +157,10 @@ describe('performUpdateKnowledgeConnector', () => { knowledgeBase: KB, connectorId: 'conn-1', updates: { sourceConfig: { database: 'gone' } }, - validateSourceConfig: async () => 'Database not found', + validateSourceConfig: async () => ({ + message: 'Database not found', + errorCode: 'validation' as const, + }), }) expect(outcome).toMatchObject({ @@ -167,6 +170,25 @@ describe('performUpdateKnowledgeConnector', () => { }) }) + it('preserves the failure class the validator chose', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + + // A stale stored credential kept the route's 401; collapsing every + // rejection to `validation` had flattened it (and the 409) into a 400. + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { sourceConfig: { database: 'x' } }, + validateSourceConfig: async () => ({ + message: 'Failed to refresh access token. Please reconnect your account.', + errorCode: 'unauthorized' as const, + }), + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'unauthorized' }) + }) + it('clears the failure counters when a paused connector is resumed', async () => { dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) dbChainMockFns.returning.mockResolvedValueOnce([ @@ -196,6 +218,24 @@ describe('performSyncKnowledgeConnector', () => { afterAll(resetDbChainMock) + it('resolves the payer before writing the audit, not after', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + const rejects = vi.fn().mockRejectedValue(new Error('billing attribution header is malformed')) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + resolveBillingAttribution: rejects, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockDispatchSync).not.toHaveBeenCalled() + }) + it('refuses to stack a sync on one already running', async () => { dbChainMockFns.limit.mockResolvedValueOnce([ { id: 'conn-1', connectorType: 'notion', status: 'syncing' }, diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index c2ba8c86f1d..01d18f5ccea 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -13,7 +13,7 @@ import { and, eq, inArray, isNull, sql } from 'drizzle-orm' import { encryptApiKey } from '@/lib/api-key/crypto' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' -import { OrchestrationError } from '@/lib/core/orchestration/types' +import { OrchestrationError, type OrchestrationErrorCode } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { allocateTagSlots } from '@/lib/knowledge/constants' import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service' @@ -46,6 +46,12 @@ type ConnectorRow = KnowledgeConnectorRow /** The connector row as it reaches every caller: never carrying the stored API key. */ export type ConnectorWithoutSecret = Omit +/** A refused `sourceConfig`, with the failure class the caller wants surfaced. */ +export interface SourceConfigRejection { + message: string + errorCode: OrchestrationErrorCode +} + /** The knowledge base a connector operation targets, already authorized by the caller. */ export interface ConnectorKnowledgeBase { id: string @@ -225,6 +231,17 @@ export async function performCreateKnowledgeConnector( finalSourceConfig = { ...finalSourceConfig, tagSlotMapping } } + // Resolved before the write, not after: every guard that can cheaply reject + // the request has already run, and `requireBillingAttributionHeader` throws on + // a malformed header. Resolving it post-commit would leave a live connector + // behind a 500 and let a retry create a duplicate. + let billingAttribution: BillingAttributionSnapshot + try { + billingAttribution = await resolveBillingAttribution() + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + const now = new Date() const connectorId = generateId() const nextSyncAt = @@ -319,7 +336,6 @@ export async function performCreateKnowledgeConnector( ...(request ? { request } : {}), }) - const billingAttribution = await resolveBillingAttribution() const dispatchSync = await loadDispatchSync() dispatchSync(connectorId, { billingAttribution, requestId }).catch((error) => { logger.error( @@ -342,12 +358,16 @@ export interface PerformUpdateKnowledgeConnectorParams extends KnowledgeOperatio /** * Validates a replacement `sourceConfig` against the live source. Supplied by * the caller because resolving the connector's token needs the requesting - * identity. Returning a message rejects the update. + * identity. Returning a rejection fails the update. + * + * The rejection carries its own `errorCode` so a stale credential and a bad + * config stay distinguishable — collapsing every rejection to `validation` + * flattened the route's 401 and 409 into a 400. */ validateSourceConfig?: ( connector: KnowledgeConnectorRow, sourceConfig: Record - ) => Promise + ) => Promise } /** Loads an active connector scoped to its knowledge base. */ @@ -409,7 +429,7 @@ export async function performUpdateKnowledgeConnector( if (updates.sourceConfig !== undefined && validateSourceConfig) { const rejection = await validateSourceConfig(existing, updates.sourceConfig) if (rejection) { - return fail(rejection, 'validation') + return fail(rejection.message, rejection.errorCode) } } @@ -660,7 +680,14 @@ export async function performSyncKnowledgeConnector( if (!kb.workspaceId) { return fail('Knowledge base is missing workspace billing context', 'conflict') } - const billingAttribution = await resolveBillingAttribution() + // Resolved before the audit is written, so a rejected payer lookup returns a + // classified failure rather than escaping as a 500 with a sync already recorded. + let billingAttribution: BillingAttributionSnapshot + try { + billingAttribution = await resolveBillingAttribution() + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Sync connector ${connectorId}`) + } logger.info( `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` diff --git a/apps/sim/lib/knowledge/orchestration/index.ts b/apps/sim/lib/knowledge/orchestration/index.ts index afef6127d04..2f9d033a355 100644 --- a/apps/sim/lib/knowledge/orchestration/index.ts +++ b/apps/sim/lib/knowledge/orchestration/index.ts @@ -7,6 +7,7 @@ export { performDeleteKnowledgeConnector, performSyncKnowledgeConnector, performUpdateKnowledgeConnector, + type SourceConfigRejection, } from './connectors' export { type CreatedKnowledgeDocument,