From 3a6edafcd0b28af701bd95d44e149bd14df1ef99 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 11:27:37 -0700 Subject: [PATCH 1/8] feat(knowledge): hybrid lexical + vector retrieval for KB search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KB search ranked purely on pgvector cosine distance, which retrieves exact tokens (error codes, ticket keys, identifiers, rare product names) poorly. Add a full-text leg over the already-present generated `embedding.content_tsv` column and its GIN index — no migration, no re-indexing — and fuse it with the vector leg by reciprocal rank. Both legs run concurrently and share the same visibility and tag-filter predicates; the lexical leg is best-effort and falls back to vector-only on failure. Hybrid is the default for every caller. `searchMode: 'vector'` on the internal and v1 contracts (and an advanced Retrieval Mode dropdown on the Knowledge block) restores the previous behavior. Both search routes now share one `executeKnowledgeSearch` dispatch instead of duplicating the three-branch retrieval logic. --- .../app/api/knowledge/search/route.test.ts | 94 +++++--- apps/sim/app/api/knowledge/search/route.ts | 33 +-- .../app/api/knowledge/search/utils.test.ts | 213 +++++++++++++++++- apps/sim/app/api/knowledge/search/utils.ts | 190 ++++++++++++++++ .../app/api/v1/knowledge/search/route.test.ts | 18 +- apps/sim/app/api/v1/knowledge/search/route.ts | 33 +-- apps/sim/blocks/blocks/knowledge.ts | 16 ++ .../sim/lib/api/contracts/knowledge/search.ts | 16 ++ .../lib/api/contracts/v1/knowledge/index.ts | 6 + apps/sim/tools/knowledge/search.ts | 8 + packages/testing/src/mocks/database.mock.ts | 15 +- 11 files changed, 547 insertions(+), 95 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/route.test.ts b/apps/sim/app/api/knowledge/search/route.test.ts index 88e379dafe7..069e2ad5425 100644 --- a/apps/sim/app/api/knowledge/search/route.test.ts +++ b/apps/sim/app/api/knowledge/search/route.test.ts @@ -21,17 +21,13 @@ import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vites const { mockGetDocumentTagDefinitions, - mockHandleTagOnlySearch, - mockHandleVectorOnlySearch, - mockHandleTagAndVectorSearch, + mockExecuteKnowledgeSearch, mockGetQueryStrategy, mockGenerateSearchEmbedding, mockGetDocumentMetadataByIds, } = vi.hoisted(() => ({ mockGetDocumentTagDefinitions: vi.fn(), - mockHandleTagOnlySearch: vi.fn(), - mockHandleVectorOnlySearch: vi.fn(), - mockHandleTagAndVectorSearch: vi.fn(), + mockExecuteKnowledgeSearch: vi.fn(), mockGetQueryStrategy: vi.fn(), mockGenerateSearchEmbedding: vi.fn(), mockGetDocumentMetadataByIds: vi.fn(), @@ -69,9 +65,7 @@ vi.mock('@/lib/knowledge/tags/service', () => ({ })) vi.mock('./utils', () => ({ - handleTagOnlySearch: mockHandleTagOnlySearch, - handleVectorOnlySearch: mockHandleVectorOnlySearch, - handleTagAndVectorSearch: mockHandleTagAndVectorSearch, + executeKnowledgeSearch: mockExecuteKnowledgeSearch, getQueryStrategy: mockGetQueryStrategy, generateSearchEmbedding: mockGenerateSearchEmbedding, getDocumentMetadataByIds: mockGetDocumentMetadataByIds, @@ -118,9 +112,7 @@ describe('Knowledge Search API Route', () => { resetDbChainMock() setEnv({ OPENAI_API_KEY: 'test-api-key' }) - mockHandleTagOnlySearch.mockClear() - mockHandleVectorOnlySearch.mockClear() - mockHandleTagAndVectorSearch.mockClear() + mockExecuteKnowledgeSearch.mockClear() mockGetQueryStrategy.mockClear().mockReturnValue({ useParallel: false, distanceThreshold: 1.0, @@ -192,7 +184,7 @@ describe('Knowledge Search API Route', () => { dbChainMockFns.limit.mockResolvedValue([]) - mockHandleVectorOnlySearch.mockResolvedValue(mockSearchResults) + mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) mockFetch.mockResolvedValue({ ok: true, @@ -212,14 +204,50 @@ describe('Knowledge Search API Route', () => { expect(data.data.results[0].similarity).toBe(0.8) // 1 - 0.2 expect(data.data.query).toBe(validSearchData.query) expect(data.data.knowledgeBaseIds).toEqual(['kb-123']) - expect(mockHandleVectorOnlySearch).toHaveBeenCalledWith({ + expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ knowledgeBaseIds: ['kb-123'], topK: 10, + searchMode: 'hybrid', + query: validSearchData.query, queryVector: JSON.stringify(mockEmbedding), - distanceThreshold: expect.any(Number), + structuredFilters: undefined, }) }) + it('should forward the searchMode opt-out to the retrieval layer', async () => { + mockGetUserId.mockResolvedValue('user-123') + + mockCheckKnowledgeBaseAccess.mockResolvedValue({ + hasAccess: true, + knowledgeBase: { + id: 'kb-123', + userId: 'user-123', + name: 'Test KB', + deletedAt: null, + }, + }) + + dbChainMockFns.limit.mockResolvedValue([]) + + mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) + + mockFetch.mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + data: [{ embedding: mockEmbedding }], + }), + }) + + const req = createMockRequest('POST', { ...validSearchData, searchMode: 'vector' }) + const response = await POST(req) + + expect(response.status).toBe(200) + expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith( + expect.objectContaining({ searchMode: 'vector' }) + ) + }) + it('should perform search successfully with multiple knowledge bases', async () => { const multiKbData = { ...validSearchData, @@ -239,7 +267,7 @@ describe('Knowledge Search API Route', () => { dbChainMockFns.limit.mockResolvedValue([]) - mockHandleVectorOnlySearch.mockResolvedValue(mockSearchResults) + mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) mockFetch.mockResolvedValue({ ok: true, @@ -256,11 +284,13 @@ describe('Knowledge Search API Route', () => { expect(response.status).toBe(200) expect(data.success).toBe(true) expect(data.data.knowledgeBaseIds).toEqual(['kb-123', 'kb-456']) - expect(mockHandleVectorOnlySearch).toHaveBeenCalledWith({ + expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ knowledgeBaseIds: ['kb-123', 'kb-456'], topK: 10, + searchMode: 'hybrid', + query: multiKbData.query, queryVector: JSON.stringify(mockEmbedding), - distanceThreshold: expect.any(Number), + structuredFilters: undefined, }) }) @@ -284,7 +314,7 @@ describe('Knowledge Search API Route', () => { dbChainMockFns.limit.mockResolvedValue([]) - mockHandleVectorOnlySearch.mockResolvedValue(mockSearchResults) + mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) mockFetch.mockResolvedValue({ ok: true, @@ -348,7 +378,7 @@ describe('Knowledge Search API Route', () => { embeddingModel: 'text-embedding-3-small', }, }) - mockHandleVectorOnlySearch.mockResolvedValue(mockSearchResults) + mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) const attribution = encodeURIComponent( JSON.stringify({ actorUserId: 'user-123', @@ -532,7 +562,7 @@ describe('Knowledge Search API Route', () => { mockGetUserId.mockResolvedValue('user-123') dbChainMockFns.limit.mockResolvedValueOnce(mockKnowledgeBases) - mockHandleVectorOnlySearch.mockRejectedValueOnce(new Error('Database error')) + mockExecuteKnowledgeSearch.mockRejectedValueOnce(new Error('Database error')) const req = createMockRequest('POST', validSearchData) const response = await POST(req) @@ -750,7 +780,7 @@ describe('Knowledge Search API Route', () => { dbChainMockFns.limit.mockResolvedValueOnce(mockTagDefinitions) - mockHandleTagOnlySearch.mockResolvedValue(mockTaggedResults) + mockExecuteKnowledgeSearch.mockResolvedValue(mockTaggedResults) const req = createMockRequest('POST', tagOnlyData) const response = await POST(req) @@ -763,9 +793,10 @@ describe('Knowledge Search API Route', () => { expect(data.data.query).toBe('') // Empty query expect(data.data.cost).toBeUndefined() // No cost for tag-only search expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled() // No embedding API call - expect(mockHandleTagOnlySearch).toHaveBeenCalledWith({ + expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ knowledgeBaseIds: ['kb-123'], topK: 10, + searchMode: 'hybrid', structuredFilters: [ { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api', valueTo: undefined }, ], @@ -796,7 +827,7 @@ describe('Knowledge Search API Route', () => { dbChainMockFns.limit.mockResolvedValueOnce(mockTagDefinitions) - mockHandleTagAndVectorSearch.mockResolvedValue(mockSearchResults) + mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) mockFetch.mockResolvedValue({ ok: true, @@ -816,14 +847,15 @@ describe('Knowledge Search API Route', () => { expect(data.data.query).toBe('test search') expect(data.data.cost).toBeDefined() // Cost included for vector search expect(mockGenerateSearchEmbedding).toHaveBeenCalled() // Embedding API called - expect(mockHandleTagAndVectorSearch).toHaveBeenCalledWith({ + expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ knowledgeBaseIds: ['kb-123'], topK: 10, + searchMode: 'hybrid', + query: 'test search', + queryVector: JSON.stringify(mockEmbedding), structuredFilters: [ { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api', valueTo: undefined }, ], - queryVector: JSON.stringify(mockEmbedding), - distanceThreshold: 1, // Single KB uses threshold of 1.0 }) }) @@ -987,7 +1019,7 @@ describe('Knowledge Search API Route', () => { mockGetDocumentTagDefinitions.mockResolvedValue(mockTagDefinitions) - mockHandleTagOnlySearch.mockResolvedValue(mockTaggedResults) + mockExecuteKnowledgeSearch.mockResolvedValue(mockTaggedResults) dbChainMockFns.limit.mockResolvedValueOnce(mockTagDefinitions) @@ -1016,7 +1048,7 @@ describe('Knowledge Search API Route', () => { }, }) - mockHandleVectorOnlySearch.mockResolvedValue([ + mockExecuteKnowledgeSearch.mockResolvedValue([ { id: 'chunk-1', content: 'Content from active document', @@ -1092,7 +1124,7 @@ describe('Knowledge Search API Route', () => { { tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' }, ]) - mockHandleTagOnlySearch.mockResolvedValue([ + mockExecuteKnowledgeSearch.mockResolvedValue([ { id: 'chunk-2', content: 'Content from active document with tag', @@ -1164,7 +1196,7 @@ describe('Knowledge Search API Route', () => { { tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' }, ]) - mockHandleTagAndVectorSearch.mockResolvedValue([ + mockExecuteKnowledgeSearch.mockResolvedValue([ { id: 'chunk-3', content: 'Relevant content from active document', diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index aa337524f6a..12a88854571 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -27,12 +27,9 @@ import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/ import type { StructuredFilter } from '@/lib/knowledge/types' import { estimateTokenCount } from '@/lib/tokenization/estimators' import { + executeKnowledgeSearch, generateSearchEmbedding, getDocumentMetadataByIds, - getQueryStrategy, - handleTagAndVectorSearch, - handleTagOnlySearch, - handleVectorOnlySearch, type SearchResult, } from '@/app/api/knowledge/search/utils' import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils' @@ -318,32 +315,26 @@ export const POST = withRouteHandler(async (request: NextRequest) => { : validatedData.topK if (!hasQuery && hasFilters) { - results = await handleTagOnlySearch({ + results = await executeKnowledgeSearch({ knowledgeBaseIds: accessibleKbIds, topK: validatedData.topK, + searchMode: validatedData.searchMode, structuredFilters, }) - } else if (hasQuery && hasFilters) { - logger.debug(`[${requestId}] Executing tag + vector search with filters:`, structuredFilters) - const strategy = getQueryStrategy(accessibleKbIds.length, candidateTopK) - const queryVector = JSON.stringify((await queryEmbeddingPromise)?.embedding ?? null) - - results = await handleTagAndVectorSearch({ - knowledgeBaseIds: accessibleKbIds, - topK: candidateTopK, - structuredFilters, - queryVector, - distanceThreshold: strategy.distanceThreshold, - }) - } else if (hasQuery && !hasFilters) { - const strategy = getQueryStrategy(accessibleKbIds.length, candidateTopK) + } else if (hasQuery) { + logger.debug( + `[${requestId}] Executing ${validatedData.searchMode} search`, + hasFilters ? { structuredFilters } : undefined + ) const queryVector = JSON.stringify((await queryEmbeddingPromise)?.embedding ?? null) - results = await handleVectorOnlySearch({ + results = await executeKnowledgeSearch({ knowledgeBaseIds: accessibleKbIds, topK: candidateTopK, + searchMode: validatedData.searchMode, + query: validatedData.query, queryVector, - distanceThreshold: strategy.distanceThreshold, + structuredFilters: hasFilters ? structuredFilters : undefined, }) } else { return NextResponse.json( diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 4a9c5e1f0e5..227408ecb56 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -4,7 +4,14 @@ * * @vitest-environment node */ -import { mockNextFetchResponse, setupGlobalFetchMock } from '@sim/testing/mocks' +import { + dbChainMockFns, + mockNextFetchResponse, + queueTableRows, + resetDbChainMock, + schemaMock, + setupGlobalFetchMock, +} from '@sim/testing/mocks' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { env } from '@/lib/core/config/env' import * as documentsUtilsModule from '@/lib/knowledge/documents/utils' @@ -39,12 +46,45 @@ afterEach(() => { }) import { + executeKnowledgeSearch, + fuseByReciprocalRank, generateSearchEmbedding, handleTagAndVectorSearch, handleTagOnlySearch, handleVectorOnlySearch, + RRF_K, + type SearchResult, } from '@/app/api/knowledge/search/utils' +/** Minimal SearchResult builder — only the fields fusion and ordering read. */ +function makeResult(id: string, distance = 0.1): SearchResult { + return { + id, + content: `content-${id}`, + documentId: `doc-${id}`, + chunkIndex: 0, + tag1: null, + tag2: null, + tag3: null, + tag4: null, + tag5: null, + tag6: null, + tag7: null, + number1: null, + number2: null, + number3: null, + number4: null, + number5: null, + date1: null, + date2: null, + boolean1: null, + boolean2: null, + boolean3: null, + distance, + knowledgeBaseId: 'kb-123', + } +} + describe('Knowledge Search Utils', () => { beforeEach(() => { vi.clearAllMocks() @@ -183,6 +223,177 @@ describe('Knowledge Search Utils', () => { }) }) + describe('fuseByReciprocalRank', () => { + it('ranks a row found by both legs above rows found by only one', () => { + const shared = makeResult('shared') + const vectorOnly = makeResult('vector-only') + const keywordOnly = makeResult('keyword-only') + + const fused = fuseByReciprocalRank( + [ + [vectorOnly, shared], + [keywordOnly, shared], + ], + 10 + ) + + expect(fused.map((r) => r.id)).toEqual(['shared', 'vector-only', 'keyword-only']) + }) + + it('dedupes by chunk id, keeping the first occurrence', () => { + const fromVector = makeResult('chunk-1', 0.2) + const fromKeyword = { ...makeResult('chunk-1', 0.9), content: 'stale copy' } + + const fused = fuseByReciprocalRank([[fromVector], [fromKeyword]], 10) + + expect(fused).toHaveLength(1) + expect(fused[0].content).toBe('content-chunk-1') + expect(fused[0].distance).toBe(0.2) + }) + + it('preserves leg ordering when only one leg returns rows', () => { + const rows = [makeResult('a'), makeResult('b'), makeResult('c')] + + expect(fuseByReciprocalRank([rows, []], 10).map((r) => r.id)).toEqual(['a', 'b', 'c']) + expect(fuseByReciprocalRank([[], rows], 10).map((r) => r.id)).toEqual(['a', 'b', 'c']) + }) + + it('scores by reciprocal rank so a deep double hit beats a shallow single hit', () => { + const deepShared = makeResult('deep-shared') + const topSingle = makeResult('top-single') + + /** + * `deep-shared` sits at rank 2 in both legs: 2 / (RRF_K + 2). + * `top-single` sits at rank 1 in one leg only: 1 / (RRF_K + 1). + * With RRF_K = 60 the double hit wins. + */ + expect(2 / (RRF_K + 2)).toBeGreaterThan(1 / (RRF_K + 1)) + + const fused = fuseByReciprocalRank( + [ + [topSingle, deepShared], + [makeResult('other'), deepShared], + ], + 10 + ) + + expect(fused[0].id).toBe('deep-shared') + }) + + it('trims the fused list to topK', () => { + const rows = Array.from({ length: 8 }, (_, i) => makeResult(`chunk-${i}`)) + + expect(fuseByReciprocalRank([rows, []], 3)).toHaveLength(3) + }) + + it('returns an empty list when every leg is empty', () => { + expect(fuseByReciprocalRank([[], []], 10)).toEqual([]) + }) + }) + + describe('executeKnowledgeSearch', () => { + beforeEach(() => { + resetDbChainMock() + }) + + it('throws when neither a query nor tag filters are provided', async () => { + await expect( + executeKnowledgeSearch({ + knowledgeBaseIds: ['kb-123'], + topK: 10, + searchMode: 'hybrid', + }) + ).rejects.toThrow('A search query or tag filters are required') + }) + + it('throws when a query is provided without a query vector', async () => { + await expect( + executeKnowledgeSearch({ + knowledgeBaseIds: ['kb-123'], + topK: 10, + searchMode: 'hybrid', + query: 'PROJ-1234', + }) + ).rejects.toThrow('Query vector is required') + }) + + it('runs a single retrieval leg in vector mode', async () => { + queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) + + const results = await executeKnowledgeSearch({ + knowledgeBaseIds: ['kb-123'], + topK: 10, + searchMode: 'vector', + query: 'PROJ-1234', + queryVector: JSON.stringify([0.1, 0.2, 0.3]), + }) + + expect(results.map((r) => r.id)).toEqual(['vector-hit']) + expect(dbChainMockFns.select).toHaveBeenCalledTimes(1) + }) + + it('runs both legs and fuses them in hybrid mode', async () => { + queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) + queueTableRows(schemaMock.embedding, [makeResult('keyword-hit')]) + + const results = await executeKnowledgeSearch({ + knowledgeBaseIds: ['kb-123'], + topK: 10, + searchMode: 'hybrid', + query: 'PROJ-1234', + queryVector: JSON.stringify([0.1, 0.2, 0.3]), + }) + + expect(results.map((r) => r.id).sort()).toEqual(['keyword-hit', 'vector-hit']) + expect(dbChainMockFns.select).toHaveBeenCalledTimes(2) + }) + + it('falls back to vector results when the keyword leg fails', async () => { + queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) + + /** + * Both legs share one `orderBy` spy, so target the keyword leg by its + * ranking expression. Calling the untouched spy first captures the + * sentinel that tells the mock to build its normal chain, which the + * vector leg still needs. + */ + const chainDefault = dbChainMockFns.orderBy() + dbChainMockFns.orderBy.mockImplementation((fragment: unknown) => { + const text = (fragment as { strings?: string[] })?.strings?.join('') ?? '' + if (text.includes('ts_rank_cd')) { + throw new Error('tsquery blew up') + } + return chainDefault + }) + + const results = await executeKnowledgeSearch({ + knowledgeBaseIds: ['kb-123'], + topK: 10, + searchMode: 'hybrid', + query: 'PROJ-1234', + queryVector: JSON.stringify([0.1, 0.2, 0.3]), + }) + + expect(results.map((r) => r.id)).toEqual(['vector-hit']) + }) + + it('skips both query legs when only tag filters are provided', async () => { + queueTableRows(schemaMock.embedding, [makeResult('tag-hit')]) + + const results = await executeKnowledgeSearch({ + knowledgeBaseIds: ['kb-123'], + topK: 10, + searchMode: 'hybrid', + structuredFilters: [ + { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api' } as never, + ], + }) + + expect(results.map((r) => r.id)).toEqual(['tag-hit']) + expect(dbChainMockFns.select).toHaveBeenCalledTimes(1) + }) + }) + describe('generateSearchEmbedding', () => { it('should use Azure OpenAI when KB-specific config is provided', async () => { const { env } = await import('@/lib/core/config/env') diff --git a/apps/sim/app/api/knowledge/search/utils.ts b/apps/sim/app/api/knowledge/search/utils.ts index afaff875b5b..1c9892ff853 100644 --- a/apps/sim/app/api/knowledge/search/utils.ts +++ b/apps/sim/app/api/knowledge/search/utils.ts @@ -1,8 +1,12 @@ import { db } from '@sim/db' import { document, embedding } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { and, eq, inArray, isNull, sql } from 'drizzle-orm' import type { StructuredFilter } from '@/lib/knowledge/types' +const logger = createLogger('KnowledgeSearch') + export interface DocumentMetadata { filename: string sourceUrl: string | null @@ -306,6 +310,36 @@ function getStructuredTagFilters(filters: StructuredFilter[], embeddingTable: an return conditions } +/** + * Text-search configuration used to build the query. Must match the config the + * generated `embedding.content_tsv` column was built with + * (`to_tsvector('english', content)`) — a mismatch silently stops Postgres from + * using the `emb_content_fts_idx` GIN index and degrades to a sequential scan. + */ +const FTS_CONFIG = 'english' + +/** + * Reciprocal-rank-fusion damping constant. 60 is the value from the original RRF + * paper and matches the docs Ask-AI retriever (`apps/docs/app/api/chat/route.ts`). + */ +export const RRF_K = 60 + +/** + * Row visibility predicates shared by every search leg: a chunk is only + * retrievable when both it and its document are enabled, the document finished + * processing, and it has not been excluded, archived, or soft-deleted. + */ +function getVisibilityConditions() { + return [ + eq(embedding.enabled, true), + eq(document.enabled, true), + eq(document.processingStatus, 'completed'), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + ] +} + export function getQueryStrategy(kbCount: number, topK: number) { const useParallel = kbCount > 4 || (kbCount > 2 && topK > 50) const distanceThreshold = kbCount > 3 ? 0.8 : 1.0 @@ -512,6 +546,83 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise { + const { knowledgeBaseIds, topK, query, queryVector, structuredFilters } = params + + if (!query.trim()) { + return [] + } + + const tsQuery = sql`websearch_to_tsquery(${FTS_CONFIG}, ${query})` + const tagFilterConditions = structuredFilters?.length + ? getStructuredTagFilters(structuredFilters, embedding) + : [] + + return await db + .select( + getSearchResultFields( + sql`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') + ) + ) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and( + inArray(embedding.knowledgeBaseId, knowledgeBaseIds), + ...getVisibilityConditions(), + sql`${embedding.contentTsv} @@ ${tsQuery}`, + ...tagFilterConditions + ) + ) + .orderBy(sql`ts_rank_cd(${embedding.contentTsv}, ${tsQuery}) DESC`) + .limit(topK) +} + +/** + * Fuse independently-ranked result lists by reciprocal rank: + * `score(row) = Σ 1 / (RRF_K + rank)` across the lists it appears in. + * + * Rank fusion is used rather than score normalization because cosine distance + * and `ts_rank_cd` are on incomparable scales with no corpus-independent + * mapping between them. Rows are deduped by chunk id, first occurrence wins. + */ +export function fuseByReciprocalRank(rankedLists: SearchResult[][], topK: number): SearchResult[] { + const scores = new Map() + const rowById = new Map() + + for (const list of rankedLists) { + list.forEach((row, index) => { + scores.set(row.id, (scores.get(row.id) ?? 0) + 1 / (RRF_K + index + 1)) + if (!rowById.has(row.id)) { + rowById.set(row.id, row) + } + }) + } + + return [...rowById.values()] + .sort((a, b) => (scores.get(b.id) ?? 0) - (scores.get(a.id) ?? 0)) + .slice(0, topK) +} + export async function handleTagAndVectorSearch(params: SearchParams): Promise { const { knowledgeBaseIds, topK, structuredFilters, queryVector, distanceThreshold } = params @@ -537,3 +648,82 @@ export async function handleTagAndVectorSearch(params: SearchParams): Promise { + const { knowledgeBaseIds, topK, searchMode, query, queryVector, structuredFilters } = params + + const hasQuery = Boolean(query?.trim()) + const hasFilters = Boolean(structuredFilters && structuredFilters.length > 0) + + if (!hasQuery) { + if (!hasFilters) { + throw new Error('A search query or tag filters are required') + } + return await handleTagOnlySearch({ knowledgeBaseIds, topK, structuredFilters }) + } + + if (!queryVector) { + throw new Error('Query vector is required when searching with a query') + } + + const { distanceThreshold } = getQueryStrategy(knowledgeBaseIds.length, topK) + + const vectorSearch = hasFilters + ? handleTagAndVectorSearch({ + knowledgeBaseIds, + topK, + structuredFilters, + queryVector, + distanceThreshold, + }) + : handleVectorOnlySearch({ knowledgeBaseIds, topK, queryVector, distanceThreshold }) + + if (searchMode === 'vector') { + return await vectorSearch + } + + /** + * The lexical leg is best-effort: a failure there falls back to vector-only + * results rather than failing the whole search. + */ + const keywordSearch = executeKeywordSearch({ + knowledgeBaseIds, + topK, + query: query!, + queryVector, + structuredFilters, + }).catch((error) => { + logger.warn('Keyword search leg failed; falling back to vector-only results', { + error: getErrorMessage(error, 'Unknown error'), + }) + return [] as SearchResult[] + }) + + const [vectorResults, keywordResults] = await Promise.all([vectorSearch, keywordSearch]) + + return fuseByReciprocalRank([vectorResults, keywordResults], topK) +} diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index 978fbfdf751..e8d110ccfed 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -12,9 +12,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { beforeEach, describe, expect, it, vi } from 'vitest' const { - mockHandleVectorOnlySearch, - mockHandleTagOnlySearch, - mockHandleTagAndVectorSearch, + mockExecuteKnowledgeSearch, mockGetQueryStrategy, mockGenerateSearchEmbedding, mockGetDocumentMetadataByIds, @@ -24,9 +22,7 @@ const { mockResolveSystemBillingAttribution, mockRecordSearchEmbeddingUsage, } = vi.hoisted(() => ({ - mockHandleVectorOnlySearch: vi.fn(), - mockHandleTagOnlySearch: vi.fn(), - mockHandleTagAndVectorSearch: vi.fn(), + mockExecuteKnowledgeSearch: vi.fn(), mockGetQueryStrategy: vi.fn(), mockGenerateSearchEmbedding: vi.fn(), mockGetDocumentMetadataByIds: vi.fn(), @@ -51,9 +47,7 @@ const SYSTEM_BILLING_ATTRIBUTION = { } vi.mock('@/app/api/knowledge/search/utils', () => ({ - handleVectorOnlySearch: mockHandleVectorOnlySearch, - handleTagOnlySearch: mockHandleTagOnlySearch, - handleTagAndVectorSearch: mockHandleTagAndVectorSearch, + executeKnowledgeSearch: mockExecuteKnowledgeSearch, getQueryStrategy: mockGetQueryStrategy, generateSearchEmbedding: mockGenerateSearchEmbedding, getDocumentMetadataByIds: mockGetDocumentMetadataByIds, @@ -120,7 +114,7 @@ describe('v1 knowledge search route — per-KB embedding model', () => { embedding: [0.1, 0.2, 0.3], isBYOK: false, }) - mockHandleVectorOnlySearch.mockResolvedValue([]) + mockExecuteKnowledgeSearch.mockResolvedValue([]) mockGetDocumentMetadataByIds.mockResolvedValue({}) mockResolveBillingAttribution.mockImplementation( ({ actorUserId, workspaceId }: { actorUserId: string; workspaceId: string }) => @@ -218,7 +212,7 @@ describe('v1 knowledge search route — per-KB embedding model', () => { hasAccess: true, knowledgeBase: baseKb('kb-confluence', 'text-embedding-3-small'), }) - mockHandleVectorOnlySearch.mockResolvedValue([ + mockExecuteKnowledgeSearch.mockResolvedValue([ { documentId: 'doc-confluence', knowledgeBaseId: 'kb-confluence', @@ -250,7 +244,7 @@ describe('v1 knowledge search route — per-KB embedding model', () => { }) it('allows tag-only search across mixed embedding models', async () => { - mockHandleTagOnlySearch.mockResolvedValue([]) + mockExecuteKnowledgeSearch.mockResolvedValue([]) mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({ hasAccess: true, knowledgeBase: baseKb('kb-mixed', 'text-embedding-3-small'), diff --git a/apps/sim/app/api/v1/knowledge/search/route.ts b/apps/sim/app/api/v1/knowledge/search/route.ts index 3ac848b303d..490c8e88b7b 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.ts @@ -13,12 +13,9 @@ import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils' import type { StructuredFilter } from '@/lib/knowledge/types' import { + executeKnowledgeSearch, generateSearchEmbedding, getDocumentMetadataByIds, - getQueryStrategy, - handleTagAndVectorSearch, - handleTagOnlySearch, - handleVectorOnlySearch, type SearchResult, } from '@/app/api/knowledge/search/utils' import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils' @@ -49,7 +46,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) if (!parsed.success) return parsed.response - const { workspaceId, topK, query, tagFilters } = parsed.data.body + const { workspaceId, topK, query, tagFilters, searchMode } = parsed.data.body const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) if (accessError) return accessError @@ -190,29 +187,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let queryEmbeddingIsBYOK: boolean | null = null if (!hasQuery && hasFilters) { - results = await handleTagOnlySearch({ + results = await executeKnowledgeSearch({ knowledgeBaseIds: accessibleKbIds, topK, + searchMode, structuredFilters, }) - } else if (hasQuery && hasFilters) { - const strategy = getQueryStrategy(accessibleKbIds.length, topK) - const queryEmbeddingResult = await generateSearchEmbedding( - query!, - queryEmbeddingModel, - workspaceId - ) - queryEmbeddingIsBYOK = queryEmbeddingResult.isBYOK - const queryVector = JSON.stringify(queryEmbeddingResult.embedding) - results = await handleTagAndVectorSearch({ - knowledgeBaseIds: accessibleKbIds, - topK, - structuredFilters, - queryVector, - distanceThreshold: strategy.distanceThreshold, - }) } else if (hasQuery) { - const strategy = getQueryStrategy(accessibleKbIds.length, topK) const queryEmbeddingResult = await generateSearchEmbedding( query!, queryEmbeddingModel, @@ -220,11 +201,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) queryEmbeddingIsBYOK = queryEmbeddingResult.isBYOK const queryVector = JSON.stringify(queryEmbeddingResult.embedding) - results = await handleVectorOnlySearch({ + results = await executeKnowledgeSearch({ knowledgeBaseIds: accessibleKbIds, topK, + searchMode, + query, queryVector, - distanceThreshold: strategy.distanceThreshold, + structuredFilters: hasFilters ? structuredFilters : undefined, }) } else { return NextResponse.json( diff --git a/apps/sim/blocks/blocks/knowledge.ts b/apps/sim/blocks/blocks/knowledge.ts index 659f479dc0d..646d28c2656 100644 --- a/apps/sim/blocks/blocks/knowledge.ts +++ b/apps/sim/blocks/blocks/knowledge.ts @@ -88,6 +88,18 @@ export const KnowledgeBlock: BlockConfig = { dependsOn: ['knowledgeBaseSelector'], condition: { field: 'operation', value: 'search' }, }, + { + id: 'searchMode', + title: 'Retrieval Mode', + type: 'dropdown', + options: [ + { label: 'Hybrid (full-text + vector)', id: 'hybrid' }, + { label: 'Vector only', id: 'vector' }, + ], + value: () => 'hybrid', + mode: 'advanced', + condition: { field: 'operation', value: 'search' }, + }, { id: 'rerankerEnabled', title: 'Rerank Results', @@ -440,6 +452,10 @@ export const KnowledgeBlock: BlockConfig = { limit: { type: 'number', description: 'Max items to return' }, offset: { type: 'number', description: 'Pagination offset' }, tagFilters: { type: 'string', description: 'Tag filter criteria' }, + searchMode: { + type: 'string', + description: 'Retrieval mode: hybrid (full-text + vector) or vector only', + }, rerankerEnabled: { type: 'boolean', description: 'Apply Cohere reranking to search results' }, rerankerModel: { type: 'string', description: 'Cohere rerank model identifier' }, rerankerInputCount: { diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index ea1dff75ce0..a66c2955911 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -10,6 +10,16 @@ export const knowledgeSearchTagFilterSchema = z.object({ valueTo: z.union([z.string(), z.number()]).optional(), }) +export const KNOWLEDGE_SEARCH_MODES = ['hybrid', 'vector'] as const + +/** Shared by the internal and v1 search contracts so both default to hybrid. */ +export const knowledgeSearchModeSchema = z + .enum(KNOWLEDGE_SEARCH_MODES) + .optional() + .nullable() + .default('hybrid') + .transform((val) => val ?? 'hybrid') + export const knowledgeSearchBodySchema = z .object({ knowledgeBaseIds: z.union([ @@ -34,6 +44,12 @@ export const knowledgeSearchBodySchema = z .optional() .nullable() .transform((val) => val || undefined), + /** + * `hybrid` (default) fuses full-text and vector retrieval by reciprocal rank, + * which recovers exact tokens (error codes, ticket keys, identifiers) that + * embeddings alone rank poorly. `vector` opts out to semantic-only retrieval. + */ + searchMode: knowledgeSearchModeSchema, rerankerEnabled: z.boolean().optional().default(false), rerankerModel: rerankerModelSchema.optional().default(DEFAULT_RERANKER_MODEL), /** diff --git a/apps/sim/lib/api/contracts/v1/knowledge/index.ts b/apps/sim/lib/api/contracts/v1/knowledge/index.ts index 0134da1147a..1d87a6aaa9a 100644 --- a/apps/sim/lib/api/contracts/v1/knowledge/index.ts +++ b/apps/sim/lib/api/contracts/v1/knowledge/index.ts @@ -1,4 +1,5 @@ import { z } from 'zod' +import { knowledgeSearchModeSchema } from '@/lib/api/contracts/knowledge/search' import { knowledgeBaseParamsSchema, knowledgeDocumentParamsSchema, @@ -133,6 +134,11 @@ export const v1KnowledgeSearchBodySchema = z query: z.string().optional(), topK: z.number().min(1).max(100).default(10), tagFilters: z.array(v1SearchTagFilterSchema).optional(), + /** + * `hybrid` (default) fuses full-text and vector retrieval by reciprocal rank; + * `vector` opts out to semantic-only retrieval. + */ + searchMode: knowledgeSearchModeSchema, }) .refine( (data) => { diff --git a/apps/sim/tools/knowledge/search.ts b/apps/sim/tools/knowledge/search.ts index f3a028c8b5b..4fe33cf75e6 100644 --- a/apps/sim/tools/knowledge/search.ts +++ b/apps/sim/tools/knowledge/search.ts @@ -42,6 +42,13 @@ export const knowledgeSearchTool: ToolConfig = { }, }, }, + searchMode: { + type: 'string', + required: false, + visibility: 'user-only', + description: + "Retrieval mode: 'hybrid' (default) fuses full-text and vector search, 'vector' uses semantic similarity only", + }, rerankerEnabled: { type: 'boolean', required: false, @@ -114,6 +121,7 @@ export const knowledgeSearchTool: ToolConfig = { query: params.query, topK: params.topK ? Math.max(1, Math.min(100, Number(params.topK))) : 10, ...(structuredFilters.length > 0 && { tagFilters: structuredFilters }), + ...(params.searchMode === 'vector' && { searchMode: 'vector' }), ...(rerankerEnabled && { rerankerEnabled: true, rerankerModel, diff --git a/packages/testing/src/mocks/database.mock.ts b/packages/testing/src/mocks/database.mock.ts index 0ca0af53921..455c90f66c8 100644 --- a/packages/testing/src/mocks/database.mock.ts +++ b/packages/testing/src/mocks/database.mock.ts @@ -5,11 +5,16 @@ import { vi } from 'vitest' * Mimics drizzle-orm's sql tagged template. */ export function createMockSql() { - const sqlFn = (strings: TemplateStringsArray, ...values: any[]) => ({ - strings, - values, - toSQL: () => ({ sql: strings.join('?'), params: values }), - }) + const sqlFn = (strings: TemplateStringsArray, ...values: any[]) => { + const fragment = { + strings, + values, + toSQL: () => ({ sql: strings.join('?'), params: values }), + /** Mirrors drizzle's `sql``…`.as(alias)` for aliased select expressions. */ + as: (alias: string) => ({ ...fragment, alias }), + } + return fragment + } sqlFn.raw = (rawSql: string) => ({ rawSql, From 981395f9e342a6c93fb5bac041f473e559a46239 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 11:33:29 -0700 Subject: [PATCH 2/8] change(knowledge): make vector the default search mode, hybrid opt-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every existing caller — workflow block, v1 API, copilot, guardrail RAG — keeps its current ranking. Hybrid retrieval is now requested explicitly via `searchMode: 'hybrid'`. Also routes the copilot knowledge tool through the shared `executeKnowledgeSearch` dispatch so all four callers share one retrieval path, and documents `searchMode` on the public v1 search endpoint in the OpenAPI spec. --- apps/docs/openapi.json | 11 ++++- .../app/api/knowledge/search/route.test.ts | 44 +++---------------- .../app/api/v1/knowledge/search/route.test.ts | 4 -- apps/sim/blocks/blocks/knowledge.ts | 6 +-- .../sim/lib/api/contracts/knowledge/search.ts | 17 ++++--- .../lib/api/contracts/v1/knowledge/index.ts | 4 +- .../server/knowledge/knowledge-base.test.ts | 3 +- .../tools/server/knowledge/knowledge-base.ts | 9 ++-- apps/sim/tools/knowledge/search.ts | 4 +- 9 files changed, 38 insertions(+), 64 deletions(-) diff --git a/apps/docs/openapi.json b/apps/docs/openapi.json index 4967b8c8186..b2e8ca4c523 100644 --- a/apps/docs/openapi.json +++ b/apps/docs/openapi.json @@ -6040,7 +6040,7 @@ "post": { "operationId": "searchKnowledgeBase", "summary": "Search Knowledge Base", - "description": "Perform vector similarity search across one or more knowledge bases. Supports semantic search via query text, tag-based filtering, or a combination of both.", + "description": "Search across one or more knowledge bases. Supports semantic search via query text, tag-based filtering, or a combination of both. Set `searchMode` to `hybrid` to additionally run a full-text keyword leg and fuse it with the semantic results.", "tags": ["Knowledge Bases"], "x-codeSamples": [ { @@ -6095,6 +6095,12 @@ "items": { "$ref": "#/components/schemas/TagFilter" } + }, + "searchMode": { + "type": "string", + "enum": ["vector", "hybrid"], + "default": "vector", + "description": "Retrieval strategy. `vector` ranks purely on embedding similarity. `hybrid` also runs a full-text keyword search and fuses the two rankings by reciprocal rank, which retrieves exact tokens — error codes, ticket keys, identifiers, rare product names — that embeddings alone rank poorly. Ignored when only tagFilters are provided." } } }, @@ -6102,7 +6108,8 @@ "workspaceId": "wsp_abc123", "knowledgeBaseIds": ["d2c8f4a6-1b3e-4c5d-9e7f-8a0b2c4d6e1f"], "query": "How do I reset my password?", - "topK": 5 + "topK": 5, + "searchMode": "hybrid" } } } diff --git a/apps/sim/app/api/knowledge/search/route.test.ts b/apps/sim/app/api/knowledge/search/route.test.ts index 069e2ad5425..d12fd48e0aa 100644 --- a/apps/sim/app/api/knowledge/search/route.test.ts +++ b/apps/sim/app/api/knowledge/search/route.test.ts @@ -22,13 +22,11 @@ import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vites const { mockGetDocumentTagDefinitions, mockExecuteKnowledgeSearch, - mockGetQueryStrategy, mockGenerateSearchEmbedding, mockGetDocumentMetadataByIds, } = vi.hoisted(() => ({ mockGetDocumentTagDefinitions: vi.fn(), mockExecuteKnowledgeSearch: vi.fn(), - mockGetQueryStrategy: vi.fn(), mockGenerateSearchEmbedding: vi.fn(), mockGetDocumentMetadataByIds: vi.fn(), })) @@ -66,7 +64,6 @@ vi.mock('@/lib/knowledge/tags/service', () => ({ vi.mock('./utils', () => ({ executeKnowledgeSearch: mockExecuteKnowledgeSearch, - getQueryStrategy: mockGetQueryStrategy, generateSearchEmbedding: mockGenerateSearchEmbedding, getDocumentMetadataByIds: mockGetDocumentMetadataByIds, APIError: class APIError extends Error { @@ -113,12 +110,6 @@ describe('Knowledge Search API Route', () => { setEnv({ OPENAI_API_KEY: 'test-api-key' }) mockExecuteKnowledgeSearch.mockClear() - mockGetQueryStrategy.mockClear().mockReturnValue({ - useParallel: false, - distanceThreshold: 1.0, - parallelLimit: 15, - singleQueryOptimized: true, - }) mockGenerateSearchEmbedding .mockClear() .mockResolvedValue({ embedding: [0.1, 0.2, 0.3, 0.4, 0.5], isBYOK: false }) @@ -207,14 +198,14 @@ describe('Knowledge Search API Route', () => { expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ knowledgeBaseIds: ['kb-123'], topK: 10, - searchMode: 'hybrid', + searchMode: 'vector', query: validSearchData.query, queryVector: JSON.stringify(mockEmbedding), structuredFilters: undefined, }) }) - it('should forward the searchMode opt-out to the retrieval layer', async () => { + it('should forward the hybrid searchMode opt-in to the retrieval layer', async () => { mockGetUserId.mockResolvedValue('user-123') mockCheckKnowledgeBaseAccess.mockResolvedValue({ @@ -239,12 +230,12 @@ describe('Knowledge Search API Route', () => { }), }) - const req = createMockRequest('POST', { ...validSearchData, searchMode: 'vector' }) + const req = createMockRequest('POST', { ...validSearchData, searchMode: 'hybrid' }) const response = await POST(req) expect(response.status).toBe(200) expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith( - expect.objectContaining({ searchMode: 'vector' }) + expect.objectContaining({ searchMode: 'hybrid' }) ) }) @@ -287,7 +278,7 @@ describe('Knowledge Search API Route', () => { expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ knowledgeBaseIds: ['kb-123', 'kb-456'], topK: 10, - searchMode: 'hybrid', + searchMode: 'vector', query: multiKbData.query, queryVector: JSON.stringify(mockEmbedding), structuredFilters: undefined, @@ -796,7 +787,7 @@ describe('Knowledge Search API Route', () => { expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ knowledgeBaseIds: ['kb-123'], topK: 10, - searchMode: 'hybrid', + searchMode: 'vector', structuredFilters: [ { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api', valueTo: undefined }, ], @@ -850,7 +841,7 @@ describe('Knowledge Search API Route', () => { expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ knowledgeBaseIds: ['kb-123'], topK: 10, - searchMode: 'hybrid', + searchMode: 'vector', query: 'test search', queryVector: JSON.stringify(mockEmbedding), structuredFilters: [ @@ -1066,13 +1057,6 @@ describe('Knowledge Search API Route', () => { }, ]) - mockGetQueryStrategy.mockReturnValue({ - useParallel: false, - distanceThreshold: 1.0, - parallelLimit: 15, - singleQueryOptimized: true, - }) - mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2, 0.3], isBYOK: false }) mockGetDocumentMetadataByIds.mockResolvedValue({ 'doc-active': { @@ -1142,13 +1126,6 @@ describe('Knowledge Search API Route', () => { }, ]) - mockGetQueryStrategy.mockReturnValue({ - useParallel: false, - distanceThreshold: 1.0, - parallelLimit: 15, - singleQueryOptimized: true, - }) - mockGetDocumentMetadataByIds.mockResolvedValue({ 'doc-active-tagged': { filename: 'Active Tagged Document.pdf', sourceUrl: null }, }) @@ -1214,13 +1191,6 @@ describe('Knowledge Search API Route', () => { }, ]) - mockGetQueryStrategy.mockReturnValue({ - useParallel: false, - distanceThreshold: 1.0, - parallelLimit: 15, - singleQueryOptimized: true, - }) - mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2, 0.3], isBYOK: false }) mockGetDocumentMetadataByIds.mockResolvedValue({ 'doc-active-combined': { filename: 'Active Combined Search.pdf', sourceUrl: null }, diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index e8d110ccfed..8f75f565454 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -13,7 +13,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockExecuteKnowledgeSearch, - mockGetQueryStrategy, mockGenerateSearchEmbedding, mockGetDocumentMetadataByIds, mockAuthenticateRequest, @@ -23,7 +22,6 @@ const { mockRecordSearchEmbeddingUsage, } = vi.hoisted(() => ({ mockExecuteKnowledgeSearch: vi.fn(), - mockGetQueryStrategy: vi.fn(), mockGenerateSearchEmbedding: vi.fn(), mockGetDocumentMetadataByIds: vi.fn(), mockAuthenticateRequest: vi.fn(), @@ -48,7 +46,6 @@ const SYSTEM_BILLING_ATTRIBUTION = { vi.mock('@/app/api/knowledge/search/utils', () => ({ executeKnowledgeSearch: mockExecuteKnowledgeSearch, - getQueryStrategy: mockGetQueryStrategy, generateSearchEmbedding: mockGenerateSearchEmbedding, getDocumentMetadataByIds: mockGetDocumentMetadataByIds, })) @@ -109,7 +106,6 @@ describe('v1 knowledge search route — per-KB embedding model', () => { rateLimit: {}, }) mockValidateWorkspaceAccess.mockResolvedValue(null) - mockGetQueryStrategy.mockReturnValue({ distanceThreshold: 0.5 }) mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2, 0.3], isBYOK: false, diff --git a/apps/sim/blocks/blocks/knowledge.ts b/apps/sim/blocks/blocks/knowledge.ts index 646d28c2656..7debc95c938 100644 --- a/apps/sim/blocks/blocks/knowledge.ts +++ b/apps/sim/blocks/blocks/knowledge.ts @@ -93,10 +93,10 @@ export const KnowledgeBlock: BlockConfig = { title: 'Retrieval Mode', type: 'dropdown', options: [ - { label: 'Hybrid (full-text + vector)', id: 'hybrid' }, { label: 'Vector only', id: 'vector' }, + { label: 'Hybrid (full-text + vector)', id: 'hybrid' }, ], - value: () => 'hybrid', + value: () => 'vector', mode: 'advanced', condition: { field: 'operation', value: 'search' }, }, @@ -454,7 +454,7 @@ export const KnowledgeBlock: BlockConfig = { tagFilters: { type: 'string', description: 'Tag filter criteria' }, searchMode: { type: 'string', - description: 'Retrieval mode: hybrid (full-text + vector) or vector only', + description: 'Retrieval mode: vector only (default) or hybrid (full-text + vector)', }, rerankerEnabled: { type: 'boolean', description: 'Apply Cohere reranking to search results' }, rerankerModel: { type: 'string', description: 'Cohere rerank model identifier' }, diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index a66c2955911..3550311f21d 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -10,15 +10,18 @@ export const knowledgeSearchTagFilterSchema = z.object({ valueTo: z.union([z.string(), z.number()]).optional(), }) -export const KNOWLEDGE_SEARCH_MODES = ['hybrid', 'vector'] as const +export const KNOWLEDGE_SEARCH_MODES = ['vector', 'hybrid'] as const -/** Shared by the internal and v1 search contracts so both default to hybrid. */ +/** + * Shared by the internal and v1 search contracts. Defaults to `vector` so every + * existing caller keeps its current ranking; hybrid is opt-in. + */ export const knowledgeSearchModeSchema = z .enum(KNOWLEDGE_SEARCH_MODES) .optional() .nullable() - .default('hybrid') - .transform((val) => val ?? 'hybrid') + .default('vector') + .transform((val) => val ?? 'vector') export const knowledgeSearchBodySchema = z .object({ @@ -45,9 +48,9 @@ export const knowledgeSearchBodySchema = z .nullable() .transform((val) => val || undefined), /** - * `hybrid` (default) fuses full-text and vector retrieval by reciprocal rank, - * which recovers exact tokens (error codes, ticket keys, identifiers) that - * embeddings alone rank poorly. `vector` opts out to semantic-only retrieval. + * `vector` (default) is semantic-only retrieval. `hybrid` additionally runs a + * full-text leg and fuses the two by reciprocal rank, which recovers exact + * tokens (error codes, ticket keys, identifiers) that embeddings rank poorly. */ searchMode: knowledgeSearchModeSchema, rerankerEnabled: z.boolean().optional().default(false), diff --git a/apps/sim/lib/api/contracts/v1/knowledge/index.ts b/apps/sim/lib/api/contracts/v1/knowledge/index.ts index 1d87a6aaa9a..76abfcf16a0 100644 --- a/apps/sim/lib/api/contracts/v1/knowledge/index.ts +++ b/apps/sim/lib/api/contracts/v1/knowledge/index.ts @@ -135,8 +135,8 @@ export const v1KnowledgeSearchBodySchema = z topK: z.number().min(1).max(100).default(10), tagFilters: z.array(v1SearchTagFilterSchema).optional(), /** - * `hybrid` (default) fuses full-text and vector retrieval by reciprocal rank; - * `vector` opts out to semantic-only retrieval. + * `vector` (default) is semantic-only retrieval; `hybrid` fuses a full-text + * leg with it by reciprocal rank. */ searchMode: knowledgeSearchModeSchema, }) diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts index af3374b925b..186e785d29c 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts @@ -74,8 +74,7 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ resolveWorkspaceFileReference: vi.fn(), })) vi.mock('@/app/api/knowledge/search/utils', () => ({ - getQueryStrategy: vi.fn(), - handleVectorOnlySearch: vi.fn(), + executeKnowledgeSearch: vi.fn(), })) vi.mock('@/app/api/knowledge/utils', () => ({ checkDocumentWriteAccess: vi.fn(), diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index aececd01078..630d7f0de9d 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -50,7 +50,7 @@ import { } from '@/lib/knowledge/tags/service' import { StorageService } from '@/lib/uploads' import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { getQueryStrategy, handleVectorOnlySearch } from '@/app/api/knowledge/search/utils' +import { executeKnowledgeSearch } from '@/app/api/knowledge/search/utils' import { checkDocumentWriteAccess, checkKnowledgeBaseAccess, @@ -264,13 +264,12 @@ export const knowledgeBaseServerTool: BaseServerTool = { required: false, visibility: 'user-only', description: - "Retrieval mode: 'hybrid' (default) fuses full-text and vector search, 'vector' uses semantic similarity only", + "Retrieval mode: 'vector' (default) uses semantic similarity only, 'hybrid' also runs a full-text leg and fuses both", }, rerankerEnabled: { type: 'boolean', @@ -121,7 +121,7 @@ export const knowledgeSearchTool: ToolConfig = { query: params.query, topK: params.topK ? Math.max(1, Math.min(100, Number(params.topK))) : 10, ...(structuredFilters.length > 0 && { tagFilters: structuredFilters }), - ...(params.searchMode === 'vector' && { searchMode: 'vector' }), + ...(params.searchMode === 'hybrid' && { searchMode: 'hybrid' }), ...(rerankerEnabled && { rerankerEnabled: true, rerankerModel, From 336496006a12ea29dac7a676885735c34a68549b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 11:34:42 -0700 Subject: [PATCH 3/8] docs(knowledge): document the hybrid retrieval mode Regenerates the knowledge integration reference for the new searchMode tool param, and adds a Retrieval Mode section to the knowledge base workflow guide explaining when hybrid beats vector-only. --- .../content/docs/en/integrations/knowledge.mdx | 1 + .../docs/en/knowledgebase/using-in-workflows.mdx | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/apps/docs/content/docs/en/integrations/knowledge.mdx b/apps/docs/content/docs/en/integrations/knowledge.mdx index d5493b4bb8c..42ebb1b7eb3 100644 --- a/apps/docs/content/docs/en/integrations/knowledge.mdx +++ b/apps/docs/content/docs/en/integrations/knowledge.mdx @@ -43,6 +43,7 @@ Search for similar content in a knowledge base using vector similarity | `query` | string | No | Search query text \(optional when using tag filters\) | | `topK` | number | No | Number of most similar results to return \(1-100\) | | `tagFilters` | array | No | Array of tag filters with tagName and tagValue properties | +| `searchMode` | string | No | Retrieval mode: 'vector' \(default\) uses semantic similarity only, 'hybrid' also runs a full-text leg and fuses both | | `rerankerEnabled` | boolean | No | Whether to apply Cohere reranking to vector search results | | `rerankerModel` | string | No | Cohere rerank model to use \(one of: rerank-v4.0-pro, rerank-v4.0-fast, rerank-v3.5\) | | `rerankerInputCount` | number | No | Number of vector results sent to the Cohere reranker \(1–100\). Defaults to topK × 4 capped at 100. | diff --git a/apps/docs/content/docs/en/knowledgebase/using-in-workflows.mdx b/apps/docs/content/docs/en/knowledgebase/using-in-workflows.mdx index 00ffae3f3ab..be9bb00d462 100644 --- a/apps/docs/content/docs/en/knowledgebase/using-in-workflows.mdx +++ b/apps/docs/content/docs/en/knowledgebase/using-in-workflows.mdx @@ -37,6 +37,19 @@ In our example, adding `Department equals "Billing"` makes the search consider o Filters run before the vector comparison, so they make a search both more precise and cheaper. See [Tags and filtering](/knowledgebase/tags) for the full operator list by tag type. +## Retrieval Mode + +**Retrieval Mode** is an advanced setting that chooses how matches are found. + +| Mode | What it does | +| --- | --- | +| Vector only | The default. Ranks purely on meaning, as described above. | +| Hybrid | Also runs a keyword search over the same chunks and blends the two rankings. | + +Semantic search is strong on paraphrase and weak on literal strings: an error code, a ticket key like `PROJ-1234`, a SKU, or a rare product name carries little meaning for the model, so the chunk containing it may not rank near the top. Hybrid adds a keyword pass that matches those tokens exactly, then merges the two lists so a chunk found by either signal can surface. + +Turn it on when your documents are full of identifiers, codes, or names people search for verbatim. Leave it off for prose-heavy bases where questions are asked in natural language. Hybrid costs no extra API calls — the keyword pass runs entirely in the database. + ## Rerank Results **Rerank Results** is an optional second pass. Vector search ranks by raw similarity; reranking re-scores the top matches with a dedicated relevance model (Cohere's rerank models) and reorders them, which sharpens the ordering when the best answer isn't the literal closest vector. @@ -95,6 +108,7 @@ When the agent's answer is off, the cause is usually in retrieval, not the agent - **No results, or wrong documents.** A tag filter may be excluding what you want, or the documents may not be indexed yet. A document is only searchable once its processing status is `completed`; while it is `pending`, `processing`, or `failed`, its chunks won't appear. - **Low similarity scores across the board.** The query is too vague, or the information simply isn't in the base. Rewrite the query to match how the documents phrase things. - **Right documents, wrong order.** Turn on Rerank Results, or raise Number of Results so the relevant chunk is included. +- **An exact code, ID, or name isn't found.** Switch Retrieval Mode to Hybrid so a keyword pass runs alongside the semantic one. See [debugging retrieval](/knowledgebase/debugging-retrieval) for the full diagnostic path, and [chunking strategies](/knowledgebase/chunking-strategies) for how chunk boundaries shape what a search can return. From 930a1d488d640844a0469675fec2f6b9981df1ac Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 11:49:40 -0700 Subject: [PATCH 4/8] fix(knowledge): stop rank fusion from starving the lexical leg Rank n in one leg always ties rank n in the other, so ordering the fused list by score alone let whichever leg was scored first take every tied slot. At topK=1 that meant a hybrid search returned exactly the vector-only result and discarded the exact keyword match the mode exists to recover. Selection now orders by score and drains each tie group round-robin, taking from whichever leg has contributed fewest rows so far. The lexical leg is passed first so it wins a total tie, since a chunk the vector leg ranked below its distance threshold is the case hybrid was opted into for. --- .../app/api/knowledge/search/utils.test.ts | 47 ++++++++++++- apps/sim/app/api/knowledge/search/utils.ts | 69 +++++++++++++++++-- 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 227408ecb56..ab34bb4eab6 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -237,7 +237,10 @@ describe('Knowledge Search Utils', () => { 10 ) - expect(fused.map((r) => r.id)).toEqual(['shared', 'vector-only', 'keyword-only']) + expect(fused[0].id).toBe('shared') + // The two single-leg rows tie; `shared` was already credited to leg 0, so + // the round-robin owes leg 1 the next slot. + expect(fused.map((r) => r.id)).toEqual(['shared', 'keyword-only', 'vector-only']) }) it('dedupes by chunk id, keeping the first occurrence', () => { @@ -280,6 +283,48 @@ describe('Knowledge Search Utils', () => { expect(fused[0].id).toBe('deep-shared') }) + it('does not let the first leg starve the second at small topK', () => { + const lexicalOnly = makeResult('lexical-only') + const vectorOnly = makeResult('vector-only') + + /** + * Rank 1 in each leg scores identically. Ordering by score alone would + * always emit the first list's row, so a `topK: 1` hybrid search would + * return exactly what vector-only search already returned. + */ + expect(fuseByReciprocalRank([[lexicalOnly], [vectorOnly]], 1).map((r) => r.id)).toEqual([ + 'lexical-only', + ]) + expect(fuseByReciprocalRank([[lexicalOnly], [vectorOnly]], 2).map((r) => r.id)).toEqual([ + 'lexical-only', + 'vector-only', + ]) + }) + + it('interleaves tied ranks so neither leg monopolizes the head', () => { + const legA = [makeResult('a1'), makeResult('a2'), makeResult('a3')] + const legB = [makeResult('b1'), makeResult('b2'), makeResult('b3')] + + expect(fuseByReciprocalRank([legA, legB], 6).map((r) => r.id)).toEqual([ + 'a1', + 'b1', + 'a2', + 'b2', + 'a3', + 'b3', + ]) + }) + + it('still floats a row found by both legs above every single-leg row', () => { + const shared = makeResult('shared') + const legA = [makeResult('a1'), shared] + const legB = [makeResult('b1'), shared] + + // shared is rank 2 in both legs (2/62) and outscores either rank-1 row (1/61); + // it is credited to leg A, so tied `a1`/`b1` resolve in leg B's favor. + expect(fuseByReciprocalRank([legA, legB], 3).map((r) => r.id)).toEqual(['shared', 'b1', 'a1']) + }) + it('trims the fused list to topK', () => { const rows = Array.from({ length: 8 }, (_, i) => makeResult(`chunk-${i}`)) diff --git a/apps/sim/app/api/knowledge/search/utils.ts b/apps/sim/app/api/knowledge/search/utils.ts index 1c9892ff853..276719fa045 100644 --- a/apps/sim/app/api/knowledge/search/utils.ts +++ b/apps/sim/app/api/knowledge/search/utils.ts @@ -604,6 +604,14 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise * Rank fusion is used rather than score normalization because cosine distance * and `ts_rank_cd` are on incomparable scales with no corpus-independent * mapping between them. Rows are deduped by chunk id, first occurrence wins. + * + * Equal scores are common and must not be broken by list order: rank *n* in one + * leg always ties rank *n* in every other leg, so sorting alone would let the + * first list monopolize the head of the output and starve the others entirely + * at small `topK`. Selection therefore drains the legs round-robin among tied + * candidates, and a candidate from a leg that has contributed fewer rows so far + * wins the tie. A total tie goes to the earliest list, so callers put the leg + * whose hits the other leg cannot produce first. */ export function fuseByReciprocalRank(rankedLists: SearchResult[][], topK: number): SearchResult[] { const scores = new Map() @@ -618,9 +626,56 @@ export function fuseByReciprocalRank(rankedLists: SearchResult[][], topK: number }) } - return [...rowById.values()] - .sort((a, b) => (scores.get(b.id) ?? 0) - (scores.get(a.id) ?? 0)) - .slice(0, topK) + /** Leg each row is attributed to for interleaving: where it ranked best, earliest leg wins. */ + const legOfRow = new Map() + const bestRankOfRow = new Map() + rankedLists.forEach((list, leg) => { + list.forEach((row, index) => { + const currentBest = bestRankOfRow.get(row.id) + if (currentBest === undefined || index < currentBest) { + bestRankOfRow.set(row.id, index) + legOfRow.set(row.id, leg) + } + }) + }) + + // Stable sort keeps rowById insertion order (earliest leg first) inside each tie group. + const ordered = [...rowById.values()].sort( + (a, b) => (scores.get(b.id) ?? 0) - (scores.get(a.id) ?? 0) + ) + + const contributed = rankedLists.map(() => 0) + const fused: SearchResult[] = [] + let groupStart = 0 + + while (groupStart < ordered.length && fused.length < topK) { + const groupScore = scores.get(ordered[groupStart].id) ?? 0 + let groupEnd = groupStart + while (groupEnd < ordered.length && (scores.get(ordered[groupEnd].id) ?? 0) === groupScore) { + groupEnd++ + } + + // Drain this tie group round-robin, always taking from the leg that has contributed least. + const group = ordered.slice(groupStart, groupEnd) + while (group.length > 0 && fused.length < topK) { + let pick = 0 + for (let i = 1; i < group.length; i++) { + if ( + contributed[legOfRow.get(group[i].id) ?? 0] < + contributed[legOfRow.get(group[pick].id) ?? 0] + ) { + pick = i + } + } + const [row] = group.splice(pick, 1) + fused.push(row) + contributed[legOfRow.get(row.id) ?? 0]++ + } + + groupStart = groupEnd + } + + return fused } export async function handleTagAndVectorSearch(params: SearchParams): Promise { @@ -725,5 +780,11 @@ export async function executeKnowledgeSearch( const [vectorResults, keywordResults] = await Promise.all([vectorSearch, keywordSearch]) - return fuseByReciprocalRank([vectorResults, keywordResults], topK) + /** + * Lexical leg first: on a total tie it wins, which is the behavior this mode + * exists for — an exact-token chunk the vector leg ranked below its distance + * threshold is precisely what a caller opted into hybrid to recover, and at + * `topK: 1` something has to win. + */ + return fuseByReciprocalRank([keywordResults, vectorResults], topK) } From 318b236444b7e96ced147a1cc0ea0cb2e0227b17 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 12:01:58 -0700 Subject: [PATCH 5/8] fix(knowledge): credit a shared hit to every leg that returned it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Attributing a row found by both legs to a single leg left the round-robin owing the other leg a slot it had already been served. With a shared rank-1 hit and topK 2, that evicted the lexical-only row — the exact match hybrid was enabled to recover — in favor of the vector-only one. A shared row satisfied every leg that returned it, so every one of them is now charged for it. Tie-breaking prefers the candidate whose least-served leg has been served least, which also removes the arbitrary best-rank attribution. --- .../app/api/knowledge/search/utils.test.ts | 32 ++++++++++--- apps/sim/app/api/knowledge/search/utils.ts | 47 ++++++++++--------- 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index ab34bb4eab6..10d711f202b 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -238,9 +238,9 @@ describe('Knowledge Search Utils', () => { ) expect(fused[0].id).toBe('shared') - // The two single-leg rows tie; `shared` was already credited to leg 0, so - // the round-robin owes leg 1 the next slot. - expect(fused.map((r) => r.id)).toEqual(['shared', 'keyword-only', 'vector-only']) + // `shared` is credited to both legs, so the following tie is even and + // resolves to the earliest list. + expect(fused.map((r) => r.id)).toEqual(['shared', 'vector-only', 'keyword-only']) }) it('dedupes by chunk id, keeping the first occurrence', () => { @@ -320,9 +320,29 @@ describe('Knowledge Search Utils', () => { const legA = [makeResult('a1'), shared] const legB = [makeResult('b1'), shared] - // shared is rank 2 in both legs (2/62) and outscores either rank-1 row (1/61); - // it is credited to leg A, so tied `a1`/`b1` resolve in leg B's favor. - expect(fuseByReciprocalRank([legA, legB], 3).map((r) => r.id)).toEqual(['shared', 'b1', 'a1']) + // shared is rank 2 in both legs (2/62) and outscores either rank-1 row (1/61). + expect(fuseByReciprocalRank([legA, legB], 3).map((r) => r.id)).toEqual(['shared', 'a1', 'b1']) + }) + + it('does not let a shared top hit evict the lexical-only row at topK 2', () => { + const shared = makeResult('shared') + const lexicalOnly = makeResult('lexical-only') + const vectorOnly = makeResult('vector-only') + + /** + * `shared` is rank 1 in both legs. Crediting it to only one leg would + * leave the round-robin owing the other leg the remaining slot, evicting + * the row that only the shared hit's leg could produce. + */ + const fused = fuseByReciprocalRank( + [ + [shared, lexicalOnly], + [shared, vectorOnly], + ], + 2 + ) + + expect(fused.map((r) => r.id)).toEqual(['shared', 'lexical-only']) }) it('trims the fused list to topK', () => { diff --git a/apps/sim/app/api/knowledge/search/utils.ts b/apps/sim/app/api/knowledge/search/utils.ts index 276719fa045..f591a06c81d 100644 --- a/apps/sim/app/api/knowledge/search/utils.ts +++ b/apps/sim/app/api/knowledge/search/utils.ts @@ -608,33 +608,32 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise * Equal scores are common and must not be broken by list order: rank *n* in one * leg always ties rank *n* in every other leg, so sorting alone would let the * first list monopolize the head of the output and starve the others entirely - * at small `topK`. Selection therefore drains the legs round-robin among tied - * candidates, and a candidate from a leg that has contributed fewer rows so far - * wins the tie. A total tie goes to the earliest list, so callers put the leg - * whose hits the other leg cannot produce first. + * at small `topK`. Selection therefore drains each tie group round-robin, + * preferring the candidate whose least-served leg has been served least. + * + * A row is credited to *every* leg that returned it, not to one chosen leg: it + * satisfied all of them, and charging a shared hit to a single leg would leave + * the round-robin owing the other one a slot it has already been served — + * which at small `topK` evicts a row only the shared hit's leg could produce. + * A total tie goes to the earliest list, so callers put the leg whose hits the + * other leg cannot produce first. */ export function fuseByReciprocalRank(rankedLists: SearchResult[][], topK: number): SearchResult[] { const scores = new Map() const rowById = new Map() + const legsOfRow = new Map() - for (const list of rankedLists) { + rankedLists.forEach((list, leg) => { list.forEach((row, index) => { scores.set(row.id, (scores.get(row.id) ?? 0) + 1 / (RRF_K + index + 1)) if (!rowById.has(row.id)) { rowById.set(row.id, row) } - }) - } - - /** Leg each row is attributed to for interleaving: where it ranked best, earliest leg wins. */ - const legOfRow = new Map() - const bestRankOfRow = new Map() - rankedLists.forEach((list, leg) => { - list.forEach((row, index) => { - const currentBest = bestRankOfRow.get(row.id) - if (currentBest === undefined || index < currentBest) { - bestRankOfRow.set(row.id, index) - legOfRow.set(row.id, leg) + const legs = legsOfRow.get(row.id) + if (legs) { + if (!legs.includes(leg)) legs.push(leg) + } else { + legsOfRow.set(row.id, [leg]) } }) }) @@ -645,6 +644,10 @@ export function fuseByReciprocalRank(rankedLists: SearchResult[][], topK: number ) const contributed = rankedLists.map(() => 0) + /** How starved a candidate's most-neglected leg is; lower wins the tie. */ + const starvation = (id: string) => + Math.min(...(legsOfRow.get(id) ?? [0]).map((leg) => contributed[leg])) + const fused: SearchResult[] = [] let groupStart = 0 @@ -655,21 +658,19 @@ export function fuseByReciprocalRank(rankedLists: SearchResult[][], topK: number groupEnd++ } - // Drain this tie group round-robin, always taking from the leg that has contributed least. const group = ordered.slice(groupStart, groupEnd) while (group.length > 0 && fused.length < topK) { let pick = 0 for (let i = 1; i < group.length; i++) { - if ( - contributed[legOfRow.get(group[i].id) ?? 0] < - contributed[legOfRow.get(group[pick].id) ?? 0] - ) { + if (starvation(group[i].id) < starvation(group[pick].id)) { pick = i } } const [row] = group.splice(pick, 1) fused.push(row) - contributed[legOfRow.get(row.id) ?? 0]++ + for (const leg of legsOfRow.get(row.id) ?? []) { + contributed[leg]++ + } } groupStart = groupEnd From 07d80d0431ea00fd20d444d4e766a4a321a32669 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 12:11:04 -0700 Subject: [PATCH 6/8] fix(knowledge): reject a whitespace-only copilot query explicitly The shared dispatch treats a whitespace-only query as absent and throws when no tag filters accompany it, where the previous vector-only call would have embedded the blank string and searched. Tighten the existing guard so the tool returns its normal message instead. --- apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index 630d7f0de9d..bf19100c9ec 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -221,7 +221,7 @@ export const knowledgeBaseServerTool: BaseServerTool Date: Fri, 31 Jul 2026 12:18:15 -0700 Subject: [PATCH 7/8] fix(knowledge): fan the keyword leg out per knowledge base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vector leg caps candidates per base once getQueryStrategy sets useParallel, but the keyword leg always ran one global query with a single LIMIT. Searching several bases at once let whichever one ranks strongest lexically consume every slot, so an exact-token hit in a smaller base never reached fusion — the case hybrid exists to serve. The keyword leg now uses the same strategy: per-base queries under the same parallel limit, re-ranked globally on a selected ts_rank_cd. Both legs draw candidates the same way, so fusion combines rankings over the same pool. --- .../app/api/knowledge/search/utils.test.ts | 53 +++++++++++++++++++ apps/sim/app/api/knowledge/search/utils.ts | 51 ++++++++++++++++-- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 10d711f202b..373b34f5851 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -46,9 +46,11 @@ afterEach(() => { }) import { + executeKeywordSearch, executeKnowledgeSearch, fuseByReciprocalRank, generateSearchEmbedding, + getQueryStrategy, handleTagAndVectorSearch, handleTagOnlySearch, handleVectorOnlySearch, @@ -356,6 +358,57 @@ describe('Knowledge Search Utils', () => { }) }) + describe('executeKeywordSearch', () => { + beforeEach(() => { + resetDbChainMock() + }) + + it('returns nothing for a whitespace-only query without touching the database', async () => { + const results = await executeKeywordSearch({ + knowledgeBaseIds: ['kb-123'], + topK: 10, + query: ' ', + queryVector: JSON.stringify([0.1, 0.2, 0.3]), + }) + + expect(results).toEqual([]) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('issues one query per knowledge base once the parallel threshold is crossed', async () => { + const knowledgeBaseIds = ['kb-1', 'kb-2', 'kb-3', 'kb-4', 'kb-5'] + expect(getQueryStrategy(knowledgeBaseIds.length, 10).useParallel).toBe(true) + + await executeKeywordSearch({ + knowledgeBaseIds, + topK: 10, + query: 'PROJ-1234', + queryVector: JSON.stringify([0.1, 0.2, 0.3]), + }) + + /** + * A single global LIMIT would let the lexically strongest base consume + * every slot, so an exact-token hit in a smaller base never reaches + * fusion. The vector leg already fans out here; both legs must match. + */ + expect(dbChainMockFns.select).toHaveBeenCalledTimes(knowledgeBaseIds.length) + }) + + it('uses a single query when the parallel threshold is not crossed', async () => { + const knowledgeBaseIds = ['kb-1', 'kb-2'] + expect(getQueryStrategy(knowledgeBaseIds.length, 10).useParallel).toBe(false) + + await executeKeywordSearch({ + knowledgeBaseIds, + topK: 10, + query: 'PROJ-1234', + queryVector: JSON.stringify([0.1, 0.2, 0.3]), + }) + + expect(dbChainMockFns.select).toHaveBeenCalledTimes(1) + }) + }) + describe('executeKnowledgeSearch', () => { beforeEach(() => { resetDbChainMock() diff --git a/apps/sim/app/api/knowledge/search/utils.ts b/apps/sim/app/api/knowledge/search/utils.ts index f591a06c81d..80f3e34b5ce 100644 --- a/apps/sim/app/api/knowledge/search/utils.ts +++ b/apps/sim/app/api/knowledge/search/utils.ts @@ -564,6 +564,12 @@ export interface KeywordSearchParams { * can report `similarity` for rows only the lexical leg found. Unlike the vector * leg there is no distance threshold — surfacing exact-token matches that are * semantically distant is the entire point of this leg. + * + * Candidate gathering mirrors the vector leg's `getQueryStrategy`: across many + * knowledge bases a single global `LIMIT` lets whichever base ranks strongest + * lexically consume every slot, so an exact-token hit in a smaller base would + * never reach fusion. Both legs must draw candidates the same way, or rank + * fusion is combining rankings taken over differently-shaped pools. */ export async function executeKeywordSearch(params: KeywordSearchParams): Promise { const { knowledgeBaseIds, topK, query, queryVector, structuredFilters } = params @@ -573,16 +579,51 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise } const tsQuery = sql`websearch_to_tsquery(${FTS_CONFIG}, ${query})` + const rankExpr = sql`ts_rank_cd(${embedding.contentTsv}, ${tsQuery})` const tagFilterConditions = structuredFilters?.length ? getStructuredTagFilters(structuredFilters, embedding) : [] - return await db - .select( - getSearchResultFields( - sql`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') + /** Selected alongside the row so per-base batches can be re-ranked globally. */ + const selectFields = { + ...getSearchResultFields( + sql`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') + ), + keywordRank: rankExpr.as('keyword_rank'), + } + + const strategy = getQueryStrategy(knowledgeBaseIds.length, topK) + + if (strategy.useParallel) { + const parallelLimit = Math.ceil(topK / knowledgeBaseIds.length) + 5 + + const perBase = await Promise.all( + knowledgeBaseIds.map((kbId) => + db + .select(selectFields) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and( + eq(embedding.knowledgeBaseId, kbId), + ...getVisibilityConditions(), + sql`${embedding.contentTsv} @@ ${tsQuery}`, + ...tagFilterConditions + ) + ) + .orderBy(sql`${rankExpr} DESC`) + .limit(parallelLimit) ) ) + + return perBase + .flat() + .sort((a, b) => b.keywordRank - a.keywordRank) + .slice(0, topK) + } + + return await db + .select(selectFields) .from(embedding) .innerJoin(document, eq(embedding.documentId, document.id)) .where( @@ -593,7 +634,7 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise ...tagFilterConditions ) ) - .orderBy(sql`ts_rank_cd(${embedding.contentTsv}, ${tsQuery}) DESC`) + .orderBy(sql`${rankExpr} DESC`) .limit(topK) } From f1a8e9d3c33c6fd11270a2976e85826d6cd25025 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 12:32:28 -0700 Subject: [PATCH 8/8] perf(knowledge): stop the keyword leg detoasting every match's vector Selecting the cosine distance in the ranking query made Postgres detoast the 1536-dimension embedding and compute a distance for every full-text match before the LIMIT applied, so cost tracked how common the query term was rather than topK. On a 20k-chunk base with a term matching every row that was 61,055 buffer hits against 1,030 for the same query without the projection. Rank on ids and ts_rank_cd alone, then hydrate only the rows that survive the limit. Same results, and the worst case drops to ~27ms end to end. --- .../app/api/knowledge/search/utils.test.ts | 29 ++++++- apps/sim/app/api/knowledge/search/utils.ts | 87 ++++++++++--------- 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 373b34f5851..461289c09d2 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -394,6 +394,31 @@ describe('Knowledge Search Utils', () => { expect(dbChainMockFns.select).toHaveBeenCalledTimes(knowledgeBaseIds.length) }) + it('ranks without selecting the embedding column, then hydrates the survivors', async () => { + queueTableRows(schemaMock.embedding, [{ id: 'kw-1', keywordRank: 0.9 }]) + queueTableRows(schemaMock.embedding, [makeResult('kw-1')]) + + const results = await executeKeywordSearch({ + knowledgeBaseIds: ['kb-1'], + topK: 10, + query: 'PROJ-1234', + queryVector: JSON.stringify([0.1, 0.2, 0.3]), + }) + + expect(results.map((r) => r.id)).toEqual(['kw-1']) + expect(dbChainMockFns.select).toHaveBeenCalledTimes(2) + + /** + * Projecting the distance in the ranking pass makes Postgres detoast the + * 1536-dimension vector for every full-text match before the LIMIT, so + * cost tracks how common the term is rather than topK. The ranking pass + * must select ids and relevance only. + */ + const rankingSelect = dbChainMockFns.select.mock.calls[0][0] + expect(Object.keys(rankingSelect)).toEqual(['id', 'keywordRank']) + expect(Object.keys(dbChainMockFns.select.mock.calls[1][0])).toContain('distance') + }) + it('uses a single query when the parallel threshold is not crossed', async () => { const knowledgeBaseIds = ['kb-1', 'kb-2'] expect(getQueryStrategy(knowledgeBaseIds.length, 10).useParallel).toBe(false) @@ -451,7 +476,9 @@ describe('Knowledge Search Utils', () => { }) it('runs both legs and fuses them in hybrid mode', async () => { + // Vector leg, then the keyword leg's ranking pass, then its hydration pass. queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) + queueTableRows(schemaMock.embedding, [{ id: 'keyword-hit', keywordRank: 0.9 }]) queueTableRows(schemaMock.embedding, [makeResult('keyword-hit')]) const results = await executeKnowledgeSearch({ @@ -463,7 +490,7 @@ describe('Knowledge Search Utils', () => { }) expect(results.map((r) => r.id).sort()).toEqual(['keyword-hit', 'vector-hit']) - expect(dbChainMockFns.select).toHaveBeenCalledTimes(2) + expect(dbChainMockFns.select).toHaveBeenCalledTimes(3) }) it('falls back to vector results when the keyword leg fails', async () => { diff --git a/apps/sim/app/api/knowledge/search/utils.ts b/apps/sim/app/api/knowledge/search/utils.ts index 80f3e34b5ce..1a3b62fbf87 100644 --- a/apps/sim/app/api/knowledge/search/utils.ts +++ b/apps/sim/app/api/knowledge/search/utils.ts @@ -2,7 +2,7 @@ import { db } from '@sim/db' import { document, embedding } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import { and, eq, inArray, isNull, type SQL, sql } from 'drizzle-orm' import type { StructuredFilter } from '@/lib/knowledge/types' const logger = createLogger('KnowledgeSearch') @@ -570,6 +570,14 @@ export interface KeywordSearchParams { * lexically consume every slot, so an exact-token hit in a smaller base would * never reach fusion. Both legs must draw candidates the same way, or rank * fusion is combining rankings taken over differently-shaped pools. + * + * Ranking and hydration are two steps on purpose. Projecting the cosine + * distance in the ranking query makes Postgres detoast the 1536-dimension + * vector and compute a distance for *every* full-text match before the `LIMIT` + * applies — work that scales with how common the query term is rather than + * with `topK` (measured at ~59x the buffer reads on a 20k-chunk base for a term + * matching every row). Ranking therefore touches no vectors, and only the rows + * that survive the limit are hydrated. */ export async function executeKeywordSearch(params: KeywordSearchParams): Promise { const { knowledgeBaseIds, topK, query, queryVector, structuredFilters } = params @@ -584,58 +592,55 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise ? getStructuredTagFilters(structuredFilters, embedding) : [] - /** Selected alongside the row so per-base batches can be re-ranked globally. */ - const selectFields = { - ...getSearchResultFields( - sql`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') - ), - keywordRank: rankExpr.as('keyword_rank'), - } + const rankConditions = (kbScope: SQL | undefined) => + and( + kbScope, + ...getVisibilityConditions(), + sql`${embedding.contentTsv} @@ ${tsQuery}`, + ...tagFilterConditions + ) + + /** Ranking pass: ids and relevance only, so no vector is read. */ + const rankRows = (kbScope: SQL | undefined, limit: number) => + db + .select({ id: embedding.id, keywordRank: rankExpr.as('keyword_rank') }) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where(rankConditions(kbScope)) + .orderBy(sql`${rankExpr} DESC`) + .limit(limit) const strategy = getQueryStrategy(knowledgeBaseIds.length, topK) + let ranked: { id: string; keywordRank: number }[] if (strategy.useParallel) { const parallelLimit = Math.ceil(topK / knowledgeBaseIds.length) + 5 - const perBase = await Promise.all( - knowledgeBaseIds.map((kbId) => - db - .select(selectFields) - .from(embedding) - .innerJoin(document, eq(embedding.documentId, document.id)) - .where( - and( - eq(embedding.knowledgeBaseId, kbId), - ...getVisibilityConditions(), - sql`${embedding.contentTsv} @@ ${tsQuery}`, - ...tagFilterConditions - ) - ) - .orderBy(sql`${rankExpr} DESC`) - .limit(parallelLimit) - ) + knowledgeBaseIds.map((kbId) => rankRows(eq(embedding.knowledgeBaseId, kbId), parallelLimit)) ) + ranked = perBase.flat().sort((a, b) => b.keywordRank - a.keywordRank) + } else { + ranked = await rankRows(inArray(embedding.knowledgeBaseId, knowledgeBaseIds), topK) + } - return perBase - .flat() - .sort((a, b) => b.keywordRank - a.keywordRank) - .slice(0, topK) + const topIds = ranked.slice(0, topK).map((row) => row.id) + if (topIds.length === 0) { + return [] } - return await db - .select(selectFields) - .from(embedding) - .innerJoin(document, eq(embedding.documentId, document.id)) - .where( - and( - inArray(embedding.knowledgeBaseId, knowledgeBaseIds), - ...getVisibilityConditions(), - sql`${embedding.contentTsv} @@ ${tsQuery}`, - ...tagFilterConditions + /** Hydration pass: full rows plus the cosine distance, bounded to the survivors. */ + const hydrated = await db + .select( + getSearchResultFields( + sql`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') ) ) - .orderBy(sql`${rankExpr} DESC`) - .limit(topK) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where(and(inArray(embedding.id, topIds), ...getVisibilityConditions())) + + const rowById = new Map(hydrated.map((row) => [row.id, row])) + return topIds.map((id) => rowById.get(id)).filter((row): row is SearchResult => row !== undefined) } /**