diff --git a/apps/docs/openapi-v2-tables.json b/apps/docs/openapi-v2-tables.json new file mode 100644 index 00000000000..15757b19f1e --- /dev/null +++ b/apps/docs/openapi-v2-tables.json @@ -0,0 +1,381 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Sim Tables API v2", + "version": "2.0.0-preview", + "description": "Read access to Sim tables with the typed predicate filter grammar and opaque cursor pagination. This surface is feature-gated (`tables-v2-api`): when the flag is off for the caller, every endpoint returns 404 as if it does not exist. Filters are predicate trees — `{\"all\": [...]}` (AND) or `{\"any\": [...]}` (OR) groups whose members are `{field, op, value}` conditions or nested groups. Built-in columns `id`, `createdAt`, and `updatedAt` (camelCase) are filterable and sortable alongside user columns." + }, + "servers": [{ "url": "https://www.sim.ai" }], + "security": [{ "apiKey": [] }], + "paths": { + "/api/v2/tables": { + "get": { + "operationId": "v2ListTables", + "summary": "List Tables", + "description": "List every table in a workspace with its column schema and row count.", + "tags": ["Tables v2"], + "parameters": [ + { + "name": "workspaceId", + "in": "query", + "required": true, + "schema": { "type": "string", "minLength": 1 } + } + ], + "responses": { + "200": { + "description": "Tables in the workspace. Served with `Cache-Control: private, no-store`.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["success", "data"], + "properties": { + "success": { "const": true }, + "data": { + "type": "object", + "required": ["tables", "totalCount"], + "properties": { + "tables": { + "type": "array", + "items": { "$ref": "#/components/schemas/TableSummary" } + }, + "totalCount": { "type": "integer" } + } + } + } + } + } + } + }, + "400": { "$ref": "#/components/responses/ValidationError" }, + "401": { "$ref": "#/components/responses/Unauthorized" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFoundOrGated" }, + "429": { "$ref": "#/components/responses/RateLimited" } + } + } + }, + "/api/v2/tables/{tableId}/query": { + "post": { + "operationId": "v2QueryTableRows", + "summary": "Query Rows", + "description": "Query rows with a typed predicate filter, an ordered sort spec, and opaque cursor pagination. Row `data` is keyed by column NAME; `select` cells return option names, and filter operands on select columns accept option names (resolved case-insensitively).\n\n**Pagination contract:** page by passing the previous response's `nextCursor` back as `cursor`, and stop only when it is `null` — a page may return fewer than `limit` rows and still have more behind it, so page fullness is never a termination signal. A cursor is bound to the exact query shape it was minted under: keyset cursors to the default row order, offset cursors (sorted views) to that sort. Replaying one under a different `sort` returns 400 `CURSOR_SORT_CONFLICT`. `totalCount` is computed on the first page only (requests with a `cursor` return `totalCount: null`).", + "tags": ["Tables v2"], + "parameters": [ + { + "name": "tableId", + "in": "path", + "required": true, + "schema": { "type": "string", "minLength": 1 } + } + ], + "requestBody": { + "required": true, + "description": "Bodies over 1 MB are rejected with 413.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["workspaceId"], + "properties": { + "workspaceId": { "type": "string", "minLength": 1 }, + "predicate": { "$ref": "#/components/schemas/Predicate" }, + "sort": { + "type": "array", + "maxItems": 16, + "description": "Ordered sort spec, highest priority first.", + "items": { + "type": "object", + "required": ["field", "direction"], + "properties": { + "field": { "type": "string" }, + "direction": { "enum": ["asc", "desc"] } + } + } + }, + "limit": { + "type": "integer", + "minimum": 0, + "maximum": 1000, + "default": 100, + "description": "Omitted → 100. `1..1000` → page size. `0` → the ENTIRE matching result in one response; fails with 400 `TABLE_QUERY_RESULT_TOO_LARGE` if it exceeds the 5 MB row-data budget (narrow the predicate or page instead)." + }, + "cursor": { + "type": "string", + "description": "Opaque token from a previous response's `nextCursor`. Pass back verbatim. Mutually exclusive with `sort`." + } + } + }, + "examples": { + "filtered": { + "summary": "Multi-select membership + negated pattern", + "value": { + "workspaceId": "ws_123", + "predicate": { + "all": [ + { "field": "Color", "op": "contains", "value": "Purple" }, + { "field": "name", "op": "nlike", "value": "G*" } + ] + }, + "limit": 100 + } + }, + "builtinColumns": { + "summary": "Built-in column range (UTC, timezone-independent)", + "value": { + "workspaceId": "ws_123", + "predicate": { + "all": [ + { "field": "createdAt", "op": "gte", "value": "2026-07-24T03:00:00.000Z" }, + { "field": "createdAt", "op": "lte", "value": "2026-07-25T02:59:59.999Z" } + ] + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "A page of rows. Served with `Cache-Control: private, no-store`.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["success", "data"], + "properties": { + "success": { "const": true }, + "data": { + "type": "object", + "required": ["rows", "rowCount", "nextCursor"], + "properties": { + "rows": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "data", "createdAt", "updatedAt"], + "properties": { + "id": { "type": "string" }, + "data": { + "type": "object", + "description": "Column-NAME-keyed cell values.", + "additionalProperties": true + }, + "createdAt": { "type": "string", "format": "date-time" }, + "updatedAt": { "type": "string", "format": "date-time" } + } + } + }, + "rowCount": { "type": "integer", "description": "Rows in THIS page." }, + "totalCount": { + "type": ["integer", "null"], + "description": "Rows matching the predicate across all pages. First page only; null when a cursor was supplied." + }, + "limit": { "type": ["integer", "null"] }, + "nextCursor": { + "type": ["string", "null"], + "description": "Non-null ⇒ more rows exist. The ONLY termination signal is null." + } + } + } + } + } + } + } + }, + "400": { + "description": "Validation failure. Machine-readable `code` values include `INVALID_FILTER` (unknown column, operator/type mismatch, malformed tree), `INVALID_ORDER`, `INVALID_CURSOR`, `CURSOR_SORT_CONFLICT`, and `TABLE_QUERY_RESULT_TOO_LARGE` (unbounded result exceeded the 5 MB budget).", + "content": { + "application/json": { + "schema": { "$ref": "#/components/schemas/ErrorBody" } + } + } + }, + "401": { "$ref": "#/components/responses/Unauthorized" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFoundOrGated" }, + "413": { + "description": "Request body exceeded the 1 MB cap.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "429": { "$ref": "#/components/responses/RateLimited" } + } + } + } + }, + "components": { + "securitySchemes": { + "apiKey": { "type": "apiKey", "in": "header", "name": "X-API-Key" } + }, + "schemas": { + "Predicate": { + "description": "A predicate tree: exactly one of `all` (every member must match) or `any` (at least one must). Members are conditions or nested groups; nesting expresses mixed AND/OR logic. Groups must be non-empty (1–100 members), trees at most 10 levels deep and 500 nodes total. Nodes are STRICT: unknown keys, or a node carrying both a group key and condition keys, are rejected rather than ignored.", + "oneOf": [ + { + "type": "object", + "required": ["all"], + "additionalProperties": false, + "properties": { + "all": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { "$ref": "#/components/schemas/PredicateNode" } + } + } + }, + { + "type": "object", + "required": ["any"], + "additionalProperties": false, + "properties": { + "any": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { "$ref": "#/components/schemas/PredicateNode" } + } + } + } + ] + }, + "PredicateNode": { + "oneOf": [ + { "$ref": "#/components/schemas/Predicate" }, + { "$ref": "#/components/schemas/Condition" } + ] + }, + "Condition": { + "type": "object", + "required": ["field", "op"], + "additionalProperties": false, + "properties": { + "field": { + "type": "string", + "maxLength": 128, + "description": "Column name, or a built-in: `id`, `createdAt`, `updatedAt` (camelCase — snake_case is treated as a user column and matches nothing)." + }, + "op": { + "enum": [ + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "in", + "nin", + "contains", + "ncontains", + "startsWith", + "endsWith", + "like", + "ilike", + "nlike", + "nilike", + "isEmpty", + "isNotEmpty", + "isNull", + "isNotNull" + ], + "description": "`eq`/`ne`/`in`/`nin` are case-sensitive equality/membership. `contains`/`ncontains`/`startsWith`/`endsWith` are case-insensitive text matches — except on a multi-select column, where `contains`/`ncontains` mean set membership by option name. `like`/`nlike` are case-sensitive and `ilike`/`nilike` case-insensitive patterns with `*` as the only wildcard (literal `%`/`_` match themselves). `isEmpty`/`isNotEmpty` treat null and empty string as empty; `isNull`/`isNotNull` are strict null checks. The four `is*` operators take no `value`. Negated text matches retain rows where the cell is absent. `in`/`nin` require a non-empty array of at most 1000 values; other value-taking operators reject arrays. Select columns accept only equality/membership operators appropriate to their cardinality (single: eq/ne/in/nin; multi: contains/ncontains; both: the `is*` checks)." + }, + "value": { + "description": "Operand. Omit for the `is*` operators. Ranges on `number` columns require numbers, on `date` columns ISO strings (compared as UTC, independent of any session timezone); ranges on `boolean`/`json` columns are rejected." + } + } + }, + "TableSummary": { + "type": "object", + "required": ["id", "name", "schema", "rowCount", "maxRows", "createdAt", "updatedAt"], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "schema": { + "type": "object", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "type"], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "type": { "enum": ["string", "number", "boolean", "date", "json", "select"] }, + "required": { "type": "boolean" }, + "unique": { "type": "boolean" }, + "options": { + "type": "array", + "description": "Declared choices on a `select` column.", + "items": { + "type": "object", + "properties": { "id": { "type": "string" }, "name": { "type": "string" } } + } + }, + "multiple": { "type": "boolean" } + } + } + } + } + }, + "rowCount": { "type": "integer" }, + "maxRows": { "type": "integer" }, + "createdAt": { "type": "string", "format": "date-time" }, + "updatedAt": { "type": "string", "format": "date-time" } + } + }, + "ErrorBody": { + "type": "object", + "required": ["error"], + "properties": { + "error": { + "type": "string", + "description": "Human-readable message naming the failing field/operator." + }, + "code": { + "type": "string", + "description": "Machine-readable code, present on domain validation failures." + } + } + } + }, + "responses": { + "ValidationError": { + "description": "Malformed request.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "Unauthorized": { + "description": "Missing or invalid API key.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "Forbidden": { + "description": "The key's workspace scope does not cover this workspace, or the caller lacks read access.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "NotFoundOrGated": { + "description": "Table not found — or the `tables-v2-api` feature flag is off for this caller, in which case the entire surface answers 404. The gate is evaluated after authorization, so a 404 never distinguishes rollout cohort from missing resource for callers without access.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "RateLimited": { + "description": "Rate limit exceeded for this key.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + } + } + } +} diff --git a/apps/docs/openapi.json b/apps/docs/openapi.json index 1844cc7a1e2..4967b8c8186 100644 --- a/apps/docs/openapi.json +++ b/apps/docs/openapi.json @@ -3405,7 +3405,7 @@ "name": "sort", "in": "query", "required": false, - "description": "JSON-encoded sort object. Example: {\"created_at\": \"desc\"}.", + "description": "JSON-encoded sort object. Example: {\"createdAt\": \"desc\"}. Built-in columns are camelCase: id, createdAt, updatedAt.", "schema": { "type": "string" } diff --git a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts index ce656d6be50..acace4561c5 100644 --- a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts +++ b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts @@ -5,6 +5,8 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { TableQueryValidationError } from '@/lib/table/errors' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import { cancelWorkflowGroupRuns } from '@/lib/table/workflow-columns' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -32,7 +34,10 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(cancelTableRunsContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, scope, rowId, filter, excludeRowIds } = parsed.data.body + const { workspaceId, scope, rowId, filter: wireFilter, excludeRowIds } = parsed.data.body + // Dual-grammar wire: a predicate downgrades losslessly-or-throws to the + // legacy Filter the runners/persisted payloads still compile. + const filter = toLegacyFilter(wireFilter) const result = await checkAccess(tableId, authResult.userId, 'write') if (!result.ok) return accessError(result, requestId, tableId) @@ -42,7 +47,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } - const filterError = tableFilterError(filter, table.schema.columns) + const filterError = tableFilterError(wireFilter, table.schema.columns) if (filterError) return filterError const cancelled = await cancelWorkflowGroupRuns(tableId, scope === 'row' ? rowId : undefined, { @@ -57,6 +62,11 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ success: true, data: { cancelled } }) } catch (error) { + // A predicate that Zod accepts but the downgrade rejects (hybrid node, + // eq-with-array, valueless op) is caller error, not a server fault. + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } logger.error(`[${requestId}] cancel-runs failed:`, error) return NextResponse.json({ error: 'Failed to cancel runs' }, { status: 500 }) } diff --git a/apps/sim/app/api/table/[tableId]/columns/run/route.ts b/apps/sim/app/api/table/[tableId]/columns/run/route.ts index 00856ae4a1a..824ce73ddb4 100644 --- a/apps/sim/app/api/table/[tableId]/columns/run/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/run/route.ts @@ -5,6 +5,8 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { TableQueryValidationError } from '@/lib/table/errors' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import { runWorkflowColumn } from '@/lib/table/workflow-columns' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -25,13 +27,23 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(runColumnContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, groupIds, runMode, rowIds, filter, excludeRowIds, limit } = - parsed.data.body + const { + workspaceId, + groupIds, + runMode, + rowIds, + filter: wireFilter, + excludeRowIds, + limit, + } = parsed.data.body + // Dual-grammar wire: downgrade a predicate to the legacy Filter the + // dispatcher and scheduled runs still compile. + const filter = toLegacyFilter(wireFilter) const access = await checkAccess(tableId, auth.userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) // Validate the filter up front (the dispatcher reuses it) so a bad field fails fast. - const filterError = tableFilterError(filter, access.table.schema.columns) + const filterError = tableFilterError(wireFilter, access.table.schema.columns) if (filterError) return filterError const { dispatchId } = await runWorkflowColumn({ @@ -49,6 +61,11 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ success: true, data: { dispatchId } }) } catch (error) { + // A predicate that Zod accepts but the downgrade rejects (hybrid node, + // eq-with-array, valueless op) is caller error, not a server fault. + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } if (error instanceof Error && error.message === 'Invalid workspace ID') { return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } diff --git a/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts b/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts index 38b24e06fb0..cffb5dc8810 100644 --- a/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts @@ -208,4 +208,25 @@ describe('POST /api/table/[tableId]/delete-async', () => { expect(mockReleaseJobClaim).toHaveBeenCalledWith('tbl_1', 'job-id-xyz') expect(mockRunTableDelete).not.toHaveBeenCalled() }) + + /** + * PR #6067 review finding (greptile P1 / bugbot High): a hybrid filter — group + * key AND leaf keys on one node — passes the dual-grammar union via the + * non-stripping legacy branch, and the downgrade used to convert group-first, + * silently dropping the leaf and WIDENING an async select-all delete. + */ + it('rejects a hybrid group+leaf filter with 400 instead of widening the delete', async () => { + const response = await makeRequest({ + workspaceId: 'workspace-1', + filter: { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + }, + }) + expect(response.status).toBe(400) + const body = await response.json() + expect(body.error).toMatch(/not both/) + }) }) diff --git a/apps/sim/app/api/table/[tableId]/delete-async/route.ts b/apps/sim/app/api/table/[tableId]/delete-async/route.ts index 236eb556240..83382ea2ddd 100644 --- a/apps/sim/app/api/table/[tableId]/delete-async/route.ts +++ b/apps/sim/app/api/table/[tableId]/delete-async/route.ts @@ -8,9 +8,12 @@ import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { runDetached } from '@/lib/core/utils/background' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { Filter } from '@/lib/table' import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner' +import { TableQueryValidationError } from '@/lib/table/errors' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { assertRowDelete } from '@/lib/table/mutation-locks' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import type { TableDeleteJobPayload } from '@/lib/table/types' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -43,7 +46,20 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(deleteTableRowsAsyncContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, filter, excludeRowIds, estimatedCount } = parsed.data.body + const { workspaceId, filter: wireFilter, excludeRowIds, estimatedCount } = parsed.data.body + // Dual-grammar wire: a predicate downgrades losslessly-or-throws to the + // legacy Filter the runners/persisted payloads still compile. A shape the + // union accepted but the downgrade rejects (hybrid node, eq-with-array) is + // caller error — 400, never the generic 500. + let filter: Filter | undefined + try { + filter = toLegacyFilter(wireFilter) + } catch (error) { + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + throw error + } const access = await checkAccess(tableId, userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) @@ -62,7 +78,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro assertRowDelete(table) // Validate the filter up front so the caller gets immediate feedback (the worker reuses it). - const filterError = tableFilterError(filter, table.schema.columns) + const filterError = tableFilterError(wireFilter, table.schema.columns) if (filterError) return filterError // Rows inserted after this instant are spared (created_at <= cutoff in the worker). diff --git a/apps/sim/app/api/table/[tableId]/export/route.ts b/apps/sim/app/api/table/[tableId]/export/route.ts index 6d717bdcdc3..58df047c629 100644 --- a/apps/sim/app/api/table/[tableId]/export/route.ts +++ b/apps/sim/app/api/table/[tableId]/export/route.ts @@ -111,7 +111,9 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou } } - if (result.rows.length < EXPORT_BATCH_SIZE) break + // A page can be cut by the byte budget before reaching EXPORT_BATCH_SIZE, + // so a short page does NOT mean the export is done — only a null cursor does. + if (!result.nextCursor) break offset += result.rows.length } diff --git a/apps/sim/app/api/table/[tableId]/query/route.test.ts b/apps/sim/app/api/table/[tableId]/query/route.test.ts new file mode 100644 index 00000000000..fb727cd53f3 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/query/route.test.ts @@ -0,0 +1,202 @@ +/** + * @vitest-environment node + * + * v2 query route: predicate parsing, unconditional name→id translation + * (session auth included — the string grammar is name-keyed for every caller), + * cursor validation, and the response envelope. + */ +import { hybridAuthMockFns } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockCheckAccess, mockQueryRows, mockGate } = vi.hoisted(() => ({ + mockCheckAccess: vi.fn(), + mockQueryRows: vi.fn(), + mockGate: vi.fn(), +})) + +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => + NextResponse.json({ error: 'Access denied' }, { status: result.status }), + tablesV2GateError: mockGate, + } +}) + +vi.mock('@/lib/table', async () => { + // row-wire pulls the column-keys helpers through this barrel. + const columnKeys = await import('@/lib/table/column-keys') + return { ...columnKeys } +}) + +vi.mock('@/lib/table/rows/service', () => ({ + queryRows: mockQueryRows, +})) + +import { encodeCursor } from '@/lib/table/rows/cursor' +import { POST } from '@/app/api/table/[tableId]/query/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { id: 'col_aaa', name: 'name', type: 'string' }, + { id: 'col_bbb', name: 'wins', type: 'number' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + } +} + +function authAs(authType: 'session' | 'internal_jwt') { + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType, + }) +} + +function callQuery(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/query', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) +} + +const EMPTY_RESULT = { + rows: [], + rowCount: 0, + totalCount: 0, + limit: 0, + offset: 0, + nextCursor: null, +} + +describe('POST /api/table/[tableId]/query', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockQueryRows.mockResolvedValue(EMPTY_RESULT) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + authAs('session') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(404) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + authAs('session') + mockCheckAccess.mockResolvedValue({ ok: false, status: 403 }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() + }) + + it('translates predicate/sort column names to storage ids for SESSION auth too', async () => { + authAs('session') + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { + all: [ + { field: 'name', op: 'eq', value: 'John' }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }, + sort: [{ field: 'wins', direction: 'desc' }], + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ + all: [ + { field: 'col_aaa', op: 'eq', value: 'John' }, + { field: 'col_bbb', op: 'gte', value: 10 }, + ], + }) + expect(options.sort).toEqual({ col_bbb: 'desc' }) + expect(options.withExecutions).toBe(false) + }) + + it('rejects a keyset cursor combined with a custom sort', async () => { + authAs('internal_jwt') + const cursor = encodeCursor({ + lastRow: { id: 'row_1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + const res = await callQuery({ + workspaceId: 'workspace-1', + sort: [{ field: 'wins', direction: 'desc' }], + cursor, + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.error).toMatch(/not valid for a sorted query/) + expect(body.code).toBe('CURSOR_SORT_CONFLICT') + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('returns 400 (not 500) for a cursor that decodes to a JSON primitive', async () => { + authAs('internal_jwt') + const res = await callQuery({ + workspaceId: 'workspace-1', + cursor: Buffer.from('42').toString('base64url'), + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.error).toBe('Invalid cursor') + expect(body.code).toBe('INVALID_CURSOR') + }) + + it('returns 400 for a predicate referencing an unknown column', async () => { + authAs('internal_jwt') + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'nope', op: 'eq', value: 1 }] }, + }) + + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter column/) + }) + + it('passes nextCursor through the response envelope and skips the count on later pages', async () => { + authAs('internal_jwt') + mockQueryRows.mockResolvedValue({ ...EMPTY_RESULT, nextCursor: 'tok' }) + const cursor = encodeCursor({ + lastRow: { id: 'row_1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + + const res = await callQuery({ workspaceId: 'workspace-1', cursor }) + + expect(res.status).toBe(200) + const body = await res.json() + expect(body.data.nextCursor).toBe('tok') + const options = mockQueryRows.mock.calls[0][1] + expect(options.includeTotal).toBe(false) + expect(options.after).toEqual({ orderKey: 'a1', id: 'row_1' }) + }) +}) diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts new file mode 100644 index 00000000000..34a162200d1 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -0,0 +1,134 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { rowQueryContract, TABLE_QUERY_MAX_BODY_BYTES } from '@/lib/api/contracts/tables' +import { parseRequest } from '@/lib/api/server' +import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' +import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { Sort, TableSchema } from '@/lib/table' +import { buildIdByName, sortSpecNamesToIds } from '@/lib/table/column-keys' +import { TableQueryValidationError } from '@/lib/table/errors' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' +import { queryRows } from '@/lib/table/rows/service' +import { predicateToStorage } from '@/lib/table/select-values' +import { rowWireTranslators } from '@/app/api/table/row-wire' +import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' + +const logger = createLogger('TableRowQueryAPI') + +interface RowQueryRouteParams { + params: Promise<{ tableId: string }> +} + +/** + * POST /api/table/[tableId]/query — v2 row query. Typed `predicate`/`sort` + * objects (validated server-side) + opaque cursor pagination (no offset on the + * wire). Shares the same engine as the legacy GET /rows route via `queryRows`. + */ +export const POST = withRouteHandler(async (request: NextRequest, context: RowQueryRouteParams) => { + const requestId = generateRequestId() + + try { + const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) + } + + const parsed = await parseRequest(rowQueryContract, request, context, { + maxBodyBytes: TABLE_QUERY_MAX_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + const { params, body } = parsed.data + const { tableId } = params + + const accessResult = await checkAccess(tableId, authResult.userId, 'read') + if (!accessResult.ok) return accessError(accessResult, requestId, tableId) + const { table } = accessResult + + if (body.workspaceId !== table.workspaceId) { + logger.warn( + `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${body.workspaceId}, Actual: ${table.workspaceId}` + ) + return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) + } + + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(authResult.userId, body.workspaceId) + if (gateError) return gateError + + const schema = table.schema as TableSchema + const wire = rowWireTranslators(authResult.authType, schema) + const cursor = body.cursor ? decodeCursor(body.cursor) : undefined + + // Predicate/sort fields are column-NAME-keyed by construction (the caller + // authors names), so validate against the schema then translate names → + // storage ids unconditionally — unlike row data, this is not authType-dependent. + const idByName = buildIdByName(schema) + let predicate = body.predicate + if (predicate) { + validatePredicate(predicate, schema.columns) + predicate = predicateToStorage(predicate, schema) + } + let sortSpec = body.sort + if (sortSpec?.length) { + validateSortSpec(sortSpec, schema.columns) + sortSpec = sortSpecNamesToIds(sortSpec, idByName) + } + const sort: Sort | undefined = sortSpec?.length + ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) + : undefined + + // Cursor↔sort binding: keyset cursors are default-order only; an offset + // cursor must be replayed under the exact sort it was minted with. + if (cursor) assertCursorSortBinding(cursor, sort) + + const result = await queryRows( + table, + { + predicate, + sort, + limit: body.limit, + after: cursor?.after, + offset: cursor?.offset, + // Only the first page (no inbound cursor) pays for the total count. + includeTotal: !body.cursor, + // Executions are grid UI state; the v2 surface returns row data only + // and the byte budget deliberately measures just `data`. + withExecutions: false, + }, + requestId + ) + + return NextResponse.json({ + success: true, + data: { + rows: result.rows.map((r) => ({ + id: r.id, + data: wire.dataOut(r.data), + executions: r.executions, + position: r.position, + orderKey: r.orderKey ?? undefined, + createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), + updatedAt: r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), + })), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }) + } catch (error) { + if (isZodError(error)) return validationErrorResponse(error) + if (error instanceof TableQueryValidationError) { + return NextResponse.json( + { error: error.message, ...(error.code ? { code: error.code } : {}) }, + { status: 400 } + ) + } + logger.error(`[${requestId}] Error querying rows (v2):`, error) + return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts index 139427066cb..4b892a39340 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts @@ -6,14 +6,16 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table' -const { mockCheckAccess, mockFindRowMatches } = vi.hoisted(() => ({ +const { mockCheckAccess, mockFindRowMatches, mockTableFilterError } = vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockFindRowMatches: vi.fn(), + mockTableFilterError: vi.fn(), })) vi.mock('@/app/api/table/utils', async () => { const { NextResponse } = await import('next/server') return { + tableFilterError: mockTableFilterError, checkAccess: mockCheckAccess, accessError: (result: { status: number }) => NextResponse.json( @@ -67,6 +69,7 @@ describe('GET /api/table/[tableId]/rows/find', () => { authType: 'session', }) mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockTableFilterError.mockReturnValue(null) mockFindRowMatches.mockResolvedValue({ matches: [{ ordinal: 4, rowId: 'row_4', column: 'name' }], truncated: false, @@ -120,8 +123,23 @@ describe('GET /api/table/[tableId]/rows/find', () => { ) }) + it('short-circuits with the filter gate response before searching', async () => { + const { NextResponse } = await import('next/server') + mockTableFilterError.mockReturnValueOnce( + NextResponse.json({ error: 'Unknown filter column "nope"' }, { status: 400 }) + ) + const res = await callGet({ + workspaceId: 'workspace-1', + q: 'foo', + filter: JSON.stringify({ all: [{ field: 'nope', op: 'eq', value: 1 }] }), + }) + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter column/) + expect(mockFindRowMatches).not.toHaveBeenCalled() + }) + it('maps a TableQueryValidationError to 400', async () => { - const { TableQueryValidationError } = await import('@/lib/table/sql') + const { TableQueryValidationError } = await import('@/lib/table/errors') mockFindRowMatches.mockRejectedValueOnce(new TableQueryValidationError('Invalid field name')) const res = await callGet({ workspaceId: 'workspace-1', q: 'foo' }) expect(res.status).toBe(400) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.ts index 77a476ee145..f72c86633f6 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.ts @@ -5,10 +5,11 @@ import { isZodError, validationErrorResponse } from '@/lib/api/server/validation import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Sort } from '@/lib/table' +import type { Filter, Sort, SortSpec, TablePredicate } from '@/lib/table' +import { TableQueryValidationError } from '@/lib/table/errors' +import { toLegacyFilter, toLegacySort } from '@/lib/table/query-builder/converters' import { findRowMatches } from '@/lib/table/rows/service' -import { TableQueryValidationError } from '@/lib/table/sql' -import { accessError, checkAccess } from '@/app/api/table/utils' +import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' const logger = createLogger('TableRowsFindAPI') @@ -58,9 +59,23 @@ export const GET = withRouteHandler( return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + // Same gate as the bulk/async routes: the predicate branch rejects unknown + // storage keys (a typo'd field must 400, not return zero matches), the + // legacy branch keeps its compile check. + const filterError = tableFilterError( + validated.filter as Filter | TablePredicate | undefined, + table.schema.columns + ) + if (filterError) return filterError + const { matches, truncated } = await findRowMatches( table, - { q: validated.q, filter: validated.filter, sort: validated.sort }, + { + q: validated.q, + // Dual-grammar wire: findRowMatches compiles the legacy pair. + filter: toLegacyFilter(validated.filter as Filter | TablePredicate | undefined), + sort: toLegacySort(validated.sort as Sort | SortSpec | undefined), + }, requestId ) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts index 8c5fbab9276..b6162110b57 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -6,11 +6,20 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table' -const { mockCheckAccess, mockInsertRow, mockValidateRowData, mockQueryRows } = vi.hoisted(() => ({ +const { + mockCheckAccess, + mockInsertRow, + mockValidateRowData, + mockQueryRows, + mockUpdateRowsByFilter, + mockDeleteRowsByFilter, +} = vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockInsertRow: vi.fn(), mockValidateRowData: vi.fn(), mockQueryRows: vi.fn(), + mockUpdateRowsByFilter: vi.fn(), + mockDeleteRowsByFilter: vi.fn(), })) vi.mock('@/app/api/table/utils', async () => { @@ -31,9 +40,9 @@ vi.mock('@/lib/table', async () => { insertRow: mockInsertRow, batchInsertRows: vi.fn(), batchUpdateRows: vi.fn(), - deleteRowsByFilter: vi.fn(), + deleteRowsByFilter: mockDeleteRowsByFilter, deleteRowsByIds: vi.fn(), - updateRowsByFilter: vi.fn(), + updateRowsByFilter: mockUpdateRowsByFilter, validateBatchRows: vi.fn(), validateRowData: mockValidateRowData, validateRowSize: vi.fn(() => ({ valid: true })), @@ -48,7 +57,7 @@ vi.mock('@/lib/table/sql', () => ({ TableQueryValidationError: class TableQueryValidationError extends Error {}, })) -import { GET, POST } from '@/app/api/table/[tableId]/rows/route' +import { DELETE, GET, POST, PUT } from '@/app/api/table/[tableId]/rows/route' function buildTable(): TableDefinition { return { @@ -217,4 +226,147 @@ describe('GET /api/table/[tableId]/rows', () => { const body = await res.json() expect(body.data.rows[0].data).toEqual({ col_aaa: 'Ada', col_bbb: 36 }) }) + + /** + * The grid now speaks the v2 grammar on this route: a predicate-shaped filter + * takes the NATIVE predicate path into queryRows (not a downgrade), and an + * ordered sort spec compiles to the record the engine's sort builder takes. + */ + it('routes a predicate filter + spec sort natively for session callers', async () => { + authAs('session') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }), + sort: JSON.stringify([{ field: 'col_bbb', direction: 'desc' }]), + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) + expect(options.filter).toBeUndefined() + expect(options.sort).toEqual({ col_bbb: 'desc' }) + }) + + it('translates a name-keyed predicate for internal-JWT callers', async () => { + authAs('internal_jwt') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }), + sort: JSON.stringify([{ field: 'Age', direction: 'asc' }]), + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) + expect(options.sort).toEqual({ col_bbb: 'asc' }) + }) + + /** + * Storage validation runs post-translation, mirroring bulk PUT/DELETE: a + * typo'd field must 400, not compile to a clause that matches nothing and + * read back as a plausible empty page. + */ + it('400s a predicate naming an unknown column instead of returning an empty page', async () => { + authAs('session') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'col_nope', op: 'eq', value: 'Ada' }] }), + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.error).toMatch(/Unknown filter column "col_nope"/) + expect(mockQueryRows).not.toHaveBeenCalled() + }) +}) + +describe('PUT/DELETE /api/table/[tableId]/rows — predicate filters', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockUpdateRowsByFilter.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row_1'] }) + mockDeleteRowsByFilter.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row_1'] }) + }) + + function callPut(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return PUT(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) + } + + function callDelete(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return DELETE(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) + } + + /** + * Keying follows the caller (PR #6067 review): the grid authors ID-keyed + * predicates and the session wire is identity, so ids pass through — and a + * NAME under session auth is just an unknown storage key, rejected like any + * other typo rather than half-translated. + */ + it('PUT passes an id-keyed predicate through untouched under SESSION auth', async () => { + authAs('session') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }, + data: { col_aaa: 'Grace' }, + }) + + expect(res.status).toBe(200) + const args = mockUpdateRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) + }) + + it('PUT rejects an unknown storage key under SESSION auth with 400', async () => { + authAs('session') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, + data: { col_aaa: 'Grace' }, + }) + expect(res.status).toBe(400) + expect(mockUpdateRowsByFilter).not.toHaveBeenCalled() + }) + + it('PUT translates a name-keyed predicate for INTERNAL_JWT callers', async () => { + authAs('internal_jwt') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, + data: { Name: 'Grace' }, + }) + + expect(res.status).toBe(200) + const args = mockUpdateRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) + }) + + it('DELETE accepts the predicate and rejects an unknown column with 400', async () => { + authAs('internal_jwt') + const ok = await callDelete({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Age', op: 'gte', value: 30 }] }, + }) + expect(ok.status).toBe(200) + const args = mockDeleteRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_bbb: { $gte: 30 } }] }) + + const bad = await callDelete({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Nope', op: 'eq', value: 1 }] }, + }) + expect(bad.status).toBe(400) + expect((await bad.json()).error).toMatch(/Unknown filter column/) + }) }) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 5c73dd8dff1..748c7138b50 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -9,11 +9,11 @@ import { updateRowsByFilterBodySchema, } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' -import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' +import { isZodError, parseJsonBody, validationErrorResponse } from '@/lib/api/server/validation' import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Filter, RowData, Sort, TableRowsCursor, TableSchema } from '@/lib/table' +import type { Filter, RowData, Sort, SortSpec, TableRowsCursor, TableSchema } from '@/lib/table' import { batchInsertRows, batchUpdateRows, @@ -25,13 +25,56 @@ import { validateRowData, validateRowSize, } from '@/lib/table' +import { TableQueryValidationError } from '@/lib/table/errors' +import { isTablePredicate, predicateToFilter } from '@/lib/table/query-builder/converters' +import { + validatePredicateShape, + validateStoragePredicate, +} from '@/lib/table/query-builder/validate' import { queryRows } from '@/lib/table/rows/service' -import { TableQueryValidationError } from '@/lib/table/sql' -import { rowWireTranslators } from '@/app/api/table/row-wire' +import type { TablePredicate } from '@/lib/table/types' +import { type RowWireTranslators, rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' const logger = createLogger('TableRowsAPI') +/** Dual-grammar sort: an ordered spec (v2) or the legacy record, either keying. */ +function resolveWireSort( + sort: Sort | SortSpec | undefined, + wire: RowWireTranslators +): Sort | undefined { + if (!sort) return undefined + if (!Array.isArray(sort)) return wire.sortIn(sort) + const spec = wire.sortSpecIn(sort) + return spec.length > 0 ? Object.fromEntries(spec.map((s) => [s.field, s.direction])) : undefined +} + +/** + * Resolves a bulk-op filter to a storage-id-keyed legacy `Filter`. The v2 + * predicate tree is column-NAME-keyed by construction (the caller authors + * names), so it validates then translates names → ids unconditionally — unlike + * the legacy object form, whose keying follows the caller's wire dialect + * (`wire.filterIn`: ids from the UI, names from workflow tools). + */ +function resolveBulkFilter( + raw: TablePredicate | Filter, + schema: TableSchema, + wire: RowWireTranslators +): Filter { + if (isTablePredicate(raw)) { + // Shape first (keying-agnostic: hybrid nodes, leaf value rules), then let + // the wire translate — identity for the ID-keyed grid, names→ids for + // workflow tools — and validate the RESULT against storage keys. Post- + // translation, any unresolved field is a typo in the caller's own keying, + // and on a destructive path a typo must 400, not silently match nothing. + validatePredicateShape(raw) + const translated = wire.predicateIn(raw) + validateStoragePredicate(translated, schema.columns) + return predicateToFilter(translated) + } + return wire.filterIn(raw) +} + interface TableRowsRouteParams { params: Promise<{ tableId: string }> } @@ -259,8 +302,22 @@ export const GET = withRouteHandler( const result = await queryRows( table, { - filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined, - sort: validated.sort ? wire.sortIn(validated.sort) : undefined, + ...(validated.filter && isTablePredicate(validated.filter as Filter | TablePredicate) + ? { + predicate: (() => { + // Shape-check first: nothing upstream validates this branch, and + // an unchecked hybrid node would silently widen the result. + validatePredicateShape(validated.filter as TablePredicate) + const translated = wire.predicateIn(validated.filter as TablePredicate) + // Post-translation storage check, mirroring the bulk PUT/DELETE + // paths: a typo'd field must 400, not compile to a clause that + // matches nothing and read back as an empty page. + validateStoragePredicate(translated, (table.schema as TableSchema).columns) + return translated + })(), + } + : { filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined }), + sort: resolveWireSort(validated.sort as Sort | SortSpec | undefined, wire), limit: validated.limit, offset: validated.offset, after: validated.after, @@ -287,6 +344,7 @@ export const GET = withRouteHandler( totalCount: result.totalCount, limit: result.limit, offset: result.offset, + nextCursor: result.nextCursor, }, }) } catch (error) { @@ -316,12 +374,12 @@ export const PUT = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = updateRowsByFilterBodySchema.parse(body) @@ -351,7 +409,11 @@ export const PUT = withRouteHandler( const result = await updateRowsByFilter( table, { - filter: wire.filterIn(validated.filter as Filter), + filter: resolveBulkFilter( + validated.filter as TablePredicate | Filter, + table.schema as TableSchema, + wire + ), data: patchData, limit: validated.limit, actorUserId: authResult.userId, @@ -410,12 +472,12 @@ export const DELETE = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = deleteTableRowsBodySchema.parse(body) @@ -457,7 +519,11 @@ export const DELETE = withRouteHandler( const result = await deleteRowsByFilter( table, { - filter: wire.filterIn(validated.filter as Filter), + filter: resolveBulkFilter( + validated.filter as TablePredicate | Filter, + table.schema as TableSchema, + wire + ), limit: validated.limit, }, requestId @@ -504,12 +570,12 @@ export const PATCH = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = batchUpdateTableRowsBodySchema.parse(body) diff --git a/apps/sim/app/api/table/row-wire.ts b/apps/sim/app/api/table/row-wire.ts index d8458e56ec5..f4c0a85d988 100644 --- a/apps/sim/app/api/table/row-wire.ts +++ b/apps/sim/app/api/table/row-wire.ts @@ -1,13 +1,14 @@ import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' -import type { Filter, RowData, Sort, TableSchema } from '@/lib/table' +import type { Filter, RowData, Sort, SortSpec, TablePredicate, TableSchema } from '@/lib/table' import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, filterNamesToIds, rowDataNameToId, sortNamesToIds, + sortSpecNamesToIds, } from '@/lib/table/column-keys' -import { resolveFilterSelectValues } from '@/lib/table/select-values' +import { predicateToStorage, resolveFilterSelectValues } from '@/lib/table/select-values' export interface RowWireTranslators { /** Inbound row data: wire keys → storage column ids. */ @@ -18,6 +19,10 @@ export interface RowWireTranslators { filterIn: (filter: Filter) => Filter /** Inbound sort: wire field refs → storage column ids. */ sortIn: (sort: Sort) => Sort + /** Inbound v2 predicate: wire field refs → storage column ids. */ + predicateIn: (predicate: TablePredicate) => TablePredicate + /** Inbound v2 sort spec: wire field refs → storage column ids. */ + sortSpecIn: (sort: SortSpec) => SortSpec } /** @@ -33,15 +38,26 @@ export function rowWireTranslators( ): RowWireTranslators { if (authType !== AuthType.INTERNAL_JWT) { const identity = (value: T): T => value - return { dataIn: identity, dataOut: identity, filterIn: identity, sortIn: identity } + return { + dataIn: identity, + dataOut: identity, + filterIn: identity, + sortIn: identity, + predicateIn: identity, + sortSpecIn: identity, + } } const idByName = buildIdByName(schema) return { dataOut: namedRowMapper(schema.columns), dataIn: (data) => rowDataNameToId(data, idByName), - // Rekey field refs name → id, then resolve select operand names → ids. + // Rekey field refs name → id, then resolve select operand names → ids. Both + // grammars need that second step: a select cell stores an option id, so a + // filter written with the option NAME matches nothing without it. filterIn: (filter) => resolveFilterSelectValues(filterNamesToIds(filter, idByName), schema.columns), sortIn: (sort) => sortNamesToIds(sort, idByName), + predicateIn: (predicate) => predicateToStorage(predicate, schema), + sortSpecIn: (sort) => sortSpecNamesToIds(sort, idByName), } } diff --git a/apps/sim/app/api/table/utils.test.ts b/apps/sim/app/api/table/utils.test.ts index 2b23d45ffe9..99d0ce0c5a5 100644 --- a/apps/sim/app/api/table/utils.test.ts +++ b/apps/sim/app/api/table/utils.test.ts @@ -3,7 +3,8 @@ */ import { describe, expect, it } from 'vitest' import { TableRowLimitError } from '@/lib/table/billing' -import { rootErrorMessage, rowWriteErrorResponse } from '@/app/api/table/utils' +import type { ColumnDefinition } from '@/lib/table/types' +import { rootErrorMessage, rowWriteErrorResponse, tableFilterError } from '@/app/api/table/utils' /** Mimics drizzle's DrizzleQueryError: message is the failed SQL, real error on `cause`. */ function wrapLikeDrizzle(cause: Error): Error { @@ -53,3 +54,50 @@ describe('rowWriteErrorResponse', () => { expect(rowWriteErrorResponse(wrapLikeDrizzle(new Error('deadlock detected')))).toBeNull() }) }) + +/** + * The async destructive routes (delete-async, cancel-runs, columns/run) + * validate the WIRE filter here. The predicate branch must reject unknown + * storage keys the way the sync bulk routes do — the `toLegacyFilter` + * downgrade compiles a typo'd field into a clause that silently matches + * nothing, turning a scoped delete/run into a no-op. + */ +describe('tableFilterError', () => { + const columns: ColumnDefinition[] = [{ id: 'col_status', name: 'status', type: 'string' }] + + it('returns null for an absent filter and a valid id-keyed predicate', () => { + expect(tableFilterError(undefined, columns)).toBeNull() + expect( + tableFilterError({ all: [{ field: 'col_status', op: 'eq', value: 'x' }] }, columns) + ).toBeNull() + expect(tableFilterError({ all: [{ field: 'createdAt', op: 'isNotNull' }] }, columns)).toBeNull() + }) + + it('400s a predicate naming an unknown storage key', async () => { + const response = tableFilterError( + { all: [{ field: 'statuss', op: 'eq', value: 'x' }] }, + columns + ) + expect(response?.status).toBe(400) + const body = await response?.json() + expect(body.error).toMatch(/Unknown filter column "statuss"/) + }) + + it('400s a structurally invalid predicate (empty group, dual group keys)', () => { + expect(tableFilterError({ all: [] } as never, columns)?.status).toBe(400) + expect( + tableFilterError( + { + all: [{ field: 'col_status', op: 'eq', value: 'a' }], + any: [{ field: 'col_status', op: 'eq', value: 'b' }], + } as never, + columns + )?.status + ).toBe(400) + }) + + it('still validates the legacy grammar through buildFilterClause', () => { + expect(tableFilterError({ col_status: 'x' }, columns)).toBeNull() + expect(tableFilterError({ col_status: { $regex: 'x' } } as never, columns)?.status).toBe(400) + }) +}) diff --git a/apps/sim/app/api/table/utils.ts b/apps/sim/app/api/table/utils.ts index f3c62d56022..835866d829a 100644 --- a/apps/sim/app/api/table/utils.ts +++ b/apps/sim/app/api/table/utils.ts @@ -7,12 +7,35 @@ import { deleteTableColumnBodySchema, updateTableColumnBodySchema, } from '@/lib/api/contracts/tables' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import type { MultipartError } from '@/lib/core/utils/multipart' -import type { ColumnDefinition, Filter, TableDefinition } from '@/lib/table' +import type { ColumnDefinition, Filter, TableDefinition, TablePredicate } from '@/lib/table' import { buildFilterClause, getTableById, TableQueryValidationError } from '@/lib/table' import { USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableLockedError } from '@/lib/table/mutation-locks' +import { isTablePredicate } from '@/lib/table/query-builder/converters' +import { validateStoragePredicate } from '@/lib/table/query-builder/validate' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' +import { getWorkspaceOrganizationId } from '@/lib/workspaces/utils' + +/** + * Gate for the v2 tables HTTP API (`tables-v2-api` flag). Returns a 404 response + * when the flag is off for the caller — the surface behaves as if it doesn't + * exist — or `null` to proceed. Gated by userId + the workspace's org cohort. + * + * **Call this AFTER the authz check, never before.** Ahead of authz it does a + * primary-DB read keyed on a caller-supplied `workspaceId`, and the 404-vs-403 + * split tells an unauthorized caller whether that workspace's org is in the + * rollout cohort. + */ +export async function tablesV2GateError( + userId: string, + workspaceId: string +): Promise { + const orgId = await getWorkspaceOrganizationId(workspaceId) + if (await isFeatureEnabled('tables-v2-api', { userId, orgId })) return null + return NextResponse.json({ error: 'Not found' }, { status: 404 }) +} /** * Maps a {@link TableLockedError} thrown by the service layer to a 423 response @@ -33,17 +56,30 @@ export function tableLockErrorResponse(error: unknown): NextResponse | null { } /** - * Validates a `filter` against the table's column schema, returning a 400 response on a bad field - * (or `null` when the filter is valid or absent). Shared by the routes that accept a filter - * (`delete-async`, `columns/run`) so a bad field fails fast with a clear message. + * Validates a wire `filter` (either grammar) against the table's column schema, + * returning a 400 response on a bad field (or `null` when the filter is valid or + * absent). Shared by the routes that accept a filter (`delete-async`, + * `cancel-runs`, `columns/run`) so a bad field fails fast with a clear message. + * + * Pass the WIRE filter, not the `toLegacyFilter` downgrade: the downgrade + * compiles cleanly through `buildFilterClause` even when a predicate leaf names + * a column that doesn't exist, so validating only the downgraded form lets a + * typo'd field become a clause that silently matches nothing — a no-op where + * the sync bulk routes 400. */ export function tableFilterError( - filter: Filter | undefined, + filter: Filter | TablePredicate | undefined, columns: ColumnDefinition[] ): NextResponse | null { if (!filter) return null try { - buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, columns) + if (isTablePredicate(filter)) { + // These routes speak storage keys (session grid uses column ids; system + // columns keep their names) — same keying the sync bulk routes validate. + validateStoragePredicate(filter, columns) + } else { + buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, columns) + } return null } catch (error) { if (error instanceof TableQueryValidationError) { diff --git a/apps/sim/app/api/v1/middleware.ts b/apps/sim/app/api/v1/middleware.ts index 25357c19d18..bdc82a6613a 100644 --- a/apps/sim/app/api/v1/middleware.ts +++ b/apps/sim/app/api/v1/middleware.ts @@ -18,7 +18,12 @@ import { authenticateV1Request } from '@/app/api/v1/auth' const logger = createLogger('V1Middleware') const rateLimiter = new RateLimiter() -export type V1Endpoint = +/** + * Endpoint labels for public API auth/rate-limit telemetry. Version-neutral: the + * v1 and v2 public surfaces share the same `authenticateV1Request` + `api-endpoint` + * rate bucket, so the label is only a log/metric dimension, not a policy switch. + */ +export type ApiEndpoint = | 'logs' | 'logs-detail' | 'workflows' @@ -39,6 +44,8 @@ export type V1Endpoint = | 'knowledge-detail' | 'knowledge-search' | 'copilot-chat' + | 'v2-tables' + | 'v2-table-rows' export interface RateLimitResult { allowed: boolean @@ -65,7 +72,7 @@ export interface AuthorizedRequest { export async function checkRateLimit( request: NextRequest, - endpoint: V1Endpoint = 'logs' + endpoint: ApiEndpoint = 'logs' ): Promise { try { const auth = await authenticateV1Request(request) @@ -144,7 +151,7 @@ export async function checkRateLimit( */ export async function authenticateRequest( request: NextRequest, - endpoint: V1Endpoint + endpoint: ApiEndpoint ): Promise { const requestId = generateRequestId() const rateLimit = await checkRateLimit(request, endpoint) diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts index cc56cc61183..bef97b4d8f0 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts @@ -28,9 +28,9 @@ import { rowDataNameToId, sortNamesToIds, } from '@/lib/table/column-keys' +import { TableQueryValidationError } from '@/lib/table/errors' import { queryRows } from '@/lib/table/rows/service' import { resolveFilterSelectValues } from '@/lib/table/select-values' -import { TableQueryValidationError } from '@/lib/table/sql' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' import { checkRateLimit, @@ -190,6 +190,10 @@ export const GET = withRouteHandler(async (request: NextRequest, context: TableR totalCount: result.totalCount, limit: result.limit, offset: result.offset, + // Non-null when more rows exist; a page may return fewer than `limit` + // rows (byte budget) with more remaining, so page fullness is not a + // termination signal — external pagers should stop on null. + nextCursor: result.nextCursor, }, }) } catch (error) { diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts new file mode 100644 index 00000000000..37d8e774d8b --- /dev/null +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts @@ -0,0 +1,255 @@ +/** + * @vitest-environment node + * + * Public v2 query POST: typed predicate name→id translation, bounded-default vs + * explicit-unbounded limit, cursor validation, workspace scoping, + * and name-keyed row output. + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockCheckAccess, mockQueryRows, mockCheckRateLimit, mockCheckWorkspaceScope, mockGate } = + vi.hoisted(() => ({ + mockCheckAccess: vi.fn(), + mockQueryRows: vi.fn(), + mockCheckRateLimit: vi.fn(), + mockCheckWorkspaceScope: vi.fn(), + mockGate: vi.fn(), + })) + +vi.mock('@/app/api/v1/middleware', async () => { + const { NextResponse } = await import('next/server') + return { + checkRateLimit: mockCheckRateLimit, + checkWorkspaceScope: mockCheckWorkspaceScope, + createRateLimitResponse: (r: { error?: string }) => + NextResponse.json( + { error: r.error ?? 'Rate limit exceeded' }, + { status: r.error ? 401 : 429 } + ), + } +}) + +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => + NextResponse.json({ error: 'Access denied' }, { status: result.status }), + tablesV2GateError: mockGate, + } +}) + +vi.mock('@/lib/table', async () => { + const columnKeys = await import('@/lib/table/column-keys') + return { ...columnKeys } +}) + +vi.mock('@/lib/table/rows/service', () => ({ queryRows: mockQueryRows })) + +import { encodeCursor } from '@/lib/table/rows/cursor' +import { POST } from '@/app/api/v2/tables/[tableId]/query/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { id: 'col_name', name: 'name', type: 'string' }, + { id: 'col_wins', name: 'wins', type: 'number' }, + { id: 'col_status', name: 'status', type: 'string' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + } +} + +const EMPTY_RESULT = { + rows: [], + rowCount: 0, + totalCount: 0, + limit: 100, + offset: 0, + nextCursor: null, +} + +function callQuery(body: Record) { + const req = new NextRequest('http://localhost:3000/api/v2/tables/tbl_1/query', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) +} + +describe('POST /api/v2/tables/[tableId]/query', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckRateLimit.mockResolvedValue({ + allowed: true, + userId: 'user-1', + keyType: 'workspace', + workspaceId: 'workspace-1', + }) + mockCheckWorkspaceScope.mockResolvedValue(null) + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockQueryRows.mockResolvedValue(EMPTY_RESULT) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(404) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + mockCheckAccess.mockResolvedValue({ ok: false, status: 403 }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() + }) + + it('translates a name-keyed predicate to storage ids', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { + all: [ + { field: 'status', op: 'eq', value: 'active' }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }, + }) + expect(res.status).toBe(200) + expect(mockQueryRows.mock.calls[0][1].predicate).toEqual({ + all: [ + { field: 'col_status', op: 'eq', value: 'active' }, + { field: 'col_wins', op: 'gte', value: 10 }, + ], + }) + }) + + it('applies the bounded default limit when omitted', async () => { + await callQuery({ workspaceId: 'workspace-1' }) + expect(mockQueryRows.mock.calls[0][1].limit).toBe(100) + }) + + it('treats limit=0 as the explicit unbounded opt-in', async () => { + await callQuery({ workspaceId: 'workspace-1', limit: 0 }) + expect(mockQueryRows.mock.calls[0][1].limit).toBeUndefined() + }) + + it('rejects a limit above the max', async () => { + const res = await callQuery({ workspaceId: 'workspace-1', limit: 5000 }) + expect(res.status).toBe(400) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('rejects the removed regex ops at the contract boundary', async () => { + // `match`/`imatch` are no longer in FILTER_OPS — a catastrophic-backtracking + // pattern can pin a shared-pool connection, and nothing shipped depends on them. + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'name', op: 'match', value: '^jo' }] }, + }) + expect(res.status).toBe(400) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('rejects a predicate referencing an unknown column', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'nope', op: 'eq', value: 1 }] }, + }) + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter column/) + }) + + it('rejects a keyset cursor combined with a custom sort', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'r1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + const res = await callQuery({ + workspaceId: 'workspace-1', + sort: [{ field: 'wins', direction: 'desc' }], + cursor, + }) + expect(res.status).toBe(400) + expect((await res.json()).code).toBe('CURSOR_SORT_CONFLICT') + }) + + it('returns 400 INVALID_CURSOR for a malformed cursor', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + cursor: Buffer.from('42').toString('base64url'), + }) + expect(res.status).toBe(400) + expect((await res.json()).code).toBe('INVALID_CURSOR') + }) + + it('surfaces a workspace-scope 403 from the middleware', async () => { + const { NextResponse } = await import('next/server') + mockCheckWorkspaceScope.mockResolvedValue( + NextResponse.json({ error: 'not authorized' }, { status: 403 }) + ) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('rejects a workspace-id mismatch against the table', async () => { + const res = await callQuery({ workspaceId: 'other-ws' }) + expect(res.status).toBe(400) + expect((await res.json()).error).toBe('Invalid workspace ID') + }) + + it('returns name-keyed row data with no storage internals and a private cache header', async () => { + mockQueryRows.mockResolvedValue({ + rows: [ + { + id: 'r1', + data: { col_status: 'active', col_wins: 12 }, + position: 3, + orderKey: 'a5', + executions: {}, + createdAt: new Date('2024-02-02'), + updatedAt: new Date('2024-02-03'), + }, + ], + rowCount: 1, + totalCount: 1, + limit: 100, + offset: 0, + nextCursor: null, + }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.headers.get('Cache-Control')).toBe('private, no-store') + expect((await res.json()).data.rows[0]).toEqual({ + id: 'r1', + data: { status: 'active', wins: 12 }, + createdAt: '2024-02-02T00:00:00.000Z', + updatedAt: '2024-02-03T00:00:00.000Z', + }) + }) + + it('returns the rate-limit response when the limiter denies the request', async () => { + mockCheckRateLimit.mockResolvedValue({ allowed: false }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(429) + expect(mockCheckAccess).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts new file mode 100644 index 00000000000..4337c0b402c --- /dev/null +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts @@ -0,0 +1,151 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { TABLE_QUERY_MAX_BODY_BYTES } from '@/lib/api/contracts/tables' +import { V2_DEFAULT_ROW_LIMIT, v2QueryRowsContract } from '@/lib/api/contracts/v2/tables' +import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { Sort, TablePredicate, TableSchema } from '@/lib/table' +import { buildIdByName, sortSpecNamesToIds } from '@/lib/table' +import { namedRowMapper } from '@/lib/table/cell-format' +import { TableQueryValidationError } from '@/lib/table/errors' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' +import { queryRows } from '@/lib/table/rows/service' +import { predicateToStorage } from '@/lib/table/select-values' +import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' +import { + checkRateLimit, + checkWorkspaceScope, + createRateLimitResponse, +} from '@/app/api/v1/middleware' + +const logger = createLogger('V2TableQueryAPI') + +export const dynamic = 'force-dynamic' +export const revalidate = 0 + +/** Filters may carry user data; keep query responses out of shared caches. */ +const PRIVATE_NO_STORE = { 'Cache-Control': 'private, no-store' } as const + +interface QueryRouteParams { + params: Promise<{ tableId: string }> +} + +/** + * POST /api/v2/tables/[tableId]/query — public row query. Typed `predicate`/`sort` + * objects + opaque cursor pagination. Default page {@link V2_DEFAULT_ROW_LIMIT}; + * `limit=0` = unbounded (whole result or 400). + */ +export const POST = withRouteHandler(async (request: NextRequest, context: QueryRouteParams) => { + const requestId = generateRequestId() + + try { + const rateLimit = await checkRateLimit(request, 'v2-table-rows') + if (!rateLimit.allowed) return createRateLimitResponse(rateLimit) + + const userId = rateLimit.userId! + const parsed = await parseRequest(v2QueryRowsContract, request, context, { + maxBodyBytes: TABLE_QUERY_MAX_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + + const { tableId } = parsed.data.params + const { workspaceId, sort, cursor: cursorToken, limit } = parsed.data.body + + const scopeError = await checkWorkspaceScope(rateLimit, workspaceId) + if (scopeError) return scopeError + + const accessResult = await checkAccess(tableId, userId, 'read') + if (!accessResult.ok) return accessError(accessResult, requestId, tableId) + const { table } = accessResult + + if (workspaceId !== table.workspaceId) { + return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) + } + + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(userId, workspaceId) + if (gateError) return gateError + + const schema = table.schema as TableSchema + const cursor = cursorToken ? decodeCursor(cursorToken) : undefined + + const idByName = buildIdByName(schema) + // Fuses the id→name key remap with select-cell value formatting, so a select + // cell surfaces its option NAME rather than the stored option id. + const toNamedRow = namedRowMapper(schema.columns) + let predicate: TablePredicate | undefined = parsed.data.body.predicate + if (predicate) { + validatePredicate(predicate, schema.columns) + predicate = predicateToStorage(predicate, schema) + } + let sortSpec = sort + if (sortSpec?.length) { + validateSortSpec(sortSpec, schema.columns) + sortSpec = sortSpecNamesToIds(sortSpec, idByName) + } + const sortObj: Sort | undefined = sortSpec?.length + ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) + : undefined + + // A cursor is only valid for the query shape it was minted under: keyset + // cursors bind to the default order, offset cursors to their sort. Runs on + // the STORAGE-keyed sort so the fingerprint matches what queryRows stamped. + if (cursor) assertCursorSortBinding(cursor, sortObj) + + // Public default is a bounded page (unlike the internal surface's unbounded + // omit). `limit=0` is the explicit unbounded opt-in. + const effectiveLimit = + limit === undefined ? V2_DEFAULT_ROW_LIMIT : limit === 0 ? undefined : limit + + const result = await queryRows( + table, + { + predicate, + sort: sortObj, + limit: effectiveLimit, + after: cursor?.after, + offset: cursor?.offset, + includeTotal: !cursorToken, + withExecutions: false, + }, + requestId + ) + + return NextResponse.json( + { + success: true, + data: { + rows: result.rows.map((r) => ({ + id: r.id, + data: toNamedRow(r.data), + createdAt: + r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), + updatedAt: + r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), + })), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }, + { headers: PRIVATE_NO_STORE } + ) + } catch (error) { + const validationResponse = validationErrorResponseFromError(error) + if (validationResponse) return validationResponse + + if (error instanceof TableQueryValidationError) { + return NextResponse.json( + { error: error.message, ...(error.code ? { code: error.code } : {}) }, + { status: 400, headers: PRIVATE_NO_STORE } + ) + } + + logger.error(`[${requestId}] Error querying rows (v2 public):`, error) + return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/api/v2/tables/route.test.ts b/apps/sim/app/api/v2/tables/route.test.ts new file mode 100644 index 00000000000..2880355c8df --- /dev/null +++ b/apps/sim/app/api/v2/tables/route.test.ts @@ -0,0 +1,131 @@ +/** + * @vitest-environment node + * + * Public v2 tables list: auth/scope gating, typed summary output, private cache header. + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockListTables, mockCheckRateLimit, mockValidateWorkspaceAccess, mockGate } = vi.hoisted( + () => ({ + mockListTables: vi.fn(), + mockCheckRateLimit: vi.fn(), + mockValidateWorkspaceAccess: vi.fn(), + mockGate: vi.fn(), + }) +) + +vi.mock('@/app/api/v1/middleware', async () => { + const { NextResponse } = await import('next/server') + return { + checkRateLimit: mockCheckRateLimit, + validateWorkspaceAccess: mockValidateWorkspaceAccess, + createRateLimitResponse: (r: { error?: string }) => + NextResponse.json( + { error: r.error ?? 'Rate limit exceeded' }, + { status: r.error ? 401 : 429 } + ), + } +}) + +vi.mock('@/lib/table', async () => { + const actual = await import('@/lib/table/column-keys') + return { ...actual, listTables: mockListTables } +}) + +vi.mock('@/app/api/table/utils', () => ({ + normalizeColumn: (col: Record) => col, + tablesV2GateError: mockGate, +})) + +import { GET } from '@/app/api/v2/tables/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: 'A table', + schema: { columns: [{ id: 'col_name', name: 'name', type: 'string' }] }, + metadata: null, + rowCount: 5, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-02'), + } +} + +function callList(query: string) { + const req = new NextRequest(`http://localhost:3000/api/v2/tables?${query}`) + return GET(req) +} + +describe('GET /api/v2/tables', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckRateLimit.mockResolvedValue({ allowed: true, userId: 'user-1', keyType: 'workspace' }) + mockValidateWorkspaceAccess.mockResolvedValue(null) + mockListTables.mockResolvedValue([buildTable()]) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(404) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + const { NextResponse } = await import('next/server') + mockValidateWorkspaceAccess.mockResolvedValue( + NextResponse.json({ error: 'Access denied' }, { status: 403 }) + ) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() + }) + + it('returns a typed table summary with a private cache header', async () => { + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(200) + expect(res.headers.get('Cache-Control')).toBe('private, no-store') + const body = await res.json() + expect(body.data.totalCount).toBe(1) + expect(body.data.tables[0]).toMatchObject({ + id: 'tbl_1', + name: 'People', + description: 'A table', + rowCount: 5, + maxRows: 100, + createdAt: '2024-01-01T00:00:00.000Z', + }) + expect(body.data.tables[0].schema.columns[0].name).toBe('name') + }) + + it('400s when workspaceId is missing', async () => { + const res = await callList('') + expect(res.status).toBe(400) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('surfaces an access-denied response from the middleware', async () => { + const { NextResponse } = await import('next/server') + mockValidateWorkspaceAccess.mockResolvedValue( + NextResponse.json({ error: 'Access denied' }, { status: 403 }) + ) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(403) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('returns the rate-limit response when denied', async () => { + mockCheckRateLimit.mockResolvedValue({ allowed: false }) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(429) + }) +}) diff --git a/apps/sim/app/api/v2/tables/route.ts b/apps/sim/app/api/v2/tables/route.ts new file mode 100644 index 00000000000..be0c59c32b6 --- /dev/null +++ b/apps/sim/app/api/v2/tables/route.ts @@ -0,0 +1,78 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { v2ListTablesContract } from '@/lib/api/contracts/v2/tables' +import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { listTables, type TableSchema } from '@/lib/table' +import { normalizeColumn, tablesV2GateError } from '@/app/api/table/utils' +import { + checkRateLimit, + createRateLimitResponse, + validateWorkspaceAccess, +} from '@/app/api/v1/middleware' + +const logger = createLogger('V2TablesAPI') + +export const dynamic = 'force-dynamic' +export const revalidate = 0 + +/** Filters/ids can appear in query strings; keep list responses out of shared caches. */ +const PRIVATE_NO_STORE = { 'Cache-Control': 'private, no-store' } as const + +/** GET /api/v2/tables — list all tables in a workspace. */ +export const GET = withRouteHandler(async (request: NextRequest) => { + const requestId = generateRequestId() + + try { + const rateLimit = await checkRateLimit(request, 'v2-tables') + if (!rateLimit.allowed) return createRateLimitResponse(rateLimit) + + const userId = rateLimit.userId! + const parsed = await parseRequest(v2ListTablesContract, request, {}) + if (!parsed.success) return parsed.response + + const { workspaceId } = parsed.data.query + + const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) + if (accessError) return accessError + + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(userId, workspaceId) + if (gateError) return gateError + + const tables = await listTables(workspaceId) + + return NextResponse.json( + { + success: true, + data: { + tables: tables.map((t) => { + const schemaData = t.schema as TableSchema + return { + id: t.id, + name: t.name, + description: t.description, + schema: { columns: schemaData.columns.map(normalizeColumn) }, + rowCount: t.rowCount, + maxRows: t.maxRows, + createdAt: + t.createdAt instanceof Date ? t.createdAt.toISOString() : String(t.createdAt), + updatedAt: + t.updatedAt instanceof Date ? t.updatedAt.toISOString() : String(t.updatedAt), + } + }), + totalCount: tables.length, + }, + }, + { headers: PRIVATE_NO_STORE } + ) + } catch (error) { + const validationResponse = validationErrorResponseFromError(error) + if (validationResponse) return validationResponse + + logger.error(`[${requestId}] Error listing tables:`, error) + return NextResponse.json({ error: 'Failed to list tables' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx index a0af627d296..61643e48d4e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx @@ -4,7 +4,7 @@ import { memo, useCallback, useMemo, useRef, useState } from 'react' import { Button, ChipDropdown, ChipInput } from '@sim/emcn' import { Plus, X } from '@sim/emcn/icons' import { generateShortId } from '@sim/utils/id' -import type { ColumnDefinition, Filter, FilterRule } from '@/lib/table' +import type { ColumnDefinition, FilterRule, TablePredicate } from '@/lib/table' import { getColumnId } from '@/lib/table/column-keys' import { COMPARISON_OPERATORS, @@ -12,7 +12,10 @@ import { SINGLE_SELECT_FILTER_OPERATORS, VALUELESS_OPERATORS, } from '@/lib/table/query-builder/constants' -import { filterRulesToFilter, filterToRules } from '@/lib/table/query-builder/converters' +import { + filterRulesToPredicate, + predicateToFilterRules, +} from '@/lib/table/query-builder/converters' const SINGLE_SELECT_COMPARISON_OPERATORS = COMPARISON_OPERATORS.filter((o) => SINGLE_SELECT_FILTER_OPERATORS.has(o.value) @@ -27,14 +30,14 @@ function selectFilterOperators(column: ColumnDefinition | undefined): Set void + filter: TablePredicate | null + onApply: (filter: TablePredicate | null) => void onClose: () => void } export function TableFilter({ columns, filter, onApply, onClose }: TableFilterProps) { const [rules, setRules] = useState(() => { - const fromFilter = filterToRules(filter) + const fromFilter = predicateToFilterRules(filter) return fromFilter.length > 0 ? fromFilter : [createRule(columns)] }) @@ -112,7 +115,7 @@ export function TableFilter({ columns, filter, onApply, onClose }: TableFilterPr const validRules = rulesRef.current.filter( (r) => r.column && (r.value || VALUELESS_OPERATORS.has(r.operator)) ) - onApply(filterRulesToFilter(validRules, columns)) + onApply(filterRulesToPredicate(validRules, columns)) }, [columns, onApply]) const handleClear = useCallback(() => { diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index ed93cdaca87..d7bf866e721 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -13,9 +13,9 @@ import type { RunLimit, RunMode, TableFindMatch } from '@/lib/api/contracts/tabl import { captureEvent } from '@/lib/posthog/client' import type { ColumnDefinition, - Filter, TableLocks, TableMetadata, + TablePredicate, TableRow as TableRowType, WorkflowGroup, } from '@/lib/table' @@ -121,7 +121,7 @@ export interface SelectionSnapshot { allRows: boolean rowCount: number /** Active filter when `allRows` is set — lets a filtered "select all" run only matching rows. */ - filter?: Filter + filter?: TablePredicate /** Deselected rows when `allRows` is set — runs/stops skip them. */ excludeRowIds?: string[] } | null @@ -199,7 +199,7 @@ interface TableGridProps { runMode: RunMode, rowIds?: string[], limit?: RunLimit, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => void /** Fire every runnable column on a single row (per-row gutter Play). */ @@ -209,14 +209,14 @@ interface TableGridProps { onRunRows: ( rowIds: string[] | undefined, runMode: RunMode, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => void /** Stop running workflows on `rowIds`. Per-row gutter Stop also funnels through here. */ onStopRows: (rowIds: string[]) => void /** Select-all Stop: table-wide, or scoped to the active filter when one is set. * `excludeRowIds` (deselected rows) keep running. */ - onStopAllRows: (filter?: Filter, excludeRowIds?: string[]) => void + onStopAllRows: (filter?: TablePredicate, excludeRowIds?: string[]) => void /** Single-row stop for the per-row gutter button. */ onStopRow: (rowId: string) => void /** diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts index 8026fbfa630..0fed31ca065 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts @@ -218,7 +218,7 @@ describe('useTable – ensureAllRowsLoaded', () => { }) it('encodes queryOptions.filter into the queryKey passed to getQueryData', async () => { - const filter = { column: 'name', operator: 'eq', value: 'Alice' } as never + const filter = { all: [{ field: 'name', op: 'eq' as const, value: 'Alice' }] } mockGetQueryData.mockReturnValue({ pages: makePages([3], 3) }) const { ensureAllRowsLoaded } = makeHook({ filter, sort: null }) await ensureAllRowsLoaded() diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts index 9fcf4dd7db5..be473ae71d3 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts @@ -4,13 +4,13 @@ import { useCallback, useMemo } from 'react' import { useQueryClient } from '@tanstack/react-query' import type { ColumnDefinition, - Filter, TableDefinition, + TablePredicate, TableRow, WorkflowGroup, } from '@/lib/table' import { TABLE_LIMITS } from '@/lib/table/constants' -import { pruneFilterForColumns } from '@/lib/table/query-builder/converters' +import { prunePredicateForColumns } from '@/lib/table/query-builder/converters' import type { FlattenOutputsBlockInput } from '@/lib/workflows/blocks/flatten-outputs' import { getBlock } from '@/blocks' import { @@ -51,7 +51,7 @@ export interface UseTableReturn { * select-all run/stop/delete — must scope with THIS, not the raw filter, or * the action targets a predicate the grid isn't displaying. */ - filter: Filter | null + filter: TablePredicate | null isLoadingRows: boolean refetchRows: () => void /** @@ -98,7 +98,7 @@ export function useTable({ workspaceId, tableId, queryOptions }: UseTableParams) // here, above every consumer of the rows query key, so the paged helpers below // can't rebuild the key from the unpruned filter and drift. const filter = useMemo( - () => pruneFilterForColumns(queryOptions.filter ?? null, tableData?.schema?.columns ?? []), + () => prunePredicateForColumns(queryOptions.filter ?? null, tableData?.schema?.columns ?? []), [queryOptions.filter, tableData?.schema?.columns] ) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 0540b5346b5..146c02c103a 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -12,10 +12,10 @@ import type { RunLimit, RunMode, TableViewWire } from '@/lib/api/contracts/table import { captureEvent } from '@/lib/posthog/client' import type { ColumnDefinition, - Filter, - Sort, SortDirection, + SortSpec, TableMetadata, + TablePredicate, TableRow as TableRowType, TableViewConfig, WorkflowGroup, @@ -255,7 +255,7 @@ export function Table({ selectionStats: { hasIncompleteOrFailed: false, hasCompleted: false, hasInFlight: false }, singleWorkflowCell: null, }) - const [filter, setFilter] = useState(null) + const [filter, setFilter] = useState(null) const [filterOpen, setFilterOpen] = useState(false) /** Hidden **column ids**. Lives here (not in the grid) because the filter * panel's Columns section edits it and the active view persists it. */ @@ -271,9 +271,9 @@ export function Table({ const hiddenColumnsRef = useRef(hiddenColumns) hiddenColumnsRef.current = hiddenColumns - /** Resolved single-column sort, or `null` when no column is active. */ - const sortQuery = useMemo( - () => (sortColumn ? { [sortColumn]: sortDirection } : null), + /** Resolved single-column sort as an ordered spec, or `null` when none is active. */ + const sortQuery = useMemo( + () => (sortColumn ? [{ field: sortColumn, direction: sortDirection }] : null), [sortColumn, sortDirection] ) @@ -426,10 +426,10 @@ export function Table({ if (!keep?.filter) setFilter(config?.filter ?? null) if (!keep?.hiddenColumns) setHiddenColumns(config?.hiddenColumns ?? []) if (keep?.sort) return - const sortEntry = config?.sort ? Object.entries(config.sort)[0] : undefined + const sortEntry = config?.sort?.[0] setTableParams({ - sort: sortEntry ? sortEntry[0] : null, - dir: sortEntry ? (sortEntry[1] as SortDirection) : null, + sort: sortEntry ? sortEntry.field : null, + dir: sortEntry ? (sortEntry.direction as SortDirection) : null, }) }, [setTableParams] @@ -651,7 +651,7 @@ export function Table({ const currentViewConfig = useMemo( () => ({ ...(activeView?.config ?? tableData?.metadata), - filter: effectiveFilter, + filter: effectiveFilter ?? null, sort: sortQuery, hiddenColumns: effectiveHiddenColumns, }), @@ -837,7 +837,7 @@ export function Table({ (args: { groupIds: string[] rowIds?: string[] - filter?: Filter + filter?: TablePredicate excludeRowIds?: string[] runMode: RunMode limit?: RunLimit @@ -876,7 +876,7 @@ export function Table({ runMode: RunMode, rowIds?: string[], limit?: RunLimit, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => { runScope({ @@ -893,7 +893,12 @@ export function Table({ ) const onRunRows = useCallback( - (rowIds: string[] | undefined, runMode: RunMode, filter?: Filter, excludeRowIds?: string[]) => { + ( + rowIds: string[] | undefined, + runMode: RunMode, + filter?: TablePredicate, + excludeRowIds?: string[] + ) => { runScope({ groupIds: tableWorkflowGroups.map((g) => g.id), rowIds, @@ -961,7 +966,7 @@ export function Table({ /** Select-all Stop — filter-scoped when a filter is active; deselected rows keep running. */ const onStopAllRows = useCallback( - (filter?: Filter, excludeRowIds?: string[]) => { + (filter?: TablePredicate, excludeRowIds?: string[]) => { // `sort` scopes the optimistic flip to the active view's cache (filtered stops // only cancel matching rows server-side). cancelRunsMutate({ scope: 'all', filter, sort: queryOptions.sort, excludeRowIds }) @@ -1061,7 +1066,7 @@ export function Table({ [columnOptions, sortColumn, sortDirection, setTableParams] ) - const handleFilterApply = (next: Filter | null) => { + const handleFilterApply = (next: TablePredicate | null) => { setFilter(next) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts index d7c7553c2ce..34618913acf 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts @@ -1,4 +1,4 @@ -import type { Filter, Sort, TableRow } from '@/lib/table' +import type { SortSpec, TablePredicate, TableRow } from '@/lib/table' /** * Reason the inline editor completed, used to determine navigation after save @@ -9,8 +9,8 @@ export type SaveReason = 'enter' | 'tab' | 'shift-tab' | 'blur' * Query options for filtering and sorting table data */ export interface QueryOptions { - filter: Filter | null - sort: Sort | null + filter: TablePredicate | null + sort: SortSpec | null } /** diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx index d11ec546900..9427c7a8ca5 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx @@ -5,7 +5,7 @@ import { Button, type ComboboxOption } from '@sim/emcn' import { generateId } from '@sim/utils/id' import { Plus } from 'lucide-react' import { useTableColumns } from '@/lib/table/hooks' -import { SORT_DIRECTIONS, type SortRule } from '@/lib/table/query-builder/constants' +import { SORT_DIRECTION_OPTIONS, type SortRule } from '@/lib/table/query-builder/constants' import { useCanonicalSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-canonical-sub-block-value' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' import { SortRuleRow } from './components/sort-rule-row' @@ -47,7 +47,7 @@ export function SortBuilder({ }, [propColumns, dynamicColumns]) const directionOptions = useMemo( - () => SORT_DIRECTIONS.map((dir) => ({ value: dir.value, label: dir.label })), + () => SORT_DIRECTION_OPTIONS.map((dir) => ({ value: dir.value, label: dir.label })), [] ) diff --git a/apps/sim/blocks/blocks/table.ts b/apps/sim/blocks/blocks/table.ts index 281bcd077e7..3e2a99f88d7 100644 --- a/apps/sim/blocks/blocks/table.ts +++ b/apps/sim/blocks/blocks/table.ts @@ -183,7 +183,7 @@ export const TableBlock: BlockConfig = { description: 'User-defined data tables', longDescription: 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows.', - docsLink: 'https://docs.simstudio.ai/tools/table', + docsLink: 'https://docs.sim.ai/integrations/table', category: 'blocks', bgColor: '#10B981', icon: TableIcon, diff --git a/apps/sim/blocks/blocks/table_v2.test.ts b/apps/sim/blocks/blocks/table_v2.test.ts new file mode 100644 index 00000000000..1288d912e82 --- /dev/null +++ b/apps/sim/blocks/blocks/table_v2.test.ts @@ -0,0 +1,119 @@ +/** + * @vitest-environment node + * + * Param-transformer behavior for the v2 Table block: filter/order resolution + * across builder vs editor modes, the required query limit, cursor artifact + * handling, and fail-fast limit parsing. + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/triggers', () => ({ + getTrigger: vi.fn(() => ({ subBlocks: [] })), +})) + +import { TableV2Block } from '@/blocks/blocks/table_v2' + +function params(input: Record): Record { + return TableV2Block.tools.config?.params?.(input as never) as Record +} + +describe('table_v2 query_rows transformer', () => { + it('keeps limit optional (byte-budget page) but fails fast on a non-numeric one', () => { + const out = params({ operation: 'query_rows', tableId: 't' }) + expect(out.limit).toBeUndefined() + expect(() => params({ operation: 'query_rows', tableId: 't', limit: 'abc' })).toThrow( + /Invalid Limit/ + ) + }) + + it('treats interpolated "null"/"undefined" cursor artifacts as absent', () => { + const out = params({ operation: 'query_rows', tableId: 't', limit: '10', cursor: 'null' }) + expect(out.cursor).toBeUndefined() + const out2 = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + cursor: 'undefined', + }) + expect(out2.cursor).toBeUndefined() + const out3 = params({ operation: 'query_rows', tableId: 't', limit: '10', cursor: 'tok' }) + expect(out3.cursor).toBe('tok') + }) + + it('compiles builder rules (basic canonical value) to a predicate + sort spec', () => { + const out = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + // filterInput/sortInput carry the ACTIVE canonical mode's value; the visual + // builders (basic) emit rule arrays. + filterInput: [ + { id: '1', logicalOperator: 'and', column: 'wins', operator: 'gte', value: '10' }, + ], + sortInput: [{ id: '1', column: 'wins', direction: 'desc' }], + }) + expect(out.filter).toEqual({ all: [{ field: 'wins', op: 'gte', value: 10 }] }) + expect(out.order).toEqual([{ field: 'wins', direction: 'desc' }]) + }) + + it('parses the editor JSON string (advanced canonical value) into a predicate', () => { + const out = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + filterInput: '{"all":[{"field":"name","op":"eq","value":"test"}]}', + }) + expect(out.filter).toEqual({ all: [{ field: 'name', op: 'eq', value: 'test' }] }) + }) +}) + +describe('table_v2 bulk transformers', () => { + const editorFilter = '{"all":[{"field":"name","op":"eq","value":"x"}]}' + + it('fails fast on a non-numeric bulk limit instead of widening to every match', () => { + expect(() => + params({ + operation: 'delete_rows_by_filter', + tableId: 't', + filterInput: editorFilter, + limit: 'abc', + }) + ).toThrow(/Invalid Limit/) + }) + + it('keeps the bulk limit optional', () => { + const out = params({ + operation: 'delete_rows_by_filter', + tableId: 't', + filterInput: editorFilter, + }) + expect(out.limit).toBeUndefined() + expect(out.filter).toEqual({ all: [{ field: 'name', op: 'eq', value: 'x' }] }) + }) +}) + +/** + * The Filter/Order fields are canonical pairs: a builder array in Builder mode, + * a JSON string in Editor mode. Clearing the editor field is the ordinary way to + * say "no filter" — it must not surface as a JSON parse error at run time. + */ +describe('table_v2 blank and malformed editor inputs', () => { + const base = { operation: 'query_rows', tableId: 'tbl_1', limit: '10' } + + it('treats a blank / whitespace filter or order as absent', () => { + for (const value of ['', ' ', '\n']) { + expect(params({ ...base, filterInput: value }).filter).toBeUndefined() + expect(params({ ...base, sortInput: value }).order).toBeUndefined() + } + }) + + it('treats an empty builder array as absent', () => { + expect(params({ ...base, filterInput: [] }).filter).toBeUndefined() + expect(params({ ...base, sortInput: [] }).order).toBeUndefined() + }) + + it('still reports genuinely malformed JSON', () => { + expect(() => params({ ...base, filterInput: '{not json}' })).toThrow(/Invalid JSON in Filter/) + expect(() => params({ ...base, sortInput: '{not json}' })).toThrow(/Invalid JSON in Sort/) + }) +}) diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts new file mode 100644 index 00000000000..74dffc275a7 --- /dev/null +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -0,0 +1,584 @@ +import { toError } from '@sim/utils/errors' +import { TableIcon } from '@/components/icons' +import { TABLE_LIMITS } from '@/lib/table/constants' +import { filterRulesToPredicate, sortRulesToSortSpec } from '@/lib/table/query-builder/converters' +import type { FilterRule, SortRule, SortSpec, TablePredicate } from '@/lib/table/types' +import type { BlockConfig } from '@/blocks/types' +import type { TableQueryV2Response } from '@/tools/table/types' +import { getTrigger } from '@/triggers' + +/** + * Table v2 — same operations as the v1 Table block, but the filter grammar is a + * typed predicate tree (`{all:[{field:'wins',op:'gte',value:10}]}`), validated + * server-side. Pagination is an opaque cursor (no offset). The filter compiler, + * upsert conflict probe, and unique checks share one case-sensitive containment + * leaf, so upserts can't wedge on a case-mismatched unique value the way they + * could under v1. + */ + +function parseJSON(value: string | unknown, fieldName: string): unknown { + if (typeof value !== 'string') return value + // A blank editor field means "no filter/order", not malformed JSON. Without + // this, clearing the field throws `Unexpected end of JSON input` at run time. + if (value.trim() === '') return undefined + try { + return JSON.parse(value) + } catch (error) { + const errorMsg = toError(error).message + const unquotedValueMatch = value.match( + /:\s*([a-zA-Z][a-zA-Z0-9_\s]*[a-zA-Z0-9]|[a-zA-Z])\s*[,}]/ + ) + let hint = + 'Make sure all property names are in double quotes (e.g., {"name": "value"} not {name: "value"}).' + if (unquotedValueMatch) { + hint = + 'It looks like a string value is not quoted. When using block references in JSON, wrap them in double quotes: {"field": ""} not {"field": }.' + } + throw new Error(`Invalid JSON in ${fieldName}: ${errorMsg}. ${hint}`) + } +} + +interface TableBlockParams { + operation: string + tableId?: string + rowId?: string + data?: string | unknown + rows?: string | unknown + filterInput?: unknown + sortInput?: unknown + limit?: string + cursor?: string + conflictColumn?: string +} + +/** + * Resolves the effective predicate from the `filterInput` canonical param, which + * carries whichever mode's value the ⟷ toggle has active: a FilterRule[] (visual + * builder / basic) or a predicate-JSON string (editor / advanced, agent-authored). + * An array is builder rules; a string is raw JSON. Both compile to the same + * `TablePredicate` the route consumes. + */ +function resolveFilter(params: TableBlockParams): TablePredicate | undefined { + const raw = params.filterInput + if (Array.isArray(raw)) { + return raw.length > 0 ? (filterRulesToPredicate(raw as FilterRule[]) ?? undefined) : undefined + } + const parsed = parseJSON(raw, 'Filter') + return (parsed as TablePredicate | undefined) || undefined +} + +function resolveOrder(params: TableBlockParams): SortSpec | undefined { + const raw = params.sortInput + if (Array.isArray(raw)) { + return raw.length > 0 ? (sortRulesToSortSpec(raw as SortRule[]) ?? undefined) : undefined + } + const parsed = parseJSON(raw, 'Sort') + return (parsed as SortSpec | undefined) || undefined +} + +interface ParsedParams { + tableId?: string + rowId?: string + data?: unknown + rows?: unknown + filter?: TablePredicate + order?: SortSpec + limit?: number + cursor?: string + conflictTarget?: string +} + +/** + * Fail-fast limit parsing for bulk ops: an unparseable limit must never + * silently widen the operation to every matching row. + */ +function parseOptionalLimit(raw: string | undefined): number | undefined { + if (!raw) return undefined + const value = Number.parseInt(raw, 10) + if (Number.isNaN(value)) { + throw new Error(`Invalid Limit "${raw}" — expected a number.`) + } + return value +} + +/** + * `` interpolates the terminating page's `null` as the + * literal string "null" — treat those artifacts as "no cursor". + */ +function resolveCursor(raw: string | undefined): string | undefined { + if (!raw || raw === 'null' || raw === 'undefined') return undefined + return raw +} + +const paramTransformers: Record ParsedParams> = { + insert_row: (params) => ({ + tableId: params.tableId, + data: parseJSON(params.data, 'Row Data'), + }), + + upsert_row: (params) => ({ + tableId: params.tableId, + data: parseJSON(params.data, 'Row Data'), + conflictTarget: params.conflictColumn || undefined, + }), + + batch_insert_rows: (params) => ({ + tableId: params.tableId, + rows: parseJSON(params.rows, 'Rows Data'), + }), + + update_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + data: parseJSON(params.data, 'Row Data'), + }), + + // Bulk write-by-filter takes a predicate object, validated server-side. + update_rows_by_filter: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + data: parseJSON(params.data, 'Row Data'), + limit: parseOptionalLimit(params.limit), + }), + + delete_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + }), + + delete_rows_by_filter: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + limit: parseOptionalLimit(params.limit), + }), + + get_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + }), + + get_schema: (params) => ({ + tableId: params.tableId, + }), + + query_rows: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + order: resolveOrder(params), + // Omitted limit returns the entire result (server fails fast over 5MB); + // with a limit, nextCursor signals when more rows remain. + limit: parseOptionalLimit(params.limit), + cursor: resolveCursor(params.cursor), + }), +} + +export const TableV2Block: BlockConfig = { + type: 'table_v2', + name: 'Table', + description: 'User-defined data tables', + longDescription: + 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows. ' + + 'Query Rows filters with a predicate tree — `{"all":[{"field":"wins","op":"gte","value":10}]}` ' + + '(`all` = AND, `any` = OR; groups nest). Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, ' + + 'nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. Order is a sort ' + + 'spec `[{"field":"wins","direction":"desc"}]`. Query Rows returns every matching row when Limit is omitted ' + + '(fails if the result exceeds 5MB — add a filter or a Limit). With a Limit, responses page: a non-null ' + + 'nextCursor means more rows exist — pass it back as the cursor.', + bestPractices: ` +- To fetch specific rows, use Query Rows with a predicate filter (e.g. {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}) — do NOT read every row and filter downstream with a Condition block. +- Use "Get Row by ID" only when you have the row's id; otherwise filter with a predicate. +- A group is {"all":[...]} (AND) or {"any":[...]} (OR); nest groups as members for mixed logic. +- Example: players who won ≥10 and are active → {"all":[{"field":"wins","op":"gte","value":10},{"field":"status","op":"eq","value":"active"}]}. +- like/ilike use * as the wildcard (e.g. {"field":"name","op":"ilike","value":"*jo*"}). +- Omit Limit to get the entire matching result in one response — the query fails with a clear error if it exceeds 5MB (narrow with a filter or set a Limit). +- With a Limit, pages can end at the Limit or the 5MB byte budget, whichever comes first — pass nextCursor back as the cursor and loop until it is null; never infer completion from page size. +- Columns are scalar (string/number/boolean/date) or opaque json — there are no array columns; for substring use ilike with *x*.`, + docsLink: 'https://docs.sim.ai/integrations/table', + category: 'blocks', + // Unreleased: hidden from every discovery surface until revealed via the hosted + // `block-visibility` AppConfig document or the `PREVIEW_BLOCKS` env allowlist. + // Placed instances always execute. At GA: drop this, add the BlockMeta + docs, + // and mark v1 `table` superseded. + preview: true, + bgColor: '#10B981', + icon: TableIcon, + subBlocks: [ + { + id: 'operation', + title: 'Operation', + type: 'dropdown', + options: [ + { label: 'Query Rows', id: 'query_rows' }, + { label: 'Insert Row', id: 'insert_row' }, + { label: 'Upsert Row', id: 'upsert_row' }, + { label: 'Batch Insert Rows', id: 'batch_insert_rows' }, + { label: 'Update Rows by Filter', id: 'update_rows_by_filter' }, + { label: 'Delete Rows by Filter', id: 'delete_rows_by_filter' }, + { label: 'Update Row by ID', id: 'update_row' }, + { label: 'Delete Row by ID', id: 'delete_row' }, + { label: 'Get Row by ID', id: 'get_row' }, + { label: 'Get Schema', id: 'get_schema' }, + ], + value: () => 'query_rows', + }, + + { + id: 'tableSelector', + title: 'Table', + type: 'table-selector', + canonicalParamId: 'tableId', + mode: 'basic', + placeholder: 'Select a table', + required: true, + }, + { + id: 'manualTableId', + title: 'Table ID', + type: 'short-input', + canonicalParamId: 'tableId', + mode: 'advanced', + placeholder: 'Enter table ID', + required: true, + }, + + { + id: 'rowId', + title: 'Row ID', + type: 'short-input', + placeholder: 'row_xxxxx', + dependsOn: ['tableId'], + condition: { field: 'operation', value: ['get_row', 'update_row', 'delete_row'] }, + required: true, + }, + + { + id: 'data', + title: 'Row Data (JSON)', + type: 'code', + placeholder: '{"column_name": "value"}', + condition: { + field: 'operation', + value: ['insert_row', 'upsert_row', 'update_row', 'update_rows_by_filter'], + }, + required: true, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate row data as a JSON object matching the table's column schema. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON object with field values based on the table's columns. No explanations or markdown. + +IMPORTANT: Reference the table schema visible in the table selector to know which columns exist and their types. + +### EXAMPLES + +Table with columns: email (string), name (string), age (number) +"user with email john@example.com and age 25" +→ {"email": "john@example.com", "name": "John", "age": 25} + +Return ONLY the data JSON:`, + generationType: 'table-schema', + }, + }, + + { + id: 'conflictColumnSelector', + title: 'Conflict Column', + type: 'column-selector', + canonicalParamId: 'conflictColumn', + mode: 'basic', + selectorKey: 'table.columns', + placeholder: 'Select a unique column', + dependsOn: ['tableSelector'], + condition: { field: 'operation', value: 'upsert_row' }, + }, + { + id: 'manualConflictColumn', + title: 'Conflict Column', + type: 'short-input', + canonicalParamId: 'conflictColumn', + mode: 'advanced', + placeholder: 'Enter the column id', + dependsOn: ['tableId'], + condition: { field: 'operation', value: 'upsert_row' }, + }, + + { + id: 'rows', + title: 'Rows Data (Array of JSON)', + type: 'code', + placeholder: '[{"col1": "val1"}, {"col1": "val2"}]', + condition: { field: 'operation', value: 'batch_insert_rows' }, + required: true, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate an array of row data objects matching the table's column schema. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON array of objects. Each object represents one row. No explanations or markdown. +Maximum ${TABLE_LIMITS.MAX_BATCH_INSERT_SIZE} rows per batch. + +Return ONLY the rows array:`, + generationType: 'table-schema', + }, + }, + + // Filter — canonical Builder⟷Editor toggle (the ⟷ swap icon on the field), + // query + bulk update/delete. Basic = visual builder (human-only); Advanced + // = predicate JSON the agent authors directly. Both members are deliberately + // NOT `required` (the canonical group must share required status): the service + // fails closed when no filter resolves for a bulk op ("Filter is required..."). + { + id: 'filterBuilder', + title: 'Filter', + type: 'filter-builder', + canonicalParamId: 'filterInput', + mode: 'basic', + paramVisibility: 'user-only', + condition: { + field: 'operation', + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + }, + }, + { + id: 'filter', + title: 'Filter', + type: 'code', + canonicalParamId: 'filterInput', + mode: 'advanced', + placeholder: '{"all":[{"field":"wins","op":"gte","value":10}]}', + condition: { + field: 'operation', + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + }, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate a predicate filter object (JSON) for selecting rows. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY the JSON object. No explanations, surrounding quotes, or markdown. + +A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {"field","op","value"} or nested groups. + +### OPERATORS +eq, ne, gt, gte, lt, lte, in, nin (in/nin take an array value), like, ilike (use * as the wildcard), nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. + +### EXAMPLES +"status is active" → {"all":[{"field":"status","op":"eq","value":"active"}]} +"wins at least 10 and active" → {"all":[{"field":"wins","op":"gte","value":10},{"field":"active","op":"eq","value":true}]} +"status active or pending" → {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]} +"name contains jo (any case)" → {"all":[{"field":"name","op":"ilike","value":"*jo*"}]} + +Return ONLY the JSON object:`, + generationType: 'table-schema', + }, + }, + + // Order — canonical Builder⟷Editor toggle, query only. Basic = visual sort + // builder (human-only); Advanced = sort-spec JSON the agent authors. + { + id: 'sortBuilder', + title: 'Order', + type: 'sort-builder', + canonicalParamId: 'sortInput', + mode: 'basic', + paramVisibility: 'user-only', + condition: { field: 'operation', value: 'query_rows' }, + }, + { + id: 'order', + title: 'Order', + type: 'short-input', + canonicalParamId: 'sortInput', + mode: 'advanced', + placeholder: '[{"field":"wins","direction":"desc"}]', + condition: { field: 'operation', value: 'query_rows' }, + }, + { + id: 'limit', + title: 'Limit', + type: 'short-input', + placeholder: 'Leave empty for all rows (fails over 5MB)', + condition: { + field: 'operation', + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + }, + }, + { + id: 'cursor', + title: 'Cursor', + type: 'short-input', + placeholder: 'Paste a nextCursor to fetch the next page', + condition: { field: 'operation', value: 'query_rows' }, + }, + ...getTrigger('table_new_row').subBlocks, + ], + + tools: { + access: [ + 'table_insert_row', + 'table_batch_insert_rows', + 'table_upsert_row', + 'table_update_row', + 'table_update_rows_by_filter', + 'table_delete_row', + 'table_delete_rows_by_filter', + 'table_query_rows_v2', + 'table_get_row', + 'table_get_schema', + ], + config: { + tool: (params) => { + const toolMap: Record = { + insert_row: 'table_insert_row', + batch_insert_rows: 'table_batch_insert_rows', + upsert_row: 'table_upsert_row', + update_row: 'table_update_row', + update_rows_by_filter: 'table_update_rows_by_filter', + delete_row: 'table_delete_row', + delete_rows_by_filter: 'table_delete_rows_by_filter', + query_rows: 'table_query_rows_v2', + get_row: 'table_get_row', + get_schema: 'table_get_schema', + } + return toolMap[params.operation] || 'table_query_rows_v2' + }, + params: (params) => { + const { operation, ...rest } = params + const transformer = paramTransformers[operation] + if (transformer) { + return transformer(rest as TableBlockParams) + } + return rest + }, + }, + }, + + inputs: { + operation: { type: 'string', description: 'Table operation to perform' }, + tableId: { type: 'string', description: 'Table identifier' }, + data: { type: 'json', description: 'Row data for insert/update' }, + rows: { type: 'array', description: 'Array of row data for batch insert' }, + rowId: { type: 'string', description: 'Row identifier for ID-based operations' }, + filterInput: { + type: 'json', + description: + 'Filter — a predicate object {"all":[{"field":"wins","op":"gte","value":10}]} (or visual builder conditions). Used by query and bulk update/delete.', + }, + sortInput: { + type: 'json', + description: + 'Order — a sort spec [{"field":"wins","direction":"desc"}] (or visual sort conditions). Query only.', + }, + limit: { + type: 'number', + description: + 'Page row limit (optional — omit to return the entire result, which fails if it exceeds 5MB); optional cap for bulk update/delete', + }, + cursor: { type: 'string', description: 'Opaque pagination cursor from a prior query response' }, + conflictColumn: { + type: 'string', + description: + 'Unique column to match on for upsert (required if the table has multiple unique columns)', + }, + }, + + outputs: { + success: { type: 'boolean', description: 'Operation success status' }, + row: { + type: 'json', + description: 'Single row data', + condition: { + field: 'operation', + value: ['get_row', 'insert_row', 'upsert_row', 'update_row'], + }, + }, + operation: { + type: 'string', + description: 'Operation performed (insert or update)', + condition: { field: 'operation', value: 'upsert_row' }, + }, + rows: { + type: 'array', + description: 'Array of rows', + condition: { field: 'operation', value: ['query_rows', 'batch_insert_rows'] }, + }, + rowCount: { + type: 'number', + description: 'Rows returned (query) or total rows in the table (get schema)', + condition: { field: 'operation', value: ['query_rows', 'get_schema'] }, + }, + totalCount: { + type: 'number', + description: 'Total rows matching the predicate (first page only)', + condition: { field: 'operation', value: 'query_rows' }, + }, + nextCursor: { + type: 'string', + description: 'Cursor to fetch the next page, or null on the last page', + condition: { field: 'operation', value: 'query_rows' }, + }, + insertedCount: { + type: 'number', + description: 'Number of rows inserted', + condition: { field: 'operation', value: 'batch_insert_rows' }, + }, + updatedCount: { + type: 'number', + description: 'Number of rows updated', + condition: { field: 'operation', value: 'update_rows_by_filter' }, + }, + updatedRowIds: { + type: 'array', + description: 'IDs of updated rows', + condition: { field: 'operation', value: 'update_rows_by_filter' }, + }, + deletedCount: { + type: 'number', + description: 'Number of rows deleted', + condition: { field: 'operation', value: ['delete_row', 'delete_rows_by_filter'] }, + }, + deletedRowIds: { + type: 'array', + description: 'IDs of deleted rows', + condition: { field: 'operation', value: 'delete_rows_by_filter' }, + }, + name: { + type: 'string', + description: 'Table name', + condition: { field: 'operation', value: 'get_schema' }, + }, + columns: { + type: 'array', + description: 'Column definitions (each includes its stable id)', + condition: { field: 'operation', value: 'get_schema' }, + }, + columnCount: { + type: 'number', + description: 'Number of columns', + condition: { field: 'operation', value: 'get_schema' }, + }, + maxRows: { + type: 'number', + description: "Max rows per table for the workspace's plan", + condition: { field: 'operation', value: 'get_schema' }, + }, + message: { type: 'string', description: 'Operation status message' }, + }, + triggers: { + enabled: true, + available: ['table_new_row'], + }, +} diff --git a/apps/sim/blocks/registry-maps.minimal.ts b/apps/sim/blocks/registry-maps.minimal.ts index 29d6a0c3875..39ab88a9ed1 100644 --- a/apps/sim/blocks/registry-maps.minimal.ts +++ b/apps/sim/blocks/registry-maps.minimal.ts @@ -27,6 +27,7 @@ import { SimWorkspaceEventBlock } from '@/blocks/blocks/sim_workspace_event' import { SlackBlock, SlackV2Block } from '@/blocks/blocks/slack' import { StartTriggerBlock } from '@/blocks/blocks/start_trigger' import { TableBlock } from '@/blocks/blocks/table' +import { TableV2Block } from '@/blocks/blocks/table_v2' import { TranslateBlock } from '@/blocks/blocks/translate' import { VariablesBlock } from '@/blocks/blocks/variables' import { WaitBlock } from '@/blocks/blocks/wait' @@ -81,6 +82,7 @@ export const BLOCK_REGISTRY: Record = { slack_v2: SlackV2Block, start_trigger: StartTriggerBlock, table: TableBlock, + table_v2: TableV2Block, translate: TranslateBlock, variables: VariablesBlock, wait: WaitBlock, diff --git a/apps/sim/blocks/registry-maps.ts b/apps/sim/blocks/registry-maps.ts index 70f443bec70..aa4dfac717f 100644 --- a/apps/sim/blocks/registry-maps.ts +++ b/apps/sim/blocks/registry-maps.ts @@ -294,6 +294,7 @@ import { STSBlock, STSBlockMeta } from '@/blocks/blocks/sts' import { SttBlock, SttV2Block } from '@/blocks/blocks/stt' import { SupabaseBlock, SupabaseBlockMeta } from '@/blocks/blocks/supabase' import { TableBlock } from '@/blocks/blocks/table' +import { TableV2Block } from '@/blocks/blocks/table_v2' import { TailscaleBlock, TailscaleBlockMeta } from '@/blocks/blocks/tailscale' import { TavilyBlock, TavilyBlockMeta } from '@/blocks/blocks/tavily' import { TelegramBlock, TelegramBlockMeta } from '@/blocks/blocks/telegram' @@ -613,6 +614,7 @@ export const BLOCK_REGISTRY: Record = { stt_v2: SttV2Block, supabase: SupabaseBlock, table: TableBlock, + table_v2: TableV2Block, tailscale: TailscaleBlock, tavily: TavilyBlock, telegram: TelegramBlock, diff --git a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts index 565d92bc9d8..8dfc98f70ba 100644 --- a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts +++ b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts @@ -27,6 +27,7 @@ import { } from '@/lib/billing/storage' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import type { DbOrTx } from '@/lib/db/types' +import { nKeysBetween } from '@/lib/table/order-key' import type { TableSchema } from '@/lib/table/types' import { deleteFile, @@ -840,6 +841,21 @@ export async function copyForkResourceContent(params: { try { let copied = 0 let afterId: string | null = null + // `order_key` is nullable, and spreading `...row` would inherit NULLs into a + // brand-new tableId that the one-shot backfill script-migration never revisits + // (it snapshots the pending set up front) — leaving rows the keyset pager has to + // special-case forever. Mint keys for the unkeyed ones instead. They sort last in + // the source (NULLS LAST, id tiebreak) and this loop pages by id, so consuming a + // pre-generated run appended after the source's max key preserves visual order. + const [{ maxKey = null, unkeyed = 0 } = {}] = await db + .select({ + maxKey: sql`max(${userTableRows.orderKey})`, + unkeyed: sql`count(*) filter (where ${userTableRows.orderKey} is null)`, + }) + .from(userTableRows) + .where(eq(userTableRows.tableId, table.sourceId)) + const mintedKeys = unkeyed > 0 ? nKeysBetween(maxKey, null, Number(unkeyed)) : [] + let mintedIdx = 0 for (;;) { const where: SQL | undefined = afterId === null @@ -858,6 +874,7 @@ export async function copyForkResourceContent(params: { id: generateId(), tableId: table.childId, workspaceId: childWorkspaceId, + orderKey: row.orderKey ?? mintedKeys[mintedIdx++] ?? null, // Repoint resource-chip URLs in cell data at the child copies (no-op when no maps). data: contentRefMaps ? remapTableRowResourceUrls(row.data, contentRefMaps) : row.data, })) diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index 0f447344bd0..a1f13eba635 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -88,13 +88,13 @@ import { buildUpgradeHref } from '@/lib/billing/upgrade-reasons' import type { CsvHeaderMapping, EnrichmentRunDetail, - Filter, RowData, RowExecutionMetadata, RowExecutions, - Sort, + SortSpec, TableDefinition, TableMetadata, + TablePredicate, TableRow, WorkflowGroup, WorkflowGroupDependencies, @@ -129,8 +129,8 @@ export const TABLE_EXPORT_JOBS_STALE_TIME = 5 * 1000 type TableRowsParams = Omit & TableIdParamsInput & { - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } export type TableRowsResponse = Pick< @@ -416,8 +416,8 @@ interface InfiniteTableRowsParams { workspaceId: string tableId: string pageSize: number - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null enabled?: boolean } @@ -433,8 +433,8 @@ interface FindTableRowsParams { workspaceId: string tableId: string q: string - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } export interface TableFindResult { @@ -1225,7 +1225,7 @@ interface DeleteTableRowsAsyncVariables { filter?: DeleteTableRowsAsyncBody['filter'] /** Active sort — together with `filter` it identifies the exact rows query to optimistically * strip, so we don't clear unrelated cached views (other filters/sorts). */ - sort?: Sort | null + sort?: SortSpec | null /** Rows deselected after "select all" — spared by the job. */ excludeRowIds?: string[] /** Doomed-row estimate shown in the confirm — persisted on the job so server counts can @@ -1259,7 +1259,13 @@ export function useDeleteTableRowsAsync({ workspaceId, tableId }: RowMutationCon // Target the exact infinite-rows query for the view the user is on — not every cached view. const activeKey = tableKeys.infiniteRows( tableId, - tableRowsParamsKey({ pageSize: TABLE_LIMITS.MAX_QUERY_LIMIT, filter: filter ?? null, sort }) + tableRowsParamsKey({ + pageSize: TABLE_LIMITS.MAX_QUERY_LIMIT, + // The wire type is the dual-grammar union; the grid only ever sends its + // own predicate state, so the cache key narrows to that shape. + filter: (filter as TablePredicate | undefined) ?? null, + sort, + }) ) await queryClient.cancelQueries({ queryKey: activeKey }) const previousRows = @@ -1557,10 +1563,10 @@ interface CancelRunsParams { scope: 'all' | 'row' rowId?: string /** Scope-`all` only: cancel just the cells on rows matching this filter (filtered select-all Stop). */ - filter?: Filter + filter?: TablePredicate /** Active sort — with `filter` it identifies the exact rows query whose cells the optimistic * cancel may flip (other cached views contain rows the server won't touch). */ - sort?: Sort | null + sort?: SortSpec | null /** Scope-`all` only: deselected rows whose cells keep running. */ excludeRowIds?: string[] } @@ -2192,7 +2198,7 @@ interface RunColumnVariables { /** "Select all under a filter" — run every row matching this filter (mutually exclusive with * `rowIds`). Optimistic stamping is skipped (like `limit`) since the matching set isn't known * client-side; the dispatcher's real pending stamps drive the UI. */ - filter?: Filter + filter?: TablePredicate /** Select-all scope only: deselected rows — skipped by the dispatcher and the optimistic stamp. */ excludeRowIds?: string[] /** Cap the run to the first `max` eligible rows. Omit for an unbounded run. diff --git a/apps/sim/lib/api/client/request.test.ts b/apps/sim/lib/api/client/request.test.ts index e28ed0837b2..f7a03a53574 100644 --- a/apps/sim/lib/api/client/request.test.ts +++ b/apps/sim/lib/api/client/request.test.ts @@ -54,18 +54,36 @@ describe('requestJson query serialization', () => { expect(calledUrl).not.toContain('[object Object]') }) - it('throws instead of silently corrupting an array-of-objects query param', async () => { - mockFetchReturning({ ok: true }) + it('JSON-encodes an array-of-objects query param instead of corrupting or throwing', async () => { + // A SortSpec ([{field, direction}]) is the everyday case: repeat-append + // would send "[object Object]", so the whole array travels as ONE JSON + // string param, mirroring plain objects; the server contract decodes it. + const fetchMock = mockFetchReturning({ ok: true }) - const badContract = defineRouteContract({ + const contract = defineRouteContract({ method: 'GET', path: '/api/test', query: z.object({ items: z.array(z.object({ a: z.string() })) }), response: { mode: 'json', schema: z.object({ ok: z.boolean() }) }, }) - await expect(requestJson(badContract, { query: { items: [{ a: 'x' }] } })).rejects.toThrow( - /arrays of objects are not URL-safe/ - ) + await requestJson(contract, { query: { items: [{ a: 'x' }] } }) + const url = String(fetchMock.mock.calls[0][0]) + expect(decodeURIComponent(url)).toContain('items=[{"a":"x"}]') + }) + + it('keeps repeat-append for scalar arrays', async () => { + const fetchMock = mockFetchReturning({ ok: true }) + + const contract = defineRouteContract({ + method: 'GET', + path: '/api/test', + query: z.object({ tags: z.array(z.string()) }), + response: { mode: 'json', schema: z.object({ ok: z.boolean() }) }, + }) + + await requestJson(contract, { query: { tags: ['a', 'b'] } }) + const url = String(fetchMock.mock.calls[0][0]) + expect(url).toContain('tags=a&tags=b') }) }) diff --git a/apps/sim/lib/api/client/request.ts b/apps/sim/lib/api/client/request.ts index a68eb96fd13..cf4b5cef479 100644 --- a/apps/sim/lib/api/client/request.ts +++ b/apps/sim/lib/api/client/request.ts @@ -67,19 +67,17 @@ function appendQuery(path: string, query: unknown): string { if (value === undefined || value === null || value === '') continue if (Array.isArray(value)) { + // An array of objects (e.g. a SortSpec) is not repeat-append-able — each + // item would stringify to "[object Object]" and silently corrupt the + // request (the knowledge tagFilters bug). Encode the WHOLE array as one + // JSON string param, mirroring how plain objects are sent below; the + // server-side contract decodes it. Scalar arrays keep repeat-append. + if (value.some((item) => item !== null && typeof item === 'object')) { + searchParams.set(key, JSON.stringify(value)) + continue + } for (const item of value) { if (item === undefined || item === null || item === '') continue - // A non-scalar in a query array would stringify to "[object Object]" and - // silently corrupt the request. Encode such values as a single JSON - // string param and decode them server-side instead. Failing loudly here - // keeps the boundary honest (this is how the knowledge tagFilters bug - // shipped undetected). - if (typeof item === 'object') { - throw new Error( - `Cannot serialize query param "${key}": arrays of objects are not URL-safe — ` + - 'encode the value as a JSON string param and decode it server-side.' - ) - } searchParams.append(key, String(item)) } continue diff --git a/apps/sim/lib/api/contracts/tables-predicate.test.ts b/apps/sim/lib/api/contracts/tables-predicate.test.ts new file mode 100644 index 00000000000..8f45d758822 --- /dev/null +++ b/apps/sim/lib/api/contracts/tables-predicate.test.ts @@ -0,0 +1,261 @@ +/** + * @vitest-environment node + * + * The v2 query/bulk filter wire format is the typed `{ all | any: [...] }` + * predicate tree. The contract validates structure; column-level validation + * (unknown field, json-op) runs server-side in `validate.ts`. + */ +import { describe, expect, it } from 'vitest' +import { + deleteTableRowsBodySchema, + predicateSchema, + rowQueryBodySchema, + tableRowsQuerySchema, + updateRowsByFilterBodySchema, +} from '@/lib/api/contracts/tables' +import { validatePredicate } from '@/lib/table/query-builder/validate' + +describe('rowQueryBodySchema', () => { + it('accepts a predicate/sort object, leaves limit unbounded, has no offset', () => { + const parsed = rowQueryBodySchema.parse({ + workspaceId: 'ws-1', + predicate: { + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'status', op: 'in', value: ['active', 'pending'] }, + ], + }, + sort: [{ field: 'wins', direction: 'desc' }], + cursor: 'abc', + }) + expect(parsed.predicate).toEqual({ + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'status', op: 'in', value: ['active', 'pending'] }, + ], + }) + // Omitted limit stays undefined — the query returns all matching rows. + expect(parsed.limit).toBeUndefined() + expect('offset' in parsed).toBe(false) + }) + + it('accepts a nested any/all predicate', () => { + expect( + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { + any: [ + { field: 'status', op: 'eq', value: 'active' }, + { all: [{ field: 'wins', op: 'gte', value: 5 }] }, + ], + }, + }).success + ).toBe(true) + }) + + it('allows omitting the predicate (match all)', () => { + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1' }).success).toBe(true) + }) + + it('rejects an unknown operator and a malformed leaf', () => { + expect( + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { all: [{ field: 'wins', op: 'bogus', value: 1 }] }, + }).success + ).toBe(false) + expect( + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { all: [{ op: 'eq', value: 1 }] }, + }).success + ).toBe(false) + }) + + it('accepts a large explicit limit (no row cap) but rejects limit < 1', () => { + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1', limit: 100000 }).success).toBe(true) + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1', limit: 0 }).success).toBe(false) + }) +}) + +describe('bulk schemas accept either a predicate tree or the legacy filter object', () => { + it('delete accepts a predicate filter', () => { + expect( + deleteTableRowsBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: { all: [{ field: 'status', op: 'eq', value: 'archived' }] }, + }).success + ).toBe(true) + }) + + it('delete still accepts the legacy object filter (v1 callers)', () => { + expect( + deleteTableRowsBodySchema.safeParse({ workspaceId: 'ws-1', filter: { status: 'archived' } }) + .success + ).toBe(true) + }) + + it('update accepts a predicate filter', () => { + expect( + updateRowsByFilterBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: { all: [{ field: 'wins', op: 'gte', value: 10 }] }, + data: { active: false }, + }).success + ).toBe(true) + }) +}) + +/** + * The predicate tree is parsed by a recursive `z.lazy` union. A few thousand + * nested groups overflow the stack inside `safeParse`, and a `RangeError` + * escaping a parser is a 500 on a public endpoint, not a 400. + */ +describe('predicate depth / size guard', () => { + it('rejects a deeply nested tree with a validation issue, not a RangeError', () => { + let node: unknown = { all: [{ field: 'a', op: 'eq', value: 1 }] } + for (let i = 0; i < 5000; i++) node = { all: [node] } + + const result = predicateSchema.safeParse(node) + + expect(result.success).toBe(false) + expect(JSON.stringify(result.error?.issues)).toMatch(/nesting is too deep/) + }) + + it('rejects a wide-but-shallow tree past the node cap', () => { + const node = { + all: Array.from({ length: 60 }, () => ({ + all: Array.from({ length: 60 }, () => ({ field: 'a', op: 'eq', value: 1 })), + })), + } + + const result = predicateSchema.safeParse(node) + + expect(result.success).toBe(false) + expect(JSON.stringify(result.error?.issues)).toMatch(/too many conditions/) + }) + + it('still accepts a realistic nested predicate', () => { + expect( + predicateSchema.safeParse({ + all: [ + { field: 'status', op: 'eq', value: 'active' }, + { + any: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'name', op: 'contains', value: 'jo' }, + ], + }, + ], + }).success + ).toBe(true) + }) +}) + +/** + * Zod strips unrecognized keys by default, so before the schemas were made + * strict a hybrid node parsed clean against the group branch with its leaf half + * silently deleted — turning "delete archived rows for tenant acme" into + * "delete EVERY row for tenant acme". `validatePredicate`'s hybrid guard could + * not catch it: the keys were gone before it ran. + */ +describe('hybrid group+leaf nodes are rejected, not silently narrowed', () => { + const hybrid = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + } + + it('rejects rather than dropping the leaf half', () => { + const result = predicateSchema.safeParse(hybrid) + expect(result.success).toBe(false) + // The dangerous outcome: parsing succeeds having quietly widened the filter. + expect(result.success ? result.data : null).not.toEqual({ all: hybrid.all }) + }) + + /** + * The bulk schemas union the predicate tree with the legacy `$`-object, and + * that legacy branch accepts any non-empty object — so it absorbs the hybrid + * and the SCHEMA cannot reject it. Crucially the legacy branch does NOT strip, + * so `all` survives, the route's `isTablePredicate` check routes it back to + * `validatePredicate`, and the hybrid guard there rejects it (→ 400 via + * `route.ts:325`). Asserted here so a future change to either layer that + * removes one of them fails loudly. + */ + it('keeps the hybrid intact through the bulk schemas so the runtime guard can see it', () => { + for (const parsed of [ + deleteTableRowsBodySchema.safeParse({ workspaceId: 'ws-1', filter: hybrid }), + updateRowsByFilterBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: hybrid, + data: { active: false }, + }), + ]) { + expect(parsed.success).toBe(true) + // The leaf half must NOT have been silently dropped on the way through. + expect(parsed.success && parsed.data.filter).toMatchObject({ + all: hybrid.all, + field: 'status', + }) + } + }) + + it('and validatePredicate then rejects it', () => { + expect(() => + validatePredicate(hybrid as never, [ + { name: 'tenant_id', type: 'string' }, + { name: 'status', type: 'string' }, + ]) + ).toThrow(/not both/) + }) + + it('rejects an unknown key on a leaf (a typo must not be dropped)', () => { + expect( + predicateSchema.safeParse({ all: [{ field: 'a', op: 'eq', vlaue: 'typo' }] }).success + ).toBe(false) + }) + + it('still accepts well-formed nodes', () => { + expect( + predicateSchema.safeParse({ + all: [{ field: 'a', op: 'eq', value: 1 }, { any: [{ field: 'b', op: 'isNull' }] }], + }).success + ).toBe(true) + }) +}) + +/** + * Wire transport: requestJson serializes structured query params as JSON + * strings; jsonQueryValue decodes them before the union runs. Without it, a + * sorted or filtered grid request 400s at the boundary. + */ +describe('query-string JSON transport (jsonQueryValue)', () => { + it('decodes string-encoded predicate, legacy filter, and sort spec', () => { + const parsed = rowQueryStringSchemaProbe({ + workspaceId: 'ws-1', + filter: JSON.stringify({ all: [{ field: 'a', op: 'eq', value: 1 }] }), + sort: JSON.stringify([{ field: 'a', direction: 'asc' }]), + }) + expect(parsed.filter).toEqual({ all: [{ field: 'a', op: 'eq', value: 1 }] }) + expect(parsed.sort).toEqual([{ field: 'a', direction: 'asc' }]) + + const legacy = rowQueryStringSchemaProbe({ + workspaceId: 'ws-1', + filter: JSON.stringify({ status: { $eq: 'x' } }), + sort: JSON.stringify({ status: 'desc' }), + }) + expect(legacy.filter).toEqual({ status: { $eq: 'x' } }) + expect(legacy.sort).toEqual({ status: 'desc' }) + }) + + it('still rejects a non-JSON garbage string with the real schema error', () => { + expect(() => rowQueryStringSchemaProbe({ workspaceId: 'ws-1', filter: 'not json' })).toThrow() + }) +}) + +function rowQueryStringSchemaProbe(input: Record) { + const result = tableRowsQuerySchema.safeParse(input) + if (!result.success) throw new Error(JSON.stringify(result.error.issues[0])) + return result.data +} diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 2da6ded270c..5bccaa52069 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -11,16 +11,27 @@ import type { CsvHeaderMapping, EnrichmentRunDetail, Filter, + Predicate, + PredicateNode, RowData, Sort, + SortSpec, TableDefinition, TableLocks, TableMetadata, + TablePredicate, TableRow, TableRowsCursor, TableViewConfig, } from '@/lib/table' -import { COLUMN_TYPES, MAX_SELECT_OPTIONS, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants' +import { + COLUMN_TYPES, + FILTER_OPS, + MAX_SELECT_OPTIONS, + NAME_PATTERN, + SORT_DIRECTIONS, + TABLE_LIMITS, +} from '@/lib/table/constants' import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table/import' export const domainObjectSchema = () => z.custom(isRecordLike) @@ -370,6 +381,137 @@ const nonEmptyFilterSchema = domainObjectSchema().refine( const filterSchema = domainObjectSchema() +/* --------------------------- v2 predicate grammar --------------------------- */ + +/** + * Body cap for the row-query routes. A query body is a predicate tree plus a + * cursor; the largest legitimate one — a 100-member predicate with 1000-element + * `in` lists — is comfortably under 1 MB. The 50 MB platform default would let a + * caller buffer two orders of magnitude more before any schema check runs. + */ +export const TABLE_QUERY_MAX_BODY_BYTES = 1024 * 1024 + +/** Max members in one `all`/`any` group — a generous bound against pathological trees. */ +const MAX_PREDICATE_GROUP_SIZE = 100 +/** Max sort keys — more than a few is already a smell. */ +const MAX_SORT_KEYS = 16 +/** Max nesting levels of `all`/`any` groups. Ten is already unreadable. */ +const MAX_PREDICATE_DEPTH = 10 +/** Max nodes in the whole tree, so a wide-but-shallow tree can't amplify either. */ +const MAX_PREDICATE_NODES = 500 + +/** + * Iterative depth/size walk over an unvalidated predicate tree. Runs BEFORE the + * recursive Zod schema: a few thousand nested `{all:[...]}` levels overflow the + * stack inside `safeParse`, and a `RangeError` from a parser is a 500, not a 400. + * The walk itself must stay iterative for the same reason. + */ +function predicateTreeTooLarge(root: unknown): string | null { + const stack: Array<{ node: unknown; depth: number }> = [{ node: root, depth: 1 }] + let nodes = 0 + + while (stack.length > 0) { + const { node, depth } = stack.pop()! + if (++nodes > MAX_PREDICATE_NODES) { + return `Filter has too many conditions (max ${MAX_PREDICATE_NODES})` + } + if (depth > MAX_PREDICATE_DEPTH) { + return `Filter nesting is too deep (max ${MAX_PREDICATE_DEPTH} levels)` + } + if (typeof node !== 'object' || node === null) continue + const group = node as { all?: unknown; any?: unknown } + const members = Array.isArray(group.all) + ? group.all + : Array.isArray(group.any) + ? group.any + : null + if (!members) continue + for (const member of members) stack.push({ node: member, depth: depth + 1 }) + } + return null +} + +/** + * v2 filter wire format: the typed `{ all | any: [...] }` predicate tree (same + * shape the engine consumes). Structure is validated here; schema-awareness + * (unknown column, json-op rejection) is enforced server-side by + * `validatePredicate` once the table's columns are known. + */ +/** + * Both node shapes are `strictObject`, and that is load-bearing rather than + * fussiness. Zod strips unrecognized keys by default, so a hybrid node carrying + * BOTH a group key and a leaf's `field`/`op`/`value` parsed clean against the + * group branch with the leaf half silently deleted. On the bulk paths that turns + * "delete archived rows for tenant acme" into "delete every row for tenant acme". + * `validatePredicate`'s hybrid guard could never catch it — the keys were gone + * before it ran. Strict on BOTH branches is required: strict on the group alone + * would just fall through to the leaf branch, which is the more dangerous reading. + */ +// double-cast-allowed: `z.unknown()` keeps the runtime permissive (a leaf value +// is arbitrary JSON), but infers `unknown`, which is wider than +// `Predicate['value']`. The narrowing is type-level only — nothing is coerced. +const predicateLeafSchema = z.strictObject({ + field: z.string().min(1, 'field is required').max(128), + op: z.enum(FILTER_OPS), + value: z.unknown().optional(), +}) as unknown as z.ZodType + +const predicateNodeSchema: z.ZodType = z.lazy(() => + z.union([predicateGroupSchema, predicateLeafSchema]) +) + +const predicateTreeSchema: z.ZodType = z.lazy(() => + z.union([ + // `.min(1)`: an empty group compiles to no WHERE clause, which on the bulk + // delete/update paths reads as "match everything" rather than "match nothing". + z.strictObject({ + all: z + .array(predicateNodeSchema) + .min(1, 'A filter group must contain at least one condition') + .max(MAX_PREDICATE_GROUP_SIZE), + }), + z.strictObject({ + any: z + .array(predicateNodeSchema) + .min(1, 'A filter group must contain at least one condition') + .max(MAX_PREDICATE_GROUP_SIZE), + }), + ]) +) +const predicateGroupSchema = predicateTreeSchema + +/** + * The boundary predicate schema: depth/size guard first, then the recursive + * structural parse. The guard is only applied at the top level — every nested + * group is strictly shallower, so re-checking inside the recursion would be + * redundant work on the hot path. + */ +export const predicateSchema = z + .unknown() + .superRefine((value, ctx) => { + const problem = predicateTreeTooLarge(value) + if (problem) ctx.addIssue({ code: 'custom', message: problem }) + }) + // double-cast-allowed: the pipe's inferred input is `unknown`, and letting TS + // widen the recursive lazy union through it makes typecheck OOM + .pipe(predicateTreeSchema) as unknown as z.ZodType + +/** v2 sort wire format: an ordered list of `{ field, direction }`. */ +export const sortSpecSchema: z.ZodType = z + .array( + z.object({ + field: z.string().min(1, 'field is required').max(128), + direction: z.enum(SORT_DIRECTIONS), + }) + ) + .max(MAX_SORT_KEYS) + +/** + * Bulk update/delete filter accepts either the v2 predicate tree (v2 block / + * agent) or the legacy `$`-operator object (v1 callers / stored workflows). + */ +const bulkFilterSchema = z.union([predicateSchema, nonEmptyFilterSchema]) + const optionalPositiveLimit = (max: number, label: string) => z.preprocess( (value) => (value === null || value === undefined || value === '' ? undefined : Number(value)), @@ -388,7 +530,7 @@ export const deleteTableRowBodySchema = z.object({ export const deleteTableRowsBodySchema = z .object({ workspaceId: workspaceIdSchema, - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), limit: optionalPositiveLimit(TABLE_LIMITS.MAX_BULK_OPERATION_SIZE, 'Limit').optional(), rowIds: z .array(z.string().min(1)) @@ -403,17 +545,37 @@ export const deleteTableRowsBodySchema = z message: 'Provide either filter or rowIds, but not both', }) +/** + * Query-param transport for a structured value: `requestJson` serializes + * objects/arrays into a single JSON-string param, and this decodes it before + * the real schema runs. Non-JSON strings pass through untouched so the inner + * schema produces the real error; already-parsed values (POST bodies reusing a + * schema) are untouched too. + */ +const jsonQueryValue = (schema: S) => + z.preprocess((value) => { + if (typeof value !== 'string' || value === '') return value + try { + return JSON.parse(value) + } catch { + return value + } + }, schema) + /** Unrefined base so v1 contracts can `.extend()` — consumers use {@link tableRowsQuerySchema}. */ export const tableRowsQueryBaseSchema = z.object({ workspaceId: workspaceIdSchema, - filter: domainObjectSchema().optional(), - sort: domainObjectSchema().optional(), + // Dual-grammar during the v2 transition: the strict predicate tree wins the + // union; anything else falls through to the legacy `$`-object. Same for sort: + // an ordered spec array vs the legacy `{col: dir}` record. + filter: jsonQueryValue(z.union([predicateSchema, domainObjectSchema()])).optional(), + sort: jsonQueryValue(z.union([sortSpecSchema, domainObjectSchema()])).optional(), /** * Keyset cursor `(orderKey, id)` for the default row order — each page is an index seek * instead of OFFSET's scan-and-discard. Mutually exclusive with `sort` (cursors only make * sense on the default order); takes precedence over `offset`. */ - after: domainObjectSchema().optional(), + after: jsonQueryValue(domainObjectSchema()).optional(), limit: z .preprocess( (value) => @@ -453,7 +615,7 @@ export const tableRowsQuerySchema = tableRowsQueryBaseSchema.refine( export const updateRowsByFilterBodySchema = z.object({ workspaceId: workspaceIdSchema, - filter: nonEmptyFilterSchema, + filter: bulkFilterSchema, data: rowDataSchema, limit: optionalPositiveLimit(TABLE_LIMITS.MAX_BULK_OPERATION_SIZE, 'Limit').optional(), }) @@ -663,16 +825,65 @@ export const listTableRowsContract = defineRouteContract({ totalCount: z.number().nullable(), limit: z.number(), offset: z.number(), + /** Non-null when more rows exist — a page may be cut by the byte budget + * before reaching `limit`, so page fullness is not a termination signal. */ + nextCursor: z.string().nullable(), + }) + ), + }, +}) + +/** + * v2 query surface: typed `predicate`/`sort` objects + opaque cursor pagination. + * No `offset` (the cursor encodes paging state) and no after/sort refine (the + * cursor carries the sort context). Structure is validated here; column-level + * validation (unknown field, json-op) runs server-side via `validatePredicate`. + */ +export const rowQueryBodySchema = z.object({ + workspaceId: z.string().min(1, 'Workspace ID is required'), + predicate: predicateSchema.optional(), + sort: sortSpecSchema.optional(), + // Omitted limit returns the ENTIRE matching result, failing fast (400) when + // it exceeds the response byte budget. An explicit limit caps the page row + // count; the byte budget may still end a page early with nextCursor set. + limit: z.preprocess( + (value) => (value === null || value === undefined || value === '' ? undefined : Number(value)), + z + .number({ error: 'Limit must be a number' }) + .int('Limit must be an integer') + .min(1, 'Limit must be at least 1') + .optional() + ), + cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), +}) + +export type RowQueryBody = z.input + +export const rowQueryContract = defineRouteContract({ + method: 'POST', + path: '/api/table/[tableId]/query', + params: tableIdParamsSchema, + body: rowQueryBodySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + rows: z.array(tableRowSchema), + rowCount: z.number(), + totalCount: z.number().nullable(), + limit: z.number(), + nextCursor: z.string().nullable(), }) ), }, }) +export type RowQueryResponse = ContractJsonResponse export const findTableRowsQuerySchema = z.object({ workspaceId: workspaceIdSchema, q: requiredFieldSchema('Search query is required'), - filter: domainObjectSchema().optional(), - sort: domainObjectSchema().optional(), + filter: jsonQueryValue(z.union([predicateSchema, domainObjectSchema()])).optional(), + sort: jsonQueryValue(z.union([sortSpecSchema, domainObjectSchema()])).optional(), }) /** One matching cell: its 0-based ordinal in the filtered+sorted view, its row id, and the column name. */ @@ -1101,7 +1312,7 @@ export const deleteTableRowsContract = defineRouteContract({ */ export const deleteTableRowsAsyncBodySchema = z.object({ workspaceId: workspaceIdSchema, - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), excludeRowIds: z .array(z.string().min(1)) .max( @@ -1296,7 +1507,7 @@ export const cancelTableRunsBodySchema = z workspaceId: workspaceIdSchema, scope: z.enum(['all', 'row']), rowId: z.string().min(1).optional(), - filter: domainObjectSchema().optional(), + filter: z.union([predicateSchema, domainObjectSchema()]).optional(), /** Scope-`all` only: rows deselected from the selection — their cells keep running. */ excludeRowIds: z .array(z.string().min(1)) @@ -1400,7 +1611,7 @@ export const runColumnBodySchema = z rowIds: z.array(z.string().min(1)).min(1).optional(), /** "Select all under a filter" — run every row matching this filter instead of `rowIds`. The * dispatcher walks only matching rows (paginated), so no id list is materialized. */ - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), /** Select-all scope only: rows deselected from the selection — the dispatcher skips them. */ excludeRowIds: z .array(z.string().min(1)) @@ -1526,8 +1737,11 @@ export const tableEventStreamContract = defineRouteContract({ * never invalidates a view. */ export const tableViewConfigSchema = tableMetadataSchema.extend({ - filter: filterSchema.nullable().optional(), - sort: domainObjectSchema().nullable().optional(), + // The v2 predicate/sort grammar — same wire as the query routes, so a saved + // view gets the same strictness and depth bounds as a live filter, and its + // config can later feed the v2 surfaces without conversion. + filter: predicateSchema.nullable().optional(), + sort: sortSpecSchema.nullable().optional(), }) satisfies z.ZodType export const tableViewSchema = z.object({ diff --git a/apps/sim/lib/api/contracts/v2/tables/index.ts b/apps/sim/lib/api/contracts/v2/tables/index.ts new file mode 100644 index 00000000000..34e3902f01d --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/tables/index.ts @@ -0,0 +1,115 @@ +import { z } from 'zod' +import { workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { + predicateSchema, + sortSpecSchema, + tableColumnSchema, + tableIdParamsSchema, +} from '@/lib/api/contracts/tables' +import { defineRouteContract } from '@/lib/api/contracts/types' + +/** + * Public v2 tables API — typed predicate grammar, cursor paging. + * + * Filters are the `{ all | any: [...] }` predicate tree (same shape the engine + * consumes); no string querystring dialect. Response bodies are fully typed. Row + * data is name-keyed and carries no storage internals (`position`/`orderKey`/ + * `executions`) — the public wire is `{ id, data, createdAt, updatedAt }`. + */ + +/** Default page size when `limit` is omitted (bounded, unlike the internal surface). */ +export const V2_DEFAULT_ROW_LIMIT = 100 +/** Hard cap on an explicit page `limit`. Larger pulls use `limit=0` or the async export. */ +export const V2_MAX_ROW_LIMIT = 1000 + +const successResponseSchema = (dataSchema: T) => + z.object({ + success: z.literal(true), + data: dataSchema, + }) + +const v2TableSummarySchema = z.object({ + id: z.string(), + name: z.string(), + description: z.string().nullable(), + schema: z.object({ columns: z.array(tableColumnSchema) }), + rowCount: z.number(), + maxRows: z.number(), + createdAt: z.string(), + updatedAt: z.string(), +}) + +/** Public row shape: name-keyed data, no storage internals. */ +export const v2TableRowSchema = z.object({ + id: z.string(), + data: z.record(z.string(), z.unknown()), + createdAt: z.string(), + updatedAt: z.string(), +}) + +export const v2ListTablesQuerySchema = z.object({ + workspaceId: workspaceIdSchema, +}) + +/** + * Rows query body. `predicate`/`sort` are the typed predicate tree / sort spec. + * `limit`: omitted → {@link V2_DEFAULT_ROW_LIMIT}; `0` → unbounded (whole result + * or a 400 `TABLE_QUERY_RESULT_TOO_LARGE`); `1..{@link V2_MAX_ROW_LIMIT}` → page cap. + */ +export const v2QueryRowsBodySchema = z.object({ + workspaceId: workspaceIdSchema, + predicate: predicateSchema.optional(), + sort: sortSpecSchema.optional(), + limit: z + .number({ error: 'Limit must be a number' }) + .int('Limit must be an integer') + .min(0, 'Limit must be at least 0 (use 0 for an unbounded query)') + .max( + V2_MAX_ROW_LIMIT, + `Limit cannot exceed ${V2_MAX_ROW_LIMIT}; use limit=0 for a full result or the async export for large datasets` + ) + .optional(), + cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), +}) + +export type V2ListTablesQuery = z.output +export type V2QueryRowsBody = z.input +export type V2TableRow = z.output + +export const v2ListTablesContract = defineRouteContract({ + method: 'GET', + path: '/api/v2/tables', + query: v2ListTablesQuerySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + tables: z.array(v2TableSummarySchema), + totalCount: z.number(), + }) + ), + }, +}) + +export const v2QueryRowsContract = defineRouteContract({ + method: 'POST', + path: '/api/v2/tables/[tableId]/query', + params: tableIdParamsSchema, + body: v2QueryRowsBodySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + rows: z.array(v2TableRowSchema), + rowCount: z.number(), + totalCount: z.number().nullable(), + limit: z.number(), + /** Opaque, short-lived token. Non-null when more rows remain; stop on null. */ + nextCursor: z.string().nullable(), + }) + ), + }, +}) + +export type V2ListTablesResponse = z.output<(typeof v2ListTablesContract)['response']['schema']> +export type V2QueryRowsResponse = z.output<(typeof v2QueryRowsContract)['response']['schema']> diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index bcdadb752fb..cda8d3097ea 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -4609,6 +4609,11 @@ export const UserTable: ToolCatalogEntry = { description: 'Array of column names to delete at once (for delete_column). Preferred over columnName when deleting multiple columns.', }, + cursor: { + type: 'string', + description: + 'Opaque pagination cursor for query_rows (optional). Omit for the first page; to fetch the next page, pass back the nextCursor from the previous result\'s "more available" message verbatim. Cannot be combined with a fresh order — the cursor already encodes the paging position.', + }, data: { type: 'object', description: 'Row data as key-value pairs (required for insert_row, update_row)', @@ -4646,7 +4651,7 @@ export const UserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'MongoDB-style filter for query_rows, update_rows_by_filter, delete_rows_by_filter', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -4681,7 +4686,7 @@ export const UserTable: ToolCatalogEntry = { limit: { type: 'number', description: - 'Maximum rows to return or affect (optional, default 100). Omit on update_rows_by_filter / delete_rows_by_filter to act on every match.', + 'Maximum rows per page for query_rows (optional). Omit to fetch the ENTIRE matching result in one response — the call fails if the result exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page may end early at the byte budget with more remaining; a non-null nextCursor in the result means more rows exist (continue with cursor). On update_rows_by_filter / delete_rows_by_filter, caps affected rows; omit to act on every match.', }, mapping: { type: 'object', @@ -4737,9 +4742,10 @@ export const UserTable: ToolCatalogEntry = { description: 'New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips.', }, - offset: { - type: 'number', - description: 'Number of rows to skip (optional for query_rows, default 0)', + order: { + type: 'array', + description: + 'Sort spec for query_rows (optional). Ordered list of {field, direction} where direction is asc or desc, e.g. [{"field":"wins","direction":"desc"},{"field":"name","direction":"asc"}].', }, options: { type: 'array', @@ -4831,11 +4837,6 @@ export const UserTable: ToolCatalogEntry = { "Cancellation scope for cancel_table_runs. 'all' cancels in-flight runs across the whole table; 'row' cancels only the row identified by rowId.", enum: ['all', 'row'], }, - sort: { - type: 'object', - description: - "Sort specification as { field: 'asc' | 'desc' } (optional for query_rows)", - }, tableId: { type: 'string', description: diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index 6029f71c0fe..ecb3c630ff0 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -3,6 +3,7 @@ import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/copilot/vfs/ import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { getColumnId } from '@/lib/table/column-keys' +import { TABLE_LIMITS } from '@/lib/table/constants' import { formatCsvCell, neutralizeCsvFormula, toCsvRow } from '@/lib/table/export-format' import { queryRows } from '@/lib/table/rows/service' import { getTableById, listTables } from '@/lib/table/service' @@ -427,7 +428,13 @@ export async function resolveInputFiles( continue } - const rows = await queryRows(table, {}, 'copilot-fn-exec') + // Keep the prior bounded mount — draining the whole table here was backed + // out for OOM, so don't ride the new unbounded queryRows default. + const rows = await queryRows( + table, + { limit: TABLE_LIMITS.DEFAULT_QUERY_LIMIT }, + 'copilot-fn-exec' + ) const columns = table.schema.columns const csvLines = [toCsvRow(columns.map((column) => neutralizeCsvFormula(column.name)))] diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts index b0dd887cb49..adebd05bd7a 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts @@ -141,6 +141,7 @@ import { normalizeSelectOptionsInput, userTableServerTool, } from '@/lib/copilot/tools/server/table/user-table' +import { encodeCursor } from '@/lib/table/rows/cursor' function buildTable(overrides: Partial = {}): TableDefinition { return { @@ -783,7 +784,7 @@ describe('userTableServerTool.query_rows', () => { }) }) - it('clamps an over-large query limit to MAX_QUERY_LIMIT instead of rejecting', async () => { + it('passes an explicit limit through unchanged (no row cap)', async () => { const result = await userTableServerTool.execute( { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 100000 } }, { userId: 'user-1', workspaceId: 'workspace-1' } @@ -791,20 +792,75 @@ describe('userTableServerTool.query_rows', () => { expect(result.success).toBe(true) const options = mockQueryRows.mock.calls[0][1] as Record - expect(options.limit).toBe(1000) + expect(options.limit).toBe(100000) }) - it('queries without execution metadata and passes limit/offset through', async () => { + it('omits the limit so queryRows returns every matching row', async () => { const result = await userTableServerTool.execute( - { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2, offset: 10 } }, + { operation: 'query_rows', args: { tableId: 'tbl_1' } }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(true) + const options = mockQueryRows.mock.calls[0][1] as Record + expect(options.limit).toBeUndefined() + }) + + it('decodes an opaque cursor into after/offset and skips the count', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'row_9', orderKey: 'a9' }, + keysetValid: true, + nextOffset: 20, + }) + const result = await userTableServerTool.execute( + { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2, cursor } }, { userId: 'user-1', workspaceId: 'workspace-1' } ) expect(result.success).toBe(true) const options = mockQueryRows.mock.calls[0][1] as Record expect(options.withExecutions).toBe(false) - expect(options.offset).toBe(10) - expect(result.data?.nextCursor).toBeUndefined() + expect(options.after).toEqual({ orderKey: 'a9', id: 'row_9' }) + // A cursor page never re-counts. + expect(options.includeTotal).toBe(false) + }) + + it('surfaces the opaque nextCursor (not an offset) in the "more available" message', async () => { + mockQueryRows.mockResolvedValueOnce({ + rows: [queryRow(1), queryRow(2)], + rowCount: 2, + totalCount: 10, + limit: 2, + offset: 0, + nextCursor: 'CURSOR_TOKEN_ABC', + }) + const result = await userTableServerTool.execute( + { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2 } }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.message).toContain('more available') + expect(result.message).toContain('cursor=CURSOR_TOKEN_ABC') + expect(result.message).not.toContain('offset=') + }) + + it('rejects a keyset cursor combined with a custom sort', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'row_9', orderKey: 'a9' }, + keysetValid: true, + nextOffset: 20, + }) + const result = await userTableServerTool.execute( + { + operation: 'query_rows', + args: { tableId: 'tbl_1', cursor, order: [{ field: 'name', direction: 'desc' }] }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(false) + expect(result.message).toMatch(/not valid for a sorted query/i) + expect(mockQueryRows).not.toHaveBeenCalled() }) }) @@ -835,7 +891,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 5000 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 5000, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -856,7 +916,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { it('deletes inline when the unbounded match count is within the cap', async () => { const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -874,7 +937,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -894,7 +961,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { }) const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) await flushDetached() @@ -929,7 +999,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { mockMarkTableJobRunning.mockResolvedValueOnce(false) const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -943,7 +1016,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -974,7 +1051,12 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 }, limit: 5000 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + limit: 5000, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -994,7 +1076,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -1015,7 +1101,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -1051,7 +1141,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { email: 'x' }, data: { email: 'y' } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'email', op: 'eq', value: 'x' }] }, + data: { email: 'y' }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -1073,7 +1167,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -1087,7 +1185,12 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index 9d3f35438ff..9f66adcb9da 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -1,6 +1,6 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' +import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId, generateShortId } from '@sim/utils/id' import { UserTable } from '@/lib/copilot/generated/tool-catalog-v1' import { @@ -30,9 +30,8 @@ import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, columnMatchesRef, - filterNamesToIds, rowDataNameToId, - sortNamesToIds, + sortSpecNamesToIds, } from '@/lib/table/column-keys' import { columnTypeForLeaf, deriveOutputColumnName } from '@/lib/table/column-naming' import { @@ -48,6 +47,9 @@ import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { assertRowDelete, assertRowUpdate, patchColumnIds } from '@/lib/table/mutation-locks' +import { predicateToFilter } from '@/lib/table/query-builder/converters' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' import { batchInsertRows, batchUpdateRows, @@ -61,15 +63,17 @@ import { updateRow, updateRowsByFilter, } from '@/lib/table/rows/service' -import { resolveFilterSelectValues } from '@/lib/table/select-values' +import { predicateToStorage } from '@/lib/table/select-values' import { createTable, deleteTable, getTableById, renameTable } from '@/lib/table/service' import type { ColumnDefinition, Filter, RowData, SelectOption, + SortSpec, TableDefinition, TableDeleteJobPayload, + TablePredicate, TableSchema, TableUpdateJobPayload, WorkflowGroup, @@ -675,33 +679,68 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) + // Typed predicate/sort objects, validated against the schema (column + // NAMES) then translated to storage ids. + let predicate: TablePredicate | undefined + if (args.filter) { + validatePredicate(args.filter, table.schema.columns) + predicate = predicateToStorage(args.filter, table.schema) + } + let orderSpec = args.order as SortSpec | undefined + if (orderSpec?.length) { + validateSortSpec(orderSpec, table.schema.columns) + orderSpec = sortSpecNamesToIds(orderSpec, idByName) + } + const sort = orderSpec?.length + ? Object.fromEntries(orderSpec.map((s) => [s.field, s.direction])) + : undefined + + // Opaque cursor pagination (keyset seek on the default order; the token + // hides an internal offset only for custom-sorted views, which a keyset + // physically can't page). A keyset cursor is bound to the default order, + // so it can't be combined with a fresh sort. + const cursor = args.cursor ? decodeCursor(args.cursor) : undefined + if (cursor) { + try { + // Keyset cursors bind to the default order; offset cursors to the + // exact sort they were minted under. + assertCursorSortBinding(cursor, sort) + } catch (bindError) { + return { success: false, message: getErrorMessage(bindError, 'Invalid cursor') } + } + } + + // No limit returns the ENTIRE matching result, failing fast once the + // 5MB byte budget is exceeded (caught below → structured tool error + // the model can react to by adding a filter or a limit). An explicit + // limit pages; byte-cut pages set nextCursor and the message says to + // continue with the opaque cursor. const toNamedRow = namedRowMapper(table.schema.columns) - // The model may request any number; we serve at most MAX_QUERY_LIMIT per page so a single - // tool result can't drain a whole table. `totalCount` in the response signals truncation, - // and the model pages with `offset`. const result = await queryRows( table, { - filter: args.filter - ? resolveFilterSelectValues( - filterNamesToIds(args.filter, idByName), - table.schema.columns - ) - : undefined, - sort: args.sort ? sortNamesToIds(args.sort, idByName) : undefined, - limit: - args.limit !== undefined - ? Math.min(args.limit, TABLE_LIMITS.MAX_QUERY_LIMIT) - : undefined, - offset: args.offset, + predicate, + sort, + limit: args.limit, + after: cursor?.after, + offset: cursor?.offset, + // Only the first page (no inbound cursor) pays for the COUNT(*). + includeTotal: !args.cursor, withExecutions: false, }, requestId ) + // nextCursor covers both cut kinds (explicit limit or the 5MB byte + // budget) — either way the truthful signal is "more rows exist". The + // token is opaque; the agent echoes it back as `cursor` to continue. + const countSuffix = result.totalCount != null ? ` of ${result.totalCount}` : '' + const message = result.nextCursor + ? `Returned ${result.rows.length}${countSuffix} rows (more available — pass cursor=${result.nextCursor} to continue)` + : `Returned ${result.rows.length}${countSuffix} rows` return { success: true, - message: `Returned ${result.rows.length} of ${result.totalCount} rows`, + message, data: { ...result, rows: result.rows.map((r) => ({ @@ -822,10 +861,11 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - const idFilter = resolveFilterSelectValues( - filterNamesToIds(args.filter, idByName), - table.schema.columns - ) + // Agent authors a predicate object; validate → translate → Filter for + // the bulk engine (same fieldPredicate leaf → identical SQL). Select + // operands arrive as option NAMES and must resolve to stored ids. + validatePredicate(args.filter, table.schema.columns) + const idFilter = predicateToFilter(predicateToStorage(args.filter, table.schema)) const idData = rowDataNameToId(args.data, idByName) // Inline handles up to MAX_BULK_OPERATION_SIZE rows in one request; a larger operation @@ -923,10 +963,11 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - const idFilter = resolveFilterSelectValues( - filterNamesToIds(args.filter, idByName), - table.schema.columns - ) + // Agent authors a predicate object; validate → translate → Filter for + // the bulk engine (same fieldPredicate leaf → identical SQL). Select + // operands arrive as option NAMES and must resolve to stored ids. + validatePredicate(args.filter, table.schema.columns) + const idFilter = predicateToFilter(predicateToStorage(args.filter, table.schema)) // Inline handles up to MAX_BULK_OPERATION_SIZE rows; a larger delete (an explicit limit // above the cap, or unbounded "delete everything matching") hands off to the background diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index ab6a5de43f1..8c28ba643f8 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -106,7 +106,7 @@ export const env = createEnv({ ENTERPRISE_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on enterprise tier (default: 10000) ENTERPRISE_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on enterprise tier (default: 1000000) TABLE_MAX_ROW_SIZE_BYTES: z.number().optional(), // Max serialized size in bytes of a single user-table row (default: 409600) - TABLE_MAX_PAGE_BYTES: z.number().optional(), // Dev-preview: byte budget per row-page read; pages cut early past it (unset = disabled) + TABLE_MAX_PAGE_BYTES: z.number().optional(), // Byte budget per row-page read; pages cut early past it (unset = disabled) TABLE_DISPATCH_CONCURRENCY_FREE: z.number().optional(), // Rows one table run executes in parallel on free tier (default: 20) TABLE_DISPATCH_CONCURRENCY_PAID: z.number().optional(), // Rows one table run executes in parallel on paid tiers (default: 50) @@ -481,6 +481,7 @@ export const env = createEnv({ SESSION_POLICIES_ENABLED: z.boolean().optional(), // Enable org session policies on self-hosted (bypasses hosted requirements) FORKING_ENABLED: z.boolean().optional(), // Enable workspace forking on self-hosted (bypasses hosted requirements) DEPLOY_AS_BLOCK: z.boolean().optional(), // Enable deploy-as-block (publish a workflow as a reusable org-wide custom block) + TABLES_V2_API: z.boolean().optional(), // Enable the v2 tables HTTP API (public /api/v2/tables + internal /api/table/[tableId]/query predicate-grammar route) TABLE_LOCKS: z.boolean().optional(), // Enable per-table mutation locks (schema/insert/update/delete toggles) TABLE_VIEWS: z.boolean().optional(), // Enable saved table views (named filter/sort/column-visibility presets) and the column show/hide menu diff --git a/apps/sim/lib/core/config/feature-flags.test.ts b/apps/sim/lib/core/config/feature-flags.test.ts index ebf9e70099b..7f0d0dbea50 100644 --- a/apps/sim/lib/core/config/feature-flags.test.ts +++ b/apps/sim/lib/core/config/feature-flags.test.ts @@ -13,6 +13,7 @@ const { mockFetch, mockIsPlatformAdmin, envRef } = vi.hoisted(() => ({ APPCONFIG_ENVIRONMENT: 'staging' as string | undefined, FORKING_ENABLED: undefined as boolean | undefined, DEPLOY_AS_BLOCK: undefined as boolean | undefined, + TABLES_V2_API: undefined as boolean | undefined, }, })) @@ -245,3 +246,29 @@ describe('isFeatureEnabled', () => { }) }) }) + +describe('tables-v2-api flag', () => { + beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isAppConfigEnabled: false }) + envRef.TABLES_V2_API = undefined + }) + + it('is off by default off-AppConfig, on when the fallback secret is set', async () => { + expect(await isFeatureEnabled('tables-v2-api')).toBe(false) + envRef.TABLES_V2_API = true + expect(await isFeatureEnabled('tables-v2-api')).toBe(true) + }) + + it('gates by org cohort via AppConfig', async () => { + withAppConfig({ 'tables-v2-api': { orgIds: ['org-1'] } }) + expect(await isFeatureEnabled('tables-v2-api', { orgId: 'org-1' })).toBe(true) + expect(await isFeatureEnabled('tables-v2-api', { orgId: 'org-2' })).toBe(false) + expect(await isFeatureEnabled('tables-v2-api', { userId: 'u1' })).toBe(false) + }) + + it('global enabled turns it on for everyone', async () => { + withAppConfig({ 'tables-v2-api': { enabled: true } }) + expect(await isFeatureEnabled('tables-v2-api')).toBe(true) + }) +}) diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index 167b1e50b6e..2989f04bcf3 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -104,6 +104,14 @@ const FEATURE_FLAGS = { 'custom-block publish/list routes. Off-AppConfig falls back to DEPLOY_AS_BLOCK.', fallback: 'DEPLOY_AS_BLOCK', }, + 'tables-v2-api': { + description: + 'Gate the v2 tables HTTP API — the public read API (GET /api/v2/tables, POST ' + + '/api/v2/tables/[tableId]/query) and the internal predicate-grammar query route (POST ' + + '/api/table/[tableId]/query). When off, those routes return 404 as if the surface does not ' + + 'exist. Gated by userId/orgId/admins via AppConfig; off-AppConfig falls back to TABLES_V2_API.', + fallback: 'TABLES_V2_API', + }, 'table-locks': { description: 'Per-table mutation locks (schema/insert/update/delete) an admin toggles to make a table ' + diff --git a/apps/sim/lib/table/__tests__/paging.test.ts b/apps/sim/lib/table/__tests__/paging.test.ts deleted file mode 100644 index 22a4a55bcf5..00000000000 --- a/apps/sim/lib/table/__tests__/paging.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @vitest-environment node - */ -import { describe, expect, it } from 'vitest' -import { trimRowsToByteBudget } from '@/lib/table/rows/paging' - -function row(id: string, bytes: number) { - // {"b":"aaa…"} serializes to bytes + 8 chars of envelope. - return { id, data: { b: 'a'.repeat(Math.max(0, bytes - 8)) } } -} - -describe('trimRowsToByteBudget', () => { - it('returns all rows when they fit the budget', () => { - const rows = [row('r1', 100), row('r2', 100)] - expect(trimRowsToByteBudget(rows, 1000)).toBe(rows) - }) - - it('keeps the longest prefix within the budget', () => { - const rows = [row('r1', 400), row('r2', 400), row('r3', 400)] - const kept = trimRowsToByteBudget(rows, 900) - expect(kept.map((r) => r.id)).toEqual(['r1', 'r2']) - }) - - it('always keeps the first row even when it alone exceeds the budget', () => { - const rows = [row('r1', 5000), row('r2', 100)] - const kept = trimRowsToByteBudget(rows, 1000) - expect(kept.map((r) => r.id)).toEqual(['r1']) - }) - - it('returns empty input unchanged', () => { - expect(trimRowsToByteBudget([], 1000)).toEqual([]) - }) -}) diff --git a/apps/sim/lib/table/__tests__/select-values.test.ts b/apps/sim/lib/table/__tests__/select-values.test.ts new file mode 100644 index 00000000000..32c0891e30c --- /dev/null +++ b/apps/sim/lib/table/__tests__/select-values.test.ts @@ -0,0 +1,128 @@ +/** + * @vitest-environment node + * + * Select-column operand resolution. A select cell stores option IDs, so a filter + * written with the option NAME must be rewritten before it reaches SQL — + * otherwise it compares a name against an id and matches nothing while reporting + * success. Both wire grammars have to do this identically. + */ +import { describe, expect, it } from 'vitest' +import { resolveFilterSelectValues, resolvePredicateSelectValues } from '@/lib/table/select-values' +import type { ColumnDefinition } from '@/lib/table/types' + +const MULTI: ColumnDefinition = { + id: 'col_color', + name: 'Color', + type: 'select', + multiple: true, + options: [ + { id: 'opt_teal', name: 'Teal' }, + { id: 'opt_green', name: 'Green' }, + ], +} +const SINGLE: ColumnDefinition = { + id: 'col_status', + name: 'Status', + type: 'select', + options: [{ id: 'opt_open', name: 'Open' }], +} +const PLAIN: ColumnDefinition = { id: 'col_name', name: 'name', type: 'string' } +const COLS = [MULTI, SINGLE, PLAIN] + +const leaf = (p: unknown) => (p as { all: Array<{ value: unknown }> }).all[0] + +describe('resolvePredicateSelectValues', () => { + /** + * Regression: `contains`/`ncontains` were excluded as "pattern ops". On a + * multi-select they are not pattern ops — the cell is an array of ids and they + * express membership. Mothership sent exactly this and silently got zero rows. + */ + it('resolves contains / ncontains on a MULTI-select (membership, not pattern)', () => { + for (const op of ['contains', 'ncontains'] as const) { + const out = resolvePredicateSelectValues( + { all: [{ field: 'col_color', op, value: 'Teal' }] }, + COLS + ) + expect(leaf(out).value).toBe('opt_teal') + } + }) + + it('resolves eq / ne / in / nin on a single select', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_status', op: 'eq', value: 'Open' }] }, + COLS + ) + ).value + ).toBe('opt_open') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_status', op: 'in', value: ['Open'] }] }, + COLS + ) + ).value + ).toEqual(['opt_open']) + }) + + it('matches option names case-insensitively and passes ids through', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'opt_teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + }) + + it('leaves non-select columns and unknown option names alone', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_name', op: 'contains', value: 'Teal' }] }, + COLS + ) + ).value + ).toBe('Teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'Nope' }] }, + COLS + ) + ).value + ).toBe('Nope') + }) + + it('recurses through nested groups', () => { + const out = resolvePredicateSelectValues( + { any: [{ all: [{ field: 'col_color', op: 'contains', value: 'Green' }] }] }, + COLS + ) as { any: Array<{ all: Array<{ value: unknown }> }> } + expect(out.any[0].all[0].value).toBe('opt_green') + }) + + /** The two grammars must resolve the same operand set, or they drift. */ + it('agrees with the $-grammar sibling on the same filter', () => { + const legacy = resolveFilterSelectValues({ col_color: { $contains: 'Teal' } }, COLS) + expect((legacy.col_color as { $contains: unknown }).$contains).toBe('opt_teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'Teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + }) +}) diff --git a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts index 582df6cd2c3..889e75de2d8 100644 --- a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts +++ b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts @@ -7,15 +7,19 @@ * timestamp for dates) are always available at the SQL builder layer — the * latent bug that PR #4657 was originally fixing. */ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, resetDbChainMock, setEnv } from '@sim/testing' import { sql } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { TABLE_LIMITS } from '@/lib/table/constants' +import { decodeCursor } from '@/lib/table/rows/cursor' import { buildFilterClause, buildSortClause } from '@/lib/table/sql' import type { ColumnDefinition, TableDefinition } from '@/lib/table/types' vi.mock('@/lib/table/sql', () => ({ buildFilterClause: vi.fn(() => sql`true`), buildSortClause: vi.fn(() => sql`true`), + buildPredicateClause: vi.fn(() => sql`true`), + TableQueryValidationError: class TableQueryValidationError extends Error {}, })) vi.mock('@/lib/table/trigger', () => ({ @@ -122,3 +126,316 @@ describe('service filter threading', () => { ) }) }) + +describe('bulk update/delete limited-subset ordering', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('orders the match query when updateRowsByFilter has a limit', async () => { + await updateRowsByFilter( + TABLE, + { filter: { score: { $gt: 0 } }, data: { name: 'x' }, limit: 5 }, + 'req-1' + ) + expect(dbChainMockFns.orderBy).toHaveBeenCalled() + expect(dbChainMockFns.limit).toHaveBeenCalledWith(5) + }) + + it('does not order an unbounded updateRowsByFilter', async () => { + dbChainMockFns.where.mockResolvedValueOnce([]) + await updateRowsByFilter(TABLE, { filter: { score: { $gt: 0 } }, data: { name: 'x' } }, 'req-1') + expect(dbChainMockFns.orderBy).not.toHaveBeenCalled() + }) + + it('orders the match query when deleteRowsByFilter has a limit', async () => { + await deleteRowsByFilter(TABLE, { filter: { score: { $gt: 0 } }, limit: 3 }, 'req-1') + expect(dbChainMockFns.orderBy).toHaveBeenCalled() + expect(dbChainMockFns.limit).toHaveBeenCalledWith(3) + }) +}) + +describe('queryRows byte budget', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + // The bounded-page byte cut is opt-in; pin it on rather than inheriting + // whatever the developer's local `.env` happens to set. + setEnv({ TABLE_MAX_PAGE_BYTES: TABLE_LIMITS.MAX_QUERY_RESULT_BYTES }) + }) + + const row = (i: number, blobBytes: number) => ({ + id: `row_${i}`, + data: { blob: 'x'.repeat(blobBytes) }, + position: i, + orderKey: `a${i}`, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + }) + + it('returns an empty page with a null cursor', async () => { + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + expect(result.rows).toEqual([]) + expect(result.nextCursor).toBeNull() + }) + + it('fails fast when an UNBOUNDED query exceeds the byte budget (no partial page)', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + // First .limit() call is pendingDeleteMask's job probe; the second is the drain batch. + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + await expect( + queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + ).rejects.toThrow(/exceeds the 5MB limit/) + }) + + it('returns the entire result when an unbounded query fits the budget', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8)]) + + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + + expect(result.rows).toHaveLength(2) + expect(result.nextCursor).toBeNull() + }) + + it('byte-cuts a BOUNDED page and returns a resume cursor instead of throwing', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0].id).toBe('row_1') + // Fractional ordering is on (global flag mock) → keyset cursor resuming + // after the last included row. + expect(decodeCursor(result.nextCursor as string)).toEqual({ + after: { orderKey: 'a1', id: 'row_1' }, + }) + }) + + it('does NOT byte-cut a bounded page when TABLE_MAX_PAGE_BYTES is unset', async () => { + // Default-off: a short page is only safe for a client that terminates on + // `nextCursor === null`. A pre-existing v1 pager stopping at + // `rows.length < limit` would read the cut as end-of-data and truncate. + setEnv({ TABLE_MAX_PAGE_BYTES: undefined }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(2) + expect(result.nextCursor).toBeNull() + }) + + it('still fails fast on an UNBOUNDED query with TABLE_MAX_PAGE_BYTES unset', async () => { + setEnv({ TABLE_MAX_PAGE_BYTES: undefined }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + await expect( + queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + ).rejects.toThrow(/exceeds the 5MB limit/) + }) + + it('returns a single over-budget row alone on a bounded page', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([ + row(1, TABLE_LIMITS.MAX_QUERY_RESULT_BYTES + 1024), + row(2, 8), + ]) + + const result = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0].id).toBe('row_1') + expect(result.nextCursor).not.toBeNull() + }) + + // First-batch worst-case bytes stay within ~4× the budget at the max row size. + const FIRST_BATCH_CAP = Math.max( + 1, + Math.floor((4 * TABLE_LIMITS.MAX_QUERY_RESULT_BYTES) / TABLE_LIMITS.MAX_ROW_SIZE_BYTES) + ) + + /** + * The seek predicate that pages the drain. `order_key` is nullable (rows + * predating the backfill, forked rows), and a bare + * `(order_key, id) > (:k, :i)` evaluates to NULL for those rows, so WHERE drops + * them. Because NULLs sort last, the whole unkeyed tail then becomes + * unreachable and the drain reports `hasMore: false` — 52 of 121 rows returned + * with no error, reproduced against a real table. + */ + it('seeks with a NULL-admitting comparison so unkeyed rows stay reachable', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([]) + + await queryRows( + TABLE, + { + after: { orderKey: 'a5', id: 'row_5' }, + limit: 10, + includeTotal: false, + withExecutions: false, + }, + 'req-1' + ) + + const seekWhere = JSON.stringify(dbChainMockFns.where.mock.calls.at(-1)) + expect(seekWhere).toMatch(/is null/i) + expect(seekWhere).toContain('a5') + expect(seekWhere).toContain('row_5') + }) + + it('resumes exactly at the last returned row when the cursor is fed back', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + // limit 2 + witness row_3 → page ends at row_2 with a resume cursor. + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8), row(3, 8)]) + + const page1 = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(page1.rows.map((r) => r.id)).toEqual(['row_1', 'row_2']) + + const cursor = decodeCursor(page1.nextCursor as string) + expect(cursor).toEqual({ after: { orderKey: 'a2', id: 'row_2' } }) + + vi.clearAllMocks() + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(3, 8), row(4, 8)]) + + const page2 = await queryRows( + TABLE, + { ...cursor, limit: 2, includeTotal: false, withExecutions: false }, + 'req-2' + ) + + // No gap (row_3 is the first row past the anchor) and no duplicate of row_2. + expect(page2.rows.map((r) => r.id)).toEqual(['row_3', 'row_4']) + const seekWhere = JSON.stringify(dbChainMockFns.where.mock.calls.at(-1)) + expect(seekWhere).toContain('a2') + expect(seekWhere).toContain('row_2') + }) + + it('caps the first unbounded batch and asks limit+1 when a limit is set', async () => { + await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + // Call 1 = pendingDeleteMask probe (limit 1); call 2 = first drain batch. + expect(dbChainMockFns.limit).toHaveBeenNthCalledWith(2, FIRST_BATCH_CAP + 1) + + vi.clearAllMocks() + resetDbChainMock() + await queryRows(TABLE, { limit: 5, includeTotal: false, withExecutions: false }, 'req-1') + expect(dbChainMockFns.limit).toHaveBeenNthCalledWith(2, 6) + }) + + it('limit-cut with a witness row emits a cursor; exact-boundary page does not', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8), row(3, 8)]) + const cut = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(cut.rows.map((r) => r.id)).toEqual(['row_1', 'row_2']) + expect(cut.nextCursor).not.toBeNull() + + vi.clearAllMocks() + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8)]) + const exact = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(exact.rows).toHaveLength(2) + // The +1 peek proved the data ended exactly at the limit — no dead cursor. + expect(exact.nextCursor).toBeNull() + }) + + it('drains across multiple batches and grows the batch size adaptively', async () => { + const firstAsk = FIRST_BATCH_CAP + 1 + const batch1 = Array.from({ length: firstAsk }, (_, i) => row(i, 8)) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce(batch1) + dbChainMockFns.limit.mockResolvedValueOnce([row(firstAsk, 8)]) + + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + + expect(result.rows).toHaveLength(firstAsk + 1) + expect(result.nextCursor).toBeNull() + const firstBatchAsk = (dbChainMockFns.limit.mock.calls[1] ?? [])[0] as number + const secondBatchAsk = (dbChainMockFns.limit.mock.calls[2] ?? [])[0] as number + expect(secondBatchAsk).toBeGreaterThan(firstBatchAsk) + }) + + it('sort path advances by offset and emits an offset cursor with inbound offset applied', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + // Sorted batch with inbound offset 40 terminates at .offset(40). + dbChainMockFns.offset.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { sort: { name: 'asc' }, offset: 40, limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(dbChainMockFns.offset).toHaveBeenCalledWith(40) + expect(result.rows).toHaveLength(1) + // The offset cursor is stamped with the sort it was minted under, so a + // replay against a different ordering is rejected instead of paging wrong. + expect(decodeCursor(result.nextCursor as string)).toEqual({ + offset: 41, + sortKey: JSON.stringify([['name', 'asc']]), + }) + }) + + it('emits a compound cursor when rows past the inbound anchor are unkeyed', async () => { + const unkeyedRow = (i: number, blobBytes: number) => ({ ...row(i, blobBytes), orderKey: null }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.4) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([ + unkeyedRow(6, perRow), + unkeyedRow(7, perRow), + unkeyedRow(8, perRow), + ]) + + const result = await queryRows( + TABLE, + { + after: { orderKey: 'a5', id: 'row_5' }, + limit: 10, + includeTotal: false, + withExecutions: false, + }, + 'req-1' + ) + + expect(result.rows.map((r) => r.id)).toEqual(['row_6', 'row_7']) + expect(decodeCursor(result.nextCursor as string)).toEqual({ + after: { orderKey: 'a5', id: 'row_5' }, + offset: 2, + }) + }) +}) diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index 367a9eb1704..dcca281da94 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -20,11 +20,13 @@ import { } from '@/lib/table/query-builder/constants' import { buildFilterClause, + buildPredicateClause, buildSortClause, + fieldPredicate, MULTI_SELECT_OPERATORS, SINGLE_SELECT_OPERATORS, } from '@/lib/table/sql' -import type { ColumnDefinition, Filter, Sort } from '@/lib/table/types' +import type { ColumnDefinition, Filter, Sort, TablePredicate } from '@/lib/table/types' type SqlNode = | { strings: ArrayLike; values: unknown[] } @@ -85,6 +87,11 @@ function render(node: unknown): string { return renderSql(node) } +/** fieldPredicate takes a ColumnDefinition; these tests only care about its type. */ +function col(type: ColumnDefinition['type'], name = 'c'): ColumnDefinition { + return { name, type } +} + const TABLE = 'user_table_rows' const NO_COLUMNS: ColumnDefinition[] = [] @@ -400,12 +407,12 @@ describe('SQL Builder', () => { expect(out).toBe(`(${TABLE}.data->>'birthDate')::timestamptz ASC NULLS LAST`) }) - it('sorts createdAt / updatedAt as direct column refs', () => { + it('sorts createdAt / updatedAt as direct snake_case column refs', () => { expect(render(buildSortClause({ createdAt: 'desc' }, TABLE, NO_COLUMNS))).toBe( - `${TABLE}.createdAt DESC` + `${TABLE}.created_at DESC` ) expect(render(buildSortClause({ updatedAt: 'asc' }, TABLE, NO_COLUMNS))).toBe( - `${TABLE}.updatedAt ASC` + `${TABLE}.updated_at ASC` ) }) @@ -592,3 +599,363 @@ describe('SQL Builder', () => { }) }) }) + +describe('fieldPredicate (shared leaf)', () => { + const r = ( + op: Parameters[2], + value: unknown, + colType?: ColumnDefinition['type'] + ) => render(fieldPredicate(TABLE, 'wins', op, value as never, colType && col(colType, 'wins'))) + + it('eq emits case-sensitive JSONB containment (no lower())', () => { + const out = render(fieldPredicate(TABLE, 'slack_user_id', 'eq', 'U333', undefined)) + expect(out).toContain('user_table_rows.data @>') + expect(out).toContain('"slack_user_id":"U333"') + expect(out).not.toContain('lower(') + // Case is preserved verbatim — U333 and u333 are distinct values. + expect(out).not.toContain('u333') + }) + + it('ne negates the containment clause', () => { + expect(r('ne', 'x')).toContain('NOT (') + expect(r('ne', 'x')).toContain('data @>') + }) + + it('in with one value is a single containment; many values OR together', () => { + expect(render(fieldPredicate(TABLE, 'slack_user_id', 'in', ['U1'], undefined))).toContain( + '"slack_user_id":"U1"' + ) + const many = render(fieldPredicate(TABLE, 'slack_user_id', 'in', ['U1', 'U2'], undefined)) + expect(many).toContain('"slack_user_id":"U1"') + expect(many).toContain('"slack_user_id":"U2"') + expect(many).toContain(' OR ') + }) + + it('nin ANDs negated containments', () => { + const out = render(fieldPredicate(TABLE, 'slack_user_id', 'nin', ['U1', 'U2'], undefined)) + expect(out).toContain('NOT (') + expect(out).toContain(' AND ') + }) + + it('empty in/nin arrays are a no-op (undefined)', () => { + expect(fieldPredicate(TABLE, 'wins', 'in', [], undefined)).toBeUndefined() + expect(fieldPredicate(TABLE, 'wins', 'nin', [], undefined)).toBeUndefined() + }) + + it('range ops cast by column type', () => { + expect(r('gte', 10, 'number')).toContain('::numeric') + expect(r('gt', '2024-01-01', 'date')).toContain('::timestamptz') + }) + + it('text ops use case-insensitive ILIKE', () => { + expect(render(fieldPredicate(TABLE, 'name', 'contains', 'jo', undefined))).toContain('ILIKE') + expect(render(fieldPredicate(TABLE, 'name', 'startsWith', 'jo', undefined))).toContain('ILIKE') + }) + + it('isEmpty / isNotEmpty emit emptiness checks (null OR empty string)', () => { + const empty = render(fieldPredicate(TABLE, 'name', 'isEmpty', undefined, undefined)) + expect(empty).toContain('IS NULL') + expect(empty).toContain("= ''") + const notEmpty = render(fieldPredicate(TABLE, 'name', 'isNotEmpty', undefined, undefined)) + expect(notEmpty).toContain('IS NOT NULL') + }) + + it('isNull / isNotNull are strict null checks (no empty-string clause)', () => { + const isNull = render(fieldPredicate(TABLE, 'name', 'isNull', undefined, undefined)) + expect(isNull).toContain('IS NULL') + expect(isNull).not.toContain("= ''") + expect(render(fieldPredicate(TABLE, 'name', 'isNotNull', undefined, undefined))).toContain( + 'IS NOT NULL' + ) + }) + + it('like / ilike map * to % and escape literal % / _', () => { + const like = render(fieldPredicate(TABLE, 'name', 'like', 'jo*n', undefined)) + expect(like).toContain("data->>'name'") + expect(like).toContain('LIKE') + expect(like).not.toContain('ILIKE') + expect(like).toContain('jo%n') + expect(render(fieldPredicate(TABLE, 'name', 'ilike', '*foo*', undefined))).toContain('ILIKE') + // literal % is escaped to match itself, not act as a wildcard + expect(render(fieldPredicate(TABLE, 'name', 'like', '50%*', undefined))).toContain('50\\%%') + }) + + it('nlike / nilike negate the match and keep null cells', () => { + // "does not match X" must retain rows where the cell is absent — otherwise a + // negated filter silently drops every row with an empty value for that column. + const nlike = render(fieldPredicate(TABLE, 'name', 'nlike', 'jo*n', undefined)) + expect(nlike).toContain('NOT LIKE') + expect(nlike).not.toContain('NOT ILIKE') + expect(nlike).toContain('jo%n') + expect(nlike).toContain('IS NULL') + + const nilike = render(fieldPredicate(TABLE, 'name', 'nilike', '*foo*', undefined)) + expect(nilike).toContain('NOT ILIKE') + expect(nilike).toContain('IS NULL') + }) + + it('reaches nlike / nilike through the legacy $-grammar too', () => { + expect(render(buildFilterClause({ name: { $nlike: 'jo*' } }, TABLE, NO_COLUMNS))).toContain( + 'NOT LIKE' + ) + expect(render(buildFilterClause({ name: { $nilike: 'jo*' } }, TABLE, NO_COLUMNS))).toContain( + 'NOT ILIKE' + ) + }) + + it('no longer accepts the regex ops (removed from FILTER_OPS)', () => { + for (const op of ['match', 'imatch'] as const) { + expect(() => fieldPredicate(TABLE, 'name', op as never, '^jo', undefined)).toThrow( + 'Invalid operator' + ) + } + expect(() => buildFilterClause({ name: { $match: '^jo' } }, TABLE, NO_COLUMNS)).toThrow( + 'Invalid operator' + ) + }) + + it('validates the field name', () => { + expect(() => fieldPredicate(TABLE, "x'; DROP", 'eq', 1, undefined)).toThrow( + 'Invalid field name' + ) + }) + + it('rejects an unknown operator', () => { + expect(() => fieldPredicate(TABLE, 'wins', 'bogus' as never, 1, undefined)).toThrow( + 'Invalid operator' + ) + }) + + it('filters createdAt / updatedAt as real timestamptz columns, not JSONB keys', () => { + const gte = render(fieldPredicate(TABLE, 'createdAt', 'gte', '2026-01-01', col('date'))) + expect(gte).toContain(`${TABLE}.created_at`) + expect(gte).toContain('::timestamptz') + expect(gte).not.toContain("data->>'createdAt'") + + const lt = render(fieldPredicate(TABLE, 'updatedAt', 'lt', '2026-06-01', col('date'))) + expect(lt).toContain(`${TABLE}.updated_at`) + expect(lt).toContain('::timestamptz') + }) + + it('supports in / isNull on system columns', () => { + expect(render(fieldPredicate(TABLE, 'createdAt', 'isNull', undefined, col('date')))).toContain( + `${TABLE}.created_at IS NULL` + ) + const inClause = render( + fieldPredicate(TABLE, 'createdAt', 'in', ['2026-01-01', '2026-02-01'], col('date')) + ) + expect(inClause).toContain(`${TABLE}.created_at`) + expect(inClause).toContain('::timestamptz') + }) +}) + +/** + * Regression coverage for #5920 — "filtering on built-in columns silently + * returns zero rows". Three distinct defects fed that report: the missing + * system-column dispatch (fixed above), `id` never being a system column at + * all, and the session-dependent `timestamptz` promotion that shifts every + * bound by the server's `TimeZone` GUC. + */ +describe('system columns (#5920)', () => { + it('normalizes timestamp bounds to UTC wall clock, not the session TimeZone', () => { + // `created_at`/`updated_at` are `timestamp WITHOUT time zone` holding UTC. + // Without `AT TIME ZONE 'UTC'` the comparison result depends on the server's + // TimeZone setting, so a UTC-3 day range (the reporter's exact query) lands + // off by the offset at both boundaries. + for (const field of ['createdAt', 'updatedAt'] as const) { + const out = render( + fieldPredicate(TABLE, field, 'gte', '2026-07-24T03:00:00.000Z', col('date')) + ) + expect(out).toContain("::timestamptz AT TIME ZONE 'UTC'") + } + }) + + it('filters id as the real text column, with no timestamptz cast', () => { + const out = render(fieldPredicate(TABLE, 'id', 'eq', 'row-123', col('string'))) + expect(out).toContain(`${TABLE}.id =`) + expect(out).not.toContain("data->>'id'") + expect(out).not.toContain('timestamptz') + }) + + it('supports the pattern ops on id (text), which are meaningless on timestamps', () => { + expect(render(fieldPredicate(TABLE, 'id', 'contains', 'abc', col('string')))).toContain( + `${TABLE}.id ILIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'startsWith', 'abc', col('string')))).toContain( + `${TABLE}.id ILIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'like', 'ab*', col('string')))).toContain( + `${TABLE}.id LIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'ncontains', 'abc', col('string')))).toContain( + 'NOT (' + ) + + expect(() => fieldPredicate(TABLE, 'createdAt', 'contains', 'abc', col('date'))).toThrow( + /not supported on the built-in column "createdAt"/ + ) + }) + + it('rejects an empty pattern on id rather than matching every row', () => { + expect(() => fieldPredicate(TABLE, 'id', 'contains', '', col('string'))).toThrow( + /requires a non-empty value/ + ) + }) + + it('supports id in ranges and membership', () => { + expect(render(fieldPredicate(TABLE, 'id', 'gt', 'row-100', col('string')))).toContain( + `${TABLE}.id >` + ) + const inClause = render(fieldPredicate(TABLE, 'id', 'in', ['a', 'b'], col('string'))) + expect(inClause).toContain(`${TABLE}.id IN (`) + }) + + it('sorts id as a direct column ref', () => { + expect(render(buildSortClause({ id: 'asc' }, TABLE, NO_COLUMNS))).toBe(`${TABLE}.id ASC`) + }) + + it('maps isEmpty/isNotEmpty to IS NULL / IS NOT NULL (not inverted)', () => { + expect(render(fieldPredicate(TABLE, 'createdAt', 'isEmpty', undefined, col('date')))).toContain( + 'IS NULL' + ) + expect( + render(fieldPredicate(TABLE, 'createdAt', 'isNotEmpty', undefined, col('date'))) + ).toContain('IS NOT NULL') + }) + + it('reaches the same clause through the legacy $-grammar', () => { + // The v1 API path in the bug report goes through buildFilterClause, so the + // shared `fieldPredicate` leaf must cover it identically. + const out = render( + buildFilterClause( + { createdAt: { $gte: '2026-07-24T03:00:00.000Z', $lte: '2026-07-25T02:59:59.999Z' } }, + TABLE, + NO_COLUMNS + ) + ) + expect(out).toContain(`${TABLE}.created_at >=`) + expect(out).toContain(`${TABLE}.created_at <=`) + expect(out).toContain("AT TIME ZONE 'UTC'") + expect(out).not.toContain('requires a number') + }) +}) + +describe('range operators on non-numeric column types', () => { + it('compares string columns lexicographically as text (no numeric cast)', () => { + const cols: ColumnDefinition[] = [{ name: 'name', type: 'string' }] + const out = render(buildFilterClause({ name: { $gte: 'M' } }, TABLE, cols)) + expect(out).toContain(`${TABLE}.data->>'name' >=`) + expect(out).not.toContain('::numeric') + }) + + it('rejects ranges on boolean / json columns with a type-naming message', () => { + for (const type of ['boolean', 'json'] as const) { + const cols: ColumnDefinition[] = [{ name: 'flag', type }] + expect(() => buildFilterClause({ flag: { $gt: 1 } }, TABLE, cols)).toThrow( + new RegExp(`\\(${type}\\) is not supported`) + ) + } + }) +}) + +describe('buildPredicateClause (v2 grammar)', () => { + it('all joins members with AND', () => { + const p: TablePredicate = { + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [{ name: 'wins', type: 'number' }])) + expect(out).toContain(' AND ') + expect(out).toContain('"slack_user_id":"U1"') + expect(out).toContain('::numeric') + }) + + it('any joins members with OR', () => { + const p: TablePredicate = { + any: [ + { field: 'status', op: 'eq', value: 'active' }, + { field: 'status', op: 'eq', value: 'pending' }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [])) + expect(out).toContain(' OR ') + expect(out).toContain('"status":"active"') + expect(out).toContain('"status":"pending"') + }) + + it('nests groups', () => { + const p: TablePredicate = { + all: [ + { field: 'wins', op: 'gte', value: 1 }, + { + any: [ + { field: 's', op: 'eq', value: 'a' }, + { field: 's', op: 'eq', value: 'b' }, + ], + }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [])) + expect(out).toContain(' AND ') + expect(out).toContain(' OR ') + }) + + it('an empty group is a no-op (undefined)', () => { + expect(buildPredicateClause({ all: [] }, TABLE, [])).toBeUndefined() + expect(buildPredicateClause({ any: [] }, TABLE, [])).toBeUndefined() + }) + + it('validates leaf field names', () => { + const p: TablePredicate = { all: [{ field: 'bad name', op: 'eq', value: 1 }] } + expect(() => buildPredicateClause(p, TABLE, [])).toThrow('Invalid field name') + }) +}) + +/** + * Cross-version safety. If a client speaking the v2 predicate grammar reaches a + * server that predates it, the predicate arrives at the LEGACY `$`-compiler as + * `{ all: [...] }`. That used to be skipped as "an array on a regular field", + * compiling to no WHERE clause — which on a bulk delete means every row rather + * than none. The guard turns that into a loud, self-describing failure, and it + * sits at the one choke point every filter path shares (`queryRows`, + * `update-runner`, `delete-runner`, inline and background). + */ +describe('legacy compiler rejects a v2 predicate (version-mismatch fail-fast)', () => { + it('throws on a top-level all/any group instead of emitting no clause', () => { + for (const group of ['all', 'any'] as const) { + expect(() => + buildFilterClause( + { [group]: [{ field: 'tenant_id', op: 'eq', value: 'acme' }] } as unknown as Filter, + TABLE, + NO_COLUMNS + ) + ).toThrow(/v2 predicate tree/) + } + }) + + it('catches one nested inside a legacy $or', () => { + expect(() => + buildFilterClause( + { + $or: [{ status: 'a' }, { all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }] }], + } as unknown as Filter, + TABLE, + NO_COLUMNS + ) + ).toThrow(/v2 predicate tree/) + }) + + it('leaves legitimate legacy filters alone', () => { + expect(buildFilterClause({ status: 'archived' }, TABLE, NO_COLUMNS)).toBeDefined() + expect( + buildFilterClause({ $or: [{ status: 'a' }, { status: 'b' }] }, TABLE, NO_COLUMNS) + ).toBeDefined() + // An ordinary column holding an array stays a silent skip — only the + // predicate discriminators `all`/`any` are treated as a version mismatch. + expect(() => + buildFilterClause({ status: ['a', 'b'] } as unknown as Filter, TABLE, NO_COLUMNS) + ).not.toThrow() + }) +}) diff --git a/apps/sim/lib/table/__tests__/validation.test.ts b/apps/sim/lib/table/__tests__/validation.test.ts index bcb9a224ab8..f6b8be7dc36 100644 --- a/apps/sim/lib/table/__tests__/validation.test.ts +++ b/apps/sim/lib/table/__tests__/validation.test.ts @@ -527,10 +527,12 @@ describe('Validation', () => { expect(result.errors[0]).toContain('abc123') }) - it('should be case-insensitive for string comparisons', () => { + it('should be case-sensitive for string comparisons', () => { + // U333 vs u333: differing case is a DISTINCT value (matches the DB + // containment leaf). This is the v2 contract that fixes the upsert wedge. const data = { id: 'ABC123', email: 'new@example.com', name: 'New User' } const result = validateUniqueConstraints(data, schema, existingRows) - expect(result.valid).toBe(false) + expect(result.valid).toBe(true) }) it('should exclude specified row from checks (for updates)', () => { diff --git a/apps/sim/lib/table/column-keys.ts b/apps/sim/lib/table/column-keys.ts index eb34f094861..71565101f4d 100644 --- a/apps/sim/lib/table/column-keys.ts +++ b/apps/sim/lib/table/column-keys.ts @@ -12,8 +12,11 @@ import { generateId } from '@sim/utils/id' import type { ColumnDefinition, Filter, + PredicateNode, RowData, Sort, + SortSpec, + TablePredicate, TableSchema, WorkflowGroup, } from '@/lib/table/types' @@ -171,6 +174,32 @@ export function sortNamesToIds(sort: Sort, idByName: ReadonlyMap return out } +/** + * Translates a v2 predicate's leaf `field` names → column ids (recursing through + * `all`/`any` groups). Fields with no matching column pass through unchanged. + * The v2 analogue of {@link filterNamesToIds}. + */ +export function predicateNamesToIds( + predicate: TablePredicate, + idByName: ReadonlyMap +): TablePredicate { + const remap = (node: PredicateNode): PredicateNode => { + // Group-first, matching isPredicateGroup/validateNode/buildPredicateNode. + if ('all' in node) return { all: node.all.map(remap) } + if ('any' in node) return { any: node.any.map(remap) } + return { ...node, field: idByName.get(node.field) ?? node.field } + } + return remap(predicate) as TablePredicate +} + +/** Translates a v2 sort spec's field names → column ids. Unknown fields pass through. */ +export function sortSpecNamesToIds( + sort: SortSpec, + idByName: ReadonlyMap +): SortSpec { + return sort.map((s) => ({ field: idByName.get(s.field) ?? s.field, direction: s.direction })) +} + // The outbound direction (stored id-keyed row → name-keyed) deliberately does // NOT live here: a `select` cell's value also needs translating, and keeping the // key half separately callable is what let boundaries translate the keys and diff --git a/apps/sim/lib/table/constants.ts b/apps/sim/lib/table/constants.ts index 5201815e8f3..1384d3dc181 100644 --- a/apps/sim/lib/table/constants.ts +++ b/apps/sim/lib/table/constants.ts @@ -15,6 +15,17 @@ export const TABLE_LIMITS = { MAX_DESCRIPTION_LENGTH: 500, DEFAULT_QUERY_LIMIT: 100, MAX_QUERY_LIMIT: 1000, + /** + * Byte budget for one query response. `queryRows` drains rows in bounded + * batches; an UNBOUNDED query (no limit) that outgrows the budget fails fast + * (the whole result was promised — a partial page would be silent truncation), + * while a BOUNDED page cuts early and returns the partial list with + * `nextCursor` set. At least one row is always returned on a bounded page. + * Measured against serialized row `data` only, not the full HTTP envelope. + */ + MAX_QUERY_RESULT_BYTES: 5 * 1024 * 1024, // 5MB + /** Hard row cap per internal fetch batch inside queryRows' bounded drain loop. */ + QUERY_BATCH_MAX_ROWS: 10000, /** Batch size for bulk update operations */ UPDATE_BATCH_SIZE: 100, /** Batch size for bulk delete operations */ @@ -62,10 +73,14 @@ export const DEFAULT_TABLE_PLAN_LIMITS = { } as const /** - * Byte budget for one page of row reads, or null when disabled (the default). - * Dev-preview of the byte-bounded pagination follow-up: set `TABLE_MAX_PAGE_BYTES` - * to cut pages early once their serialized row data exceeds the budget. The - * production version moves the cut into SQL — see the pagination-hardening plan. + * Byte budget at which a **bounded** page (one with an explicit `limit`) is cut + * short, or `null` when disabled — the default. Opt in with `TABLE_MAX_PAGE_BYTES`. + * + * Off by default because a short page is only safe for a client that terminates + * on `nextCursor === null`; a pre-existing v1 pager terminating on + * `rows.length < limit` would read the cut as end-of-data and silently truncate. + * Unbounded queries (no `limit`) are unaffected — they always fail fast at + * `TABLE_LIMITS.MAX_QUERY_RESULT_BYTES` rather than return a partial result. */ export function getMaxPageBytes(): number | null { const value = envNumber(env.TABLE_MAX_PAGE_BYTES, 0, { min: 0, integer: true }) @@ -153,6 +168,35 @@ export const COLUMN_TYPES = ['string', 'number', 'boolean', 'date', 'json', 'sel /** Maximum number of options a `select`/`multiselect` column may declare. */ export const MAX_SELECT_OPTIONS = 100 +/** + * The v2 filter operators, as a runtime tuple so both the `FilterOp` type and + * the boundary Zod enum derive from one source. Order is not significant. + */ +export const FILTER_OPS = [ + 'eq', + 'ne', + 'gt', + 'gte', + 'lt', + 'lte', + 'in', + 'nin', + 'contains', + 'ncontains', + 'startsWith', + 'endsWith', + 'like', + 'ilike', + 'nlike', + 'nilike', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +] as const + +export const SORT_DIRECTIONS = ['asc', 'desc'] as const + export const NAME_PATTERN = /^[a-z_][a-z0-9_]*$/i export const USER_TABLE_ROWS_SQL_NAME = 'user_table_rows' diff --git a/apps/sim/lib/table/delete-runner.ts b/apps/sim/lib/table/delete-runner.ts index a10a9b309fb..e2ff5e6773e 100644 --- a/apps/sim/lib/table/delete-runner.ts +++ b/apps/sim/lib/table/delete-runner.ts @@ -110,6 +110,11 @@ export async function runTableDelete(payload: TableDeletePayload): Promise const filterClause = filter ? buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, table.schema.columns) : undefined + // A filter that was SUPPLIED but compiles to no clause must never widen into + // "delete every row" — `and()` silently drops an undefined clause downstream. + // Mirrors the guard in update-runner and the inline deleteRowsByFilter path; + // an absent filter is still legitimate (delete-all is an explicit caller mode). + if (filter && !filterClause) throw new Error('Filter is required for bulk delete') const excluded = new Set(excludeRowIds ?? []) // Resume the persisted count: a retried attempt's earlier batches are already committed, diff --git a/apps/sim/lib/table/errors.ts b/apps/sim/lib/table/errors.ts new file mode 100644 index 00000000000..6b80b6a7260 --- /dev/null +++ b/apps/sim/lib/table/errors.ts @@ -0,0 +1,28 @@ +/** + * Stable, machine-readable codes for table query failures. SDKs and clients + * branch on these instead of string-matching human-facing messages. + */ +export type TableQueryErrorCode = + | 'TABLE_QUERY_RESULT_TOO_LARGE' + | 'INVALID_CURSOR' + | 'CURSOR_SORT_CONFLICT' + | 'INVALID_FILTER' + | 'INVALID_ORDER' + +/** + * Error thrown when caller-supplied filter or sort input is malformed. + * Routes should map this to HTTP 400 with the message preserved. + * + * Lives outside `sql.ts` so client-bundled modules (the block definitions pull + * in the query-builder converters) can reference it without dragging drizzle-orm + * into the browser chunk. + */ +export class TableQueryValidationError extends Error { + readonly code?: TableQueryErrorCode + + constructor(message: string, code?: TableQueryErrorCode) { + super(message) + this.name = 'TableQueryValidationError' + this.code = code + } +} diff --git a/apps/sim/lib/table/export-runner.ts b/apps/sim/lib/table/export-runner.ts index 8af3628194c..e8a9cf5fb54 100644 --- a/apps/sim/lib/table/export-runner.ts +++ b/apps/sim/lib/table/export-runner.ts @@ -80,7 +80,9 @@ export async function runTableExport(payload: TableExportPayload): Promise let exported = 0 let firstJsonRow = true - let after: { orderKey: string; id: string } | null = null + // `order_key` is nullable (rows predating the backfill), and the page query + // seeks NULLs explicitly — so the cursor has to carry a null too. + let after: { orderKey: string | null; id: string } | null = null while (true) { // Ownership gate before every page: a canceled job stops within one batch. const owns = await updateJobProgress(tableId, exported, jobId) diff --git a/apps/sim/lib/table/index.ts b/apps/sim/lib/table/index.ts index b4352ff2b21..fad60f65370 100644 --- a/apps/sim/lib/table/index.ts +++ b/apps/sim/lib/table/index.ts @@ -10,6 +10,7 @@ export * from '@/lib/table/column-keys' export * from '@/lib/table/columns/service' export * from '@/lib/table/constants' export * from '@/lib/table/dates' +export * from '@/lib/table/errors' export * from '@/lib/table/import' export * from '@/lib/table/import-data' export * from '@/lib/table/jobs/service' diff --git a/apps/sim/lib/table/jobs/service.ts b/apps/sim/lib/table/jobs/service.ts index 7b1706e32da..ca916e689b8 100644 --- a/apps/sim/lib/table/jobs/service.ts +++ b/apps/sim/lib/table/jobs/service.ts @@ -209,16 +209,17 @@ export async function getJobProgress(tableId: string, jobId: string): Promise> { +): Promise> { const deleteMask = await pendingDeleteMask(table) const rows = await db .select({ id: userTableRows.id, data: userTableRows.data, orderKey: userTableRows.orderKey }) @@ -228,14 +229,22 @@ export async function selectExportRowPage( eq(userTableRows.tableId, table.id), eq(userTableRows.workspaceId, table.workspaceId), deleteMask, + // `order_key` is nullable and sorts LAST, so the page order is "keyed rows by + // key, then the unkeyed tail by id". A bare row-constructor comparison is NULL + // for unkeyed rows (dropping them) and NULL for an unkeyed anchor (dropping + // everything), which silently truncates exports. Seek per anchor kind instead. after - ? sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id})` + ? after.orderKey === null + ? sql`${userTableRows.orderKey} IS NULL AND ${userTableRows.id} > ${after.id}` + : sql`(${userTableRows.orderKey} IS NULL OR (${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id}))` : undefined ) ) .orderBy(asc(userTableRows.orderKey), asc(userTableRows.id)) .limit(limit) - return rows as Array<{ id: string; data: RowData; orderKey: string }> + // drizzle types a jsonb column as `unknown`; every writer goes through the + // row-data validators, so narrowing here is a projection, not an assumption. + return rows.map((r) => ({ ...r, data: r.data as RowData })) } /** How long a terminal export stays listable (and re-downloadable from the tray). */ diff --git a/apps/sim/lib/table/planner.ts b/apps/sim/lib/table/planner.ts index 848e2c2d6de..85f4969b6f6 100644 --- a/apps/sim/lib/table/planner.ts +++ b/apps/sim/lib/table/planner.ts @@ -5,19 +5,52 @@ export type DbExecutor = typeof db | DbTransaction export type DbTransaction = Parameters[0]>[0] /** - * Runs `fn` with seq scans penalized (`SET LOCAL`, so the flag dies with the - * transaction). JSONB predicates and sort keys (`->>` extraction, `@>` - * containment, lateral `jsonb_each_text`) are opaque to the planner — it - * estimates a handful of matching rows and picks a parallel seq scan over the - * entire shared `user_table_rows` relation (every tenant's rows) instead of the - * tenant's own index. Measured on a 1M-row table inside a 12M-row relation: - * filtered count 12.7s → 1.0s, sorted page 9.7s → 0.76s, filtered bulk select - * 14.4s → tenant-bounded. The flag only penalizes the plan shape: if no index - * plan exists, the seq scan still runs. + * Statement/lock timeout for user-table READ queries. Reads are bounded tighter + * than the write path because filter shape is caller-controlled: a public API + * key can drive an arbitrary predicate (a catastrophic-backtracking `match` + * regex, a wide unindexed scan), and an untimed read pins a connection from the + * small shared `web` pool — one abusive key can starve every tenant. `SET LOCAL` + * scopes both to the surrounding transaction, so they die when it commits. */ -export async function withSeqscanOff(fn: (trx: DbTransaction) => Promise): Promise { +const READ_STATEMENT_TIMEOUT_MS = 15_000 +const READ_LOCK_TIMEOUT_MS = 3_000 + +async function setReadTimeouts(trx: DbTransaction): Promise { + await trx.execute(sql.raw(`SET LOCAL statement_timeout = '${READ_STATEMENT_TIMEOUT_MS}ms'`)) + await trx.execute(sql.raw(`SET LOCAL lock_timeout = '${READ_LOCK_TIMEOUT_MS}ms'`)) +} + +/** + * Runs a user-table read inside a transaction that always caps `statement_timeout` + * / `lock_timeout` (see {@link setReadTimeouts}). Pass `seqscanOff` for queries + * with no tenant-bounded index plan — custom column sorts and filtered counts — + * where the planner otherwise seq-scans the whole shared `user_table_rows` + * relation (every tenant's rows); see {@link withSeqscanOff} for the measured + * deltas. Default-order keyset/offset pages already stream the + * `(table_id, order_key, id)` index, so they take the timeout without the flag. + */ +export async function withReadGuards( + fn: (trx: DbTransaction) => Promise, + opts?: { seqscanOff?: boolean } +): Promise { return db.transaction(async (trx) => { - await trx.execute(sql`SET LOCAL enable_seqscan = off`) + await setReadTimeouts(trx) + if (opts?.seqscanOff) await trx.execute(sql`SET LOCAL enable_seqscan = off`) return fn(trx) }) } + +/** + * Runs `fn` with seq scans penalized (`SET LOCAL`, so the flag dies with the + * transaction) AND the read timeouts above. JSONB predicates and sort keys + * (`->>` extraction, `@>` containment, lateral `jsonb_each_text`) are opaque to + * the planner — it estimates a handful of matching rows and picks a parallel + * seq scan over the entire shared `user_table_rows` relation (every tenant's + * rows) instead of the tenant's own index. Measured on a 1M-row table inside a + * 12M-row relation: filtered count 12.7s → 1.0s, sorted page 9.7s → 0.76s, + * filtered bulk select 14.4s → tenant-bounded. The flag only penalizes the plan + * shape: if no index plan exists, the seq scan still runs (and the timeout caps it). + */ +export async function withSeqscanOff(fn: (trx: DbTransaction) => Promise): Promise { + return withReadGuards(fn, { seqscanOff: true }) +} diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index 131fdf7a74a..fa9dd5fb93e 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -8,10 +8,17 @@ import { describe, expect, it } from 'vitest' import { filterRulesToFilter, + filterRulesToPredicate, filterToRules, + isTablePredicate, + predicateToFilter, + predicateToFilterRules, pruneFilterForColumns, + prunePredicateForColumns, + sortRulesToSortSpec, + toLegacyFilter, } from '@/lib/table/query-builder/converters' -import type { ColumnDefinition, FilterRule } from '@/lib/table/types' +import type { ColumnDefinition, FilterRule, SortRule, TablePredicate } from '@/lib/table/types' function rule(overrides: Partial): FilterRule { return { @@ -122,6 +129,158 @@ describe('filterToRules', () => { }) }) +describe('filterRulesToPredicate (v2)', () => { + it('an AND group becomes a single all-group', () => { + const p = filterRulesToPredicate([ + rule({ column: 'slack_user_id', operator: 'in', value: 'U1, U2' }), + rule({ column: 'wins', operator: 'gte', value: '10' }), + ]) + expect(p).toEqual({ + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }) + }) + + it('an or boundary splits into an any-of-all groups', () => { + const p = filterRulesToPredicate([ + rule({ column: 'status', operator: 'eq', value: 'active' }), + rule({ column: 'status', operator: 'eq', value: 'pending', logicalOperator: 'or' }), + ]) + expect(p).toEqual({ + any: [ + { all: [{ field: 'status', op: 'eq', value: 'active' }] }, + { all: [{ field: 'status', op: 'eq', value: 'pending' }] }, + ], + }) + }) + + it('valueless ops omit the value', () => { + const p = filterRulesToPredicate([rule({ column: 'note', operator: 'isEmpty' })]) + expect(p).toEqual({ all: [{ field: 'note', op: 'isEmpty' }] }) + }) + + it('returns null for no rules', () => { + expect(filterRulesToPredicate([])).toBeNull() + }) + + it('round-trips rules → predicate → rules (operator + value preserved)', () => { + const rules: FilterRule[] = [ + rule({ column: 'wins', operator: 'gte', value: '10' }), + rule({ column: 'name', operator: 'contains', value: 'jo', logicalOperator: 'or' }), + ] + const back = predicateToFilterRules(filterRulesToPredicate(rules)) + expect(back.map((r) => [r.column, r.operator, r.value, r.logicalOperator])).toEqual([ + ['wins', 'gte', '10', 'and'], + ['name', 'contains', 'jo', 'or'], + ]) + }) +}) + +describe('predicateToFilterRules (v2)', () => { + it('flattens an all-group to and-joined rules', () => { + const p: TablePredicate = { + all: [ + { field: 'a', op: 'eq', value: 1 }, + { field: 'b', op: 'isNotEmpty' }, + ], + } + const rules = predicateToFilterRules(p) + expect(rules.map((r) => [r.column, r.operator, r.value])).toEqual([ + ['a', 'eq', '1'], + ['b', 'isNotEmpty', ''], + ]) + }) + + it('returns [] for null', () => { + expect(predicateToFilterRules(null)).toEqual([]) + }) +}) + +describe('predicateToFilter (v2 → legacy)', () => { + it('maps an all-group to $and with bare-op leaves', () => { + const p: TablePredicate = { + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + { field: 'name', op: 'eq', value: 'x' }, + ], + } + expect(predicateToFilter(p)).toEqual({ + $and: [{ slack_user_id: { $in: ['U1', 'U2'] } }, { wins: { $gte: 10 } }, { name: 'x' }], + }) + }) + + it('maps an any-group to $or and valueless ops to $empty', () => { + const p: TablePredicate = { + any: [ + { field: 'a', op: 'isEmpty' }, + { field: 'b', op: 'isNotEmpty' }, + ], + } + expect(predicateToFilter(p)).toEqual({ + $or: [{ a: { $empty: true } }, { b: { $empty: false } }], + }) + }) + + /** + * The conversion is only safe if it is lossless: `buildFilterClause` skips an + * `undefined` condition and any array-valued one, so a leaf that converts to + * either DISAPPEARS from the WHERE — and a predicate whose only leaf disappears + * compiles to no WHERE at all, turning a bulk delete into a table wipe. + */ + it('throws rather than emit a leaf the legacy compiler would discard', () => { + expect(() => + predicateToFilter({ all: [{ field: 'status', op: 'eq', value: ['a', 'b'] }] }) + ).toThrow(/does not accept an array/) + expect(() => predicateToFilter({ all: [{ field: 'status', op: 'eq' }] })).toThrow( + /requires a value/ + ) + }) +}) + +describe('sortRulesToSortSpec (v2)', () => { + it('maps rules to an ordered field/direction list', () => { + const rules: SortRule[] = [ + { id: '1', column: 'wins', direction: 'desc' }, + { id: '2', column: 'name', direction: 'asc' }, + ] + expect(sortRulesToSortSpec(rules)).toEqual([ + { field: 'wins', direction: 'desc' }, + { field: 'name', direction: 'asc' }, + ]) + }) + + it('skips column-less rules and returns null when empty', () => { + expect(sortRulesToSortSpec([{ id: '1', column: '', direction: 'asc' }])).toBeNull() + }) +}) + +it('does not throw "rules is not iterable" for a non-array builder value', () => { + expect(filterRulesToPredicate({} as unknown as FilterRule[])).toBeNull() + expect(filterRulesToPredicate(undefined as unknown as FilterRule[])).toBeNull() +}) + +it('rejects predicate-shaped members rather than dropping them', () => { + // Mixing grammars used to silently discard the predicate-shaped leaf, widening + // the filter — "delete archived rows for tenant acme" became "…for every tenant". + expect(() => + filterRulesToPredicate([ + { field: 'tenant_id', op: 'eq', value: 'acme' }, + rule({ column: 'status', operator: 'eq', value: 'archived' }), + ] as unknown as FilterRule[]) + ).toThrow(/predicate condition/) +}) + +it('still ignores a genuinely blank builder row', () => { + expect( + filterRulesToPredicate([ + rule({ column: '', operator: 'eq', value: '' }), + rule({ column: 'status', operator: 'eq', value: 'archived' }), + ]) + ).toEqual({ all: [{ field: 'status', op: 'eq', value: 'archived' }] }) +}) describe('select option ids survive as text', () => { const numericIdColumn: ColumnDefinition = { id: 'status', @@ -210,3 +369,144 @@ describe('pruneFilterForColumns', () => { expect(pruneFilterForColumns(null, [single])).toBeNull() }) }) + +describe('filterRulesToPredicate select-awareness (grid parity)', () => { + const SELECT_COLS: ColumnDefinition[] = [ + { + id: 'col_status', + name: 'col_status', + type: 'select', + options: [{ id: '123', name: 'One Two Three' }], + }, + ] + + it('never scalar-coerces a select operand (a numeric-looking option id stays a string)', () => { + const p = filterRulesToPredicate( + [rule({ column: 'col_status', operator: 'eq', value: '123' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'col_status', op: 'eq', value: '123' }] }) + }) + + it('keeps select in-list members as strings', () => { + const p = filterRulesToPredicate( + [rule({ column: 'col_status', operator: 'in', value: '123, 456' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'col_status', op: 'in', value: ['123', '456'] }] }) + }) + + it('still coerces non-select columns', () => { + const p = filterRulesToPredicate( + [rule({ column: 'wins', operator: 'eq', value: '123' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'wins', op: 'eq', value: 123 }] }) + }) +}) + +describe('prunePredicateForColumns', () => { + const COLS: ColumnDefinition[] = [ + { id: 'col_s', name: 'col_s', type: 'select', options: [{ id: 'o1', name: 'One' }] }, + { id: 'col_n', name: 'col_n', type: 'number' }, + ] + + it('drops an operator a select column no longer accepts, keeps the rest', () => { + const p = prunePredicateForColumns( + { + all: [ + { field: 'col_s', op: 'contains', value: 'o1' }, // single-select: contains not allowed + { field: 'col_n', op: 'gte', value: 5 }, + ], + }, + COLS + ) + expect(p).toEqual({ all: [{ field: 'col_n', op: 'gte', value: 5 }] }) + }) + + it('returns the same reference when nothing is dropped', () => { + const p = { all: [{ field: 'col_n', op: 'gte' as const, value: 5 }] } + expect(prunePredicateForColumns(p, COLS)).toBe(p) + }) + + it('collapses to null when every condition is dropped', () => { + expect( + prunePredicateForColumns({ all: [{ field: 'col_s', op: 'contains', value: 'o1' }] }, COLS) + ).toBeNull() + }) +}) + +describe('isTablePredicate', () => { + it('discriminates the two grammars group-first', () => { + expect(isTablePredicate({ all: [] })).toBe(true) + expect(isTablePredicate({ any: [] })).toBe(true) + expect(isTablePredicate({ status: 'active' })).toBe(false) + expect(isTablePredicate({ $or: [{ a: 1 }] } as never)).toBe(false) + }) +}) + +it('prunePredicateForColumns fails closed on a malformed value instead of throwing', () => { + expect(prunePredicateForColumns({ column: 'name', operator: 'eq' } as never, [])).toBeNull() +}) + +/** + * Review findings from PR #6067 (greptile P1 ×2, bugbot High). The dual-grammar + * union's legacy branch accepts any non-empty object WITHOUT stripping, so a + * hybrid node reaches the downgrade Zod-approved — and predicateToFilter used + * to convert it group-first, silently DROPPING the leaf half and widening the + * filter on the async destructive routes (delete-async, cancel-runs, columns/run). + */ +describe('toLegacyFilter / predicateToFilter hybrid safety', () => { + const hybrid = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + } as never + + it('predicateToFilter throws on a hybrid node instead of dropping the leaf', () => { + expect(() => predicateToFilter(hybrid)).toThrow(/not both/) + }) + + it('throws on a node with both group keys instead of dropping the "any" half', () => { + const dual = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + any: [{ field: 'status', op: 'eq', value: 'archived' }], + } as never + expect(() => predicateToFilter(dual)).toThrow(/either "all" or "any"/) + expect(() => toLegacyFilter(dual)).toThrow(/either "all" or "any"/) + }) + + it('toLegacyFilter shape-validates before converting', () => { + expect(() => toLegacyFilter(hybrid)).toThrow(/not both/) + // malformed group value is also caught, not crashed on + expect(() => toLegacyFilter({ all: [{ all: 'x' }] } as never)).toThrow() + }) + + it('toLegacyFilter passes a well-formed predicate and a legacy filter through', () => { + expect(toLegacyFilter({ all: [{ field: 'a', op: 'eq', value: 1 }] })).toEqual({ + $and: [{ a: 1 }], + }) + expect(toLegacyFilter({ status: 'archived' })).toEqual({ status: 'archived' }) + }) +}) + +describe('isTablePredicate vs columns literally named all/any', () => { + it('routes a non-array all/any value to the legacy compiler (a real column)', () => { + // NAME_PATTERN allows a column named "all". Equality shorthand and operator + // objects on it are legacy filters, not predicate groups. + expect(isTablePredicate({ all: 'x' } as never)).toBe(false) + expect(isTablePredicate({ any: { $eq: 'x' } } as never)).toBe(false) + }) + + it('array-valued all/any is a predicate group (documented precedence)', () => { + // A legacy filter with an ARRAY on a regular field was always a dropped + // no-op condition, so predicate precedence here regresses nothing. + expect(isTablePredicate({ all: [] })).toBe(true) + expect(isTablePredicate({ any: [{ field: 'a', op: 'eq', value: 1 }] })).toBe(true) + }) +}) + +it('toLegacyFilter rejects an empty group instead of downgrading it to no WHERE', () => { + expect(() => toLegacyFilter({ all: [] })).toThrow(/at least one condition/) +}) diff --git a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts new file mode 100644 index 00000000000..12eea2427a0 --- /dev/null +++ b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts @@ -0,0 +1,215 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { TableQueryValidationError } from '@/lib/table/errors' +import { + validatePredicate, + validatePredicateShape, + validateSortSpec, +} from '@/lib/table/query-builder/validate' +import type { ColumnDefinition } from '@/lib/table/types' + +const COLS: ColumnDefinition[] = [ + { name: 'wins', type: 'number' }, + { name: 'status', type: 'string' }, + { name: 'metadata', type: 'json' }, +] + +describe('validatePredicate', () => { + it('accepts a valid predicate over real + system columns', () => { + expect(() => + validatePredicate( + { + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'createdAt', op: 'lt', value: '2026-01-01' }, + { any: [{ field: 'status', op: 'eq', value: 'active' }] }, + ], + }, + COLS + ) + ).not.toThrow() + }) + + it('rejects an unknown column', () => { + expect(() => validatePredicate({ all: [{ field: 'nope', op: 'eq', value: 1 }] }, COLS)).toThrow( + /Unknown filter column/ + ) + }) + + it('rejects equality/containment ops on a json column', () => { + for (const op of ['eq', 'ne', 'in', 'nin'] as const) { + const value = op === 'in' || op === 'nin' ? ['x'] : 'x' + expect(() => validatePredicate({ all: [{ field: 'metadata', op, value }] }, COLS)).toThrow( + /json column/ + ) + } + }) + + it('allows text-match / null ops on a json column', () => { + expect(() => + validatePredicate({ all: [{ field: 'metadata', op: 'ilike', value: '*x*' }] }, COLS) + ).not.toThrow() + expect(() => + validatePredicate({ all: [{ field: 'metadata', op: 'isNull' }] }, COLS) + ).not.toThrow() + }) + + it('rejects an empty in/nin array', () => { + expect(() => + validatePredicate({ all: [{ field: 'wins', op: 'in', value: [] }] }, COLS) + ).toThrow(/non-empty array/) + }) + + it('rejects an invalid field name', () => { + expect(() => + validatePredicate({ all: [{ field: "x'; DROP", op: 'eq', value: 1 }] }, COLS) + ).toThrow(/Invalid filter column/) + }) + + it('carries the INVALID_FILTER code', () => { + try { + validatePredicate({ all: [{ field: 'nope', op: 'eq', value: 1 }] }, COLS) + expect.unreachable() + } catch (e) { + expect(e).toBeInstanceOf(TableQueryValidationError) + expect((e as TableQueryValidationError).code).toBe('INVALID_FILTER') + } + }) +}) + +describe('validateSortSpec', () => { + it('accepts real and system columns', () => { + expect(() => + validateSortSpec( + [ + { field: 'wins', direction: 'desc' }, + { field: 'updatedAt', direction: 'asc' }, + ], + COLS + ) + ).not.toThrow() + }) + + it('rejects an unknown sort column with INVALID_ORDER', () => { + try { + validateSortSpec([{ field: 'nope', direction: 'asc' }], COLS) + expect.unreachable() + } catch (e) { + expect((e as TableQueryValidationError).code).toBe('INVALID_ORDER') + } + }) +}) + +/** + * These shapes pass structural validation but compile to a clause the legacy + * `$`-grammar discards — which turns a bulk delete/update into "match everything". + */ +describe('validatePredicate — leaves that would silently widen a bulk write', () => { + it('rejects a scalar op given an array (the eq-instead-of-in mistake)', () => { + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'eq', value: ['a', 'b'] }] }, COLS) + ).toThrow(/does not accept an array — use "in"/) + }) + + it('rejects a value-taking op with no value', () => { + expect(() => validatePredicate({ all: [{ field: 'status', op: 'eq' }] }, COLS)).toThrow( + /requires a value/ + ) + expect(() => validatePredicate({ all: [{ field: 'wins', op: 'gte' }] }, COLS)).toThrow( + /requires a value/ + ) + }) + + it('still allows the valueless ops', () => { + for (const op of ['isNull', 'isNotNull', 'isEmpty', 'isNotEmpty'] as const) { + expect(() => validatePredicate({ all: [{ field: 'status', op }] }, COLS)).not.toThrow() + } + }) + + it('rejects a hybrid group+leaf node instead of silently picking one', () => { + // Read group-first by the engine, leaf-first by predicateToFilter — so the gate + // would validate one predicate and the bulk-write path execute another. + expect(() => + validatePredicate( + { + all: [{ field: 'status', op: 'eq', value: 'benign' }], + field: 'nope', + op: 'isNotNull', + } as never, + COLS + ) + ).toThrow(/not both/) + }) + + it('rejects a node carrying both group keys instead of dropping the "any" half', () => { + // Every group-first traversal reads `all` and drops `any` — half the + // caller's conditions would vanish, widening a bulk delete/update. + expect(() => + validatePredicate( + { + all: [{ field: 'status', op: 'eq', value: 'archived' }], + any: [{ field: 'status', op: 'eq', value: 'stale' }], + } as never, + COLS + ) + ).toThrow(/either "all" or "any"/) + }) + + it('caps in/nin list length', () => { + const huge = Array.from({ length: 1001 }, (_, i) => `v${i}`) + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'in', value: huge }] }, COLS) + ).toThrow(/at most 1000 values/) + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'in', value: huge.slice(0, 1000) }] }, COLS) + ).not.toThrow() + }) +}) + +/** + * The copilot's `query_user_table` catalog entry still advertises the legacy + * `$`-grammar, so the agent sends `{ status: { $eq: 'x' } }`. That shape hit + * `validateLeaf` with `field: undefined` and came back as + * `Unknown filter column "undefined"` — a message that sends an LLM retrying + * column names instead of switching grammars. + */ +describe('validatePredicate — legacy $-grammar diagnostics', () => { + it('names the grammar mistake instead of blaming a phantom column', () => { + for (const legacy of [ + { status: { $eq: 'active' } }, + { $or: [{ status: 'a' }, { status: 'b' }] }, + { wins: { $gte: 10 } }, + ]) { + expect(() => validatePredicate(legacy as never, COLS)).toThrow(/legacy operator-object/) + expect(() => validatePredicate(legacy as never, COLS)).not.toThrow(/undefined/) + } + }) + + it('still rejects a plain non-predicate object clearly', () => { + expect(() => validatePredicate({ nonsense: 'x' } as never, COLS)).toThrow( + /must be a group .* or a condition/ + ) + }) +}) + +/** + * Bugbot round 2: `{all: []}` passes the dual union via the non-empty-OBJECT + * legacy branch, then compiled to no WHERE clause — silently widening a run / + * cancel / delete scope to every row. The shape validator now mirrors the Zod + * contract's .min(1). + */ +describe('empty groups are rejected at every layer', () => { + it('validatePredicateShape rejects an empty group', () => { + expect(() => validatePredicateShape({ all: [] })).toThrow(/at least one condition/) + expect(() => validatePredicateShape({ any: [] })).toThrow(/at least one condition/) + expect(() => validatePredicateShape({ all: [{ any: [] }] } as never)).toThrow( + /at least one condition/ + ) + }) + + it('validatePredicate rejects it too', () => { + expect(() => validatePredicate({ all: [] }, COLS)).toThrow(/at least one condition/) + }) +}) diff --git a/apps/sim/lib/table/query-builder/constants.ts b/apps/sim/lib/table/query-builder/constants.ts index a881fd0aba4..8a5b9af4117 100644 --- a/apps/sim/lib/table/query-builder/constants.ts +++ b/apps/sim/lib/table/query-builder/constants.ts @@ -34,7 +34,8 @@ export const VALUELESS_OPERATORS = new Set(['isEmpty', 'isNotEmpty']) * against the whole array can never be true. * * These must stay in step with the server whitelist in `lib/table/sql.ts`: the - * filter picker offers exactly these, and `pruneFilterForColumns` DROPS + * filter picker offers exactly these, and `pruneFilterForColumns` / + * `prunePredicateForColumns` DROP * anything outside them, so an operator missing here is silently discarded from * a filter the server would have accepted. `sql.test.ts` asserts the two sets * agree through `UI_TO_WIRE_OPERATOR` rather than leaving it to a comment. @@ -69,7 +70,12 @@ export const LOGICAL_OPERATORS = [ { value: 'or', label: 'or' }, ] as const -export const SORT_DIRECTIONS = [ +/** + * Direction picker options for the sort-builder UI. Distinct from the wire-level + * `SORT_DIRECTIONS` tuple in `lib/table/constants.ts` — both are star-exported + * through `lib/table/index.ts`, and a duplicate name resolves to nothing there. + */ +export const SORT_DIRECTION_OPTIONS = [ { value: 'asc', label: 'ascending' }, { value: 'desc', label: 'descending' }, ] as const diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index 297a244c3e4..5e938c64731 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -5,18 +5,24 @@ import { generateShortId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' import { columnMatchesRef } from '@/lib/table/column-keys' +import { TableQueryValidationError } from '@/lib/table/errors' import { MULTI_SELECT_FILTER_OPERATORS, SINGLE_SELECT_FILTER_OPERATORS, } from '@/lib/table/query-builder/constants' +import { validatePredicateShape } from '@/lib/table/query-builder/validate' import type { ColumnDefinition, Filter, + FilterOp, FilterRule, JsonValue, + Predicate, Sort, SortDirection, SortRule, + SortSpec, + TablePredicate, } from '@/lib/table/types' /** @@ -112,6 +118,47 @@ export function pruneFilterForColumns( return filterRulesToFilter(kept, columns) } +/** + * Predicate-grammar sibling of {@link pruneFilterForColumns}: drops conditions a + * `select` column no longer accepts (operator stranded by a type/`multiple` + * change), so a stale applied filter can't fail every subsequent rows query. + */ +export function prunePredicateForColumns( + predicate: TablePredicate | null, + columns: ColumnDefinition[] +): TablePredicate | null { + if (!predicate) return null + // A malformed value (corrupt persisted state, a stale cast) fails CLOSED to + // "no filter" — throwing here would take down the whole table page render. + if (!('all' in predicate) && !('any' in predicate)) return null + + const rules = predicateToFilterRules(predicate) + const kept = rules.filter((rule) => { + const column = columns.find((c) => columnMatchesRef(c, rule.column)) + if (column?.type !== 'select') return true + const allowed = column.multiple ? MULTI_SELECT_FILTER_OPERATORS : SINGLE_SELECT_FILTER_OPERATORS + return allowed.has(rule.operator) + }) + + if (kept.length === rules.length) return predicate + return filterRulesToPredicate(kept, columns) +} + +/** + * Discriminates the v2 predicate tree from the legacy `$`-object on dual-grammar + * wire fields. Group-first, matching every other discrimination site. + */ +export function isTablePredicate(value: Filter | TablePredicate): value is TablePredicate { + // Require the group value to be an ARRAY: a legacy filter on a real column + // that happens to be named `all`/`any` (allowed by NAME_PATTERN) uses the + // equality shorthand ({ all: "x" }) or an operator object — neither is + // array-valued, so both keep routing to the legacy compiler. A column named + // all/any holding a literal array was already a dropped no-op condition in + // the legacy grammar, so predicate precedence on arrays regresses nothing. + const v = value as Record + return ('all' in v && Array.isArray(v.all)) || ('any' in v && Array.isArray(v.any)) +} + /** Converts a single UI sort rule to a Sort object for API queries. */ export function sortRuleToSort(rule: SortRule | null): Sort | null { if (!rule || !rule.column) return null @@ -280,3 +327,192 @@ function formatValueForBuilder(value: JsonValue): string { function normalizeSortDirection(direction: string): SortDirection { return direction === 'desc' ? 'desc' : 'asc' } + +/* ----------------------------- v2 grammar ----------------------------- */ + +const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty', 'isNull', 'isNotNull']) + +function ruleToPredicate(rule: FilterRule, keepAsText = false): Predicate { + const op = rule.operator as FilterOp + if (VALUELESS_OPS.has(op)) return { field: rule.column, op } + return { field: rule.column, op, value: parseValue(rule.value, rule.operator, keepAsText) } +} + +/** + * Converts UI builder rules to a v2 `TablePredicate`. Rules within an `or` + * boundary form an `all` group; multiple groups are combined under `any`. + * Mirrors {@link filterRulesToFilter} but emits the bare-operator grammar. + */ +export function filterRulesToPredicate( + rules: FilterRule[], + columns: ColumnDefinition[] = [] +): TablePredicate | null { + // Tolerate a non-array (the builder value can arrive malformed from an agent + // that doesn't speak the rule shape) instead of throwing "rules is not iterable". + if (!Array.isArray(rules) || rules.length === 0) return null + + const groups: Predicate[][] = [] + let current: Predicate[] = [] + + for (const rule of rules) { + if (rule.logicalOperator === 'or' && current.length > 0) { + groups.push(current) + current = [] + } + if (!rule.column) { + // A predicate-shaped member ({field, op, value}) means the caller mixed the + // two grammars — most likely an agent writing predicate leaves into the + // visual builder. Silently skipping it would DROP that condition and widen + // the result, which on a bulk delete/update is destructive. + if (isRecordLike(rule) && ('field' in rule || 'op' in rule)) { + throw new TableQueryValidationError( + 'Filter looks like a predicate condition but was supplied as a builder rule. ' + + 'Provide the whole filter as a predicate object ({ all | any: [...] }) instead of mixing shapes.', + 'INVALID_FILTER' + ) + } + // A genuinely blank builder row (no column picked yet) contributes nothing. + continue + } + // A select value is an opaque option id — never scalar-coerce it (an id + // that happens to look numeric would silently become a number and match + // nothing). Same rule as filterRulesToFilter. + const isSelect = columns.find((c) => columnMatchesRef(c, rule.column))?.type === 'select' + current.push(ruleToPredicate(rule, isSelect)) + } + if (current.length > 0) groups.push(current) + + if (groups.length === 0) return null + if (groups.length === 1) return { all: groups[0] } + return { any: groups.map((group) => ({ all: group })) } +} + +function predicateLeafToRule(p: Predicate): FilterRule { + return { + id: generateShortId(), + logicalOperator: 'and', + column: p.field, + operator: p.op, + value: VALUELESS_OPS.has(p.op) ? '' : formatValueForBuilder(p.value as JsonValue), + } +} + +/** Flattens a predicate node into builder rules (best-effort for deep nesting). */ +function predicateGroupToRules(node: TablePredicate | Predicate): FilterRule[] { + if ('field' in node) return [predicateLeafToRule(node)] + const members = 'all' in node ? node.all : node.any + return members.flatMap((member) => + 'field' in member ? [predicateLeafToRule(member)] : predicateGroupToRules(member) + ) +} + +/** Converts a v2 `TablePredicate` back to UI builder rules. */ +export function predicateToFilterRules(predicate: TablePredicate | null): FilterRule[] { + if (!predicate) return [] + if ('any' in predicate) { + const groups = predicate.any + .map((node) => predicateGroupToRules(node)) + .filter((g) => g.length > 0) + return applyLogicalOperators(groups) + } + return predicateGroupToRules(predicate) +} + +/** Converts UI sort rules to a v2 `SortSpec` (ordered `{ field, direction }`). */ +export function sortRulesToSortSpec(rules: SortRule[]): SortSpec | null { + const spec: SortSpec = [] + for (const rule of rules) { + if (rule.column) spec.push({ field: rule.column, direction: rule.direction }) + } + return spec.length > 0 ? spec : null +} + +function predicateLeafToFilterValue(p: Predicate): Filter[string] { + if (p.op === 'isEmpty') return { $empty: true } + if (p.op === 'isNotEmpty') return { $empty: false } + // Valueless null checks carry a dummy `true` so the key survives JSON transport. + if (p.op === 'isNull') return { $isNull: true } as Filter[string] + if (p.op === 'isNotNull') return { $isNotNull: true } as Filter[string] + + // Fail loud rather than emit a leaf `buildFilterClause` silently discards. It skips + // an `undefined` condition and any array-valued one, so a dropped leaf WIDENS the + // result — and on the bulk delete/update paths a predicate whose only leaf is + // dropped compiles to no WHERE at all. `op:'eq'` with an array is the realistic + // trigger (an LLM reaching for `in` and writing `eq`). + if (!VALUELESS_OPS.has(p.op) && p.value === undefined) { + throw new TableQueryValidationError( + `Operator "${p.op}" on column "${p.field}" requires a value.`, + 'INVALID_FILTER' + ) + } + if (p.op === 'eq') { + if (Array.isArray(p.value)) { + throw new TableQueryValidationError( + `Operator "eq" on column "${p.field}" does not accept an array — use "in" to match any of several values.`, + 'INVALID_FILTER' + ) + } + return p.value as Filter[string] + } + return { [`$${p.op}`]: p.value } as Filter[string] +} + +/** + * Converts a v2 `TablePredicate` to a legacy `$`-grammar `Filter`. Lossless — + * both compile through the same `fieldPredicate` leaf, so the resulting SQL is + * identical. Lets v2 surfaces author in the predicate grammar while the bulk + * update/delete engine (sync + async-job paths) keeps consuming `Filter`. + */ +export function predicateToFilter(predicate: TablePredicate): Filter { + const nodeToFilter = (node: Predicate | TablePredicate): Filter => { + // A hybrid node (group key AND leaf keys) would convert group-first here, + // silently DROPPING the leaf half — which widens the filter, and on the + // bulk delete/update paths that is destructive. Lossless-or-throw, like + // every other non-representable shape in this converter. + if (('all' in node || 'any' in node) && 'field' in node) { + throw new TableQueryValidationError( + 'A filter node must be either a group ({ all | any: [...] }) or a condition ({ field, op, value }), not both.', + 'INVALID_FILTER' + ) + } + // A node with BOTH group keys would convert `all` and silently drop `any` + // — same widening failure as the hybrid above. Lossless-or-throw. + if ('all' in node && 'any' in node) { + throw new TableQueryValidationError( + 'A filter group must use either "all" or "any", not both — nest one group inside the other instead.', + 'INVALID_FILTER' + ) + } + // Group-first, matching isPredicateGroup/validateNode/buildPredicateNode. A + // leaf-first test would execute a different predicate than the one validated. + if ('all' in node) return { $and: node.all.map(nodeToFilter) } + if ('any' in node) return { $or: node.any.map(nodeToFilter) } + return { [node.field]: predicateLeafToFilterValue(node) } + } + return nodeToFilter(predicate) +} + +/** + * Downgrades a dual-grammar wire filter to the legacy `Filter` the job runners + * and search service still compile. The predicate path throws (via + * `predicateToFilter`) on any leaf the legacy compiler would silently discard, + * so a downgraded filter can never widen. Grammar-only: field keys pass through + * untranslated (session callers already speak column ids). + */ +export function toLegacyFilter(filter: Filter | TablePredicate | undefined): Filter | undefined { + if (!filter) return undefined + if (!isTablePredicate(filter)) return filter + // Shape-validate before converting: the dual-grammar union's legacy branch + // accepts any non-empty object without stripping, so a hybrid or malformed + // tree reaches here Zod-approved. The predicate may be name- OR id-keyed + // (grid vs tools), so only the keying-agnostic checks apply. + validatePredicateShape(filter) + return predicateToFilter(filter) +} + +/** Downgrades a dual-grammar wire sort (ordered spec array or legacy record). */ +export function toLegacySort(sort: Sort | SortSpec | undefined): Sort | undefined { + if (!sort) return undefined + if (!Array.isArray(sort)) return sort + return sort.length > 0 ? Object.fromEntries(sort.map((s) => [s.field, s.direction])) : undefined +} diff --git a/apps/sim/lib/table/query-builder/use-query-builder.ts b/apps/sim/lib/table/query-builder/use-query-builder.ts index 623ec4cd8fd..2d86ae236fd 100644 --- a/apps/sim/lib/table/query-builder/use-query-builder.ts +++ b/apps/sim/lib/table/query-builder/use-query-builder.ts @@ -8,7 +8,7 @@ import { COMPARISON_OPERATORS, type FilterRule, LOGICAL_OPERATORS, - SORT_DIRECTIONS, + SORT_DIRECTION_OPTIONS, type SortRule, } from '@/lib/table/query-builder/constants' import type { ColumnOption } from '@/lib/table/types' @@ -23,7 +23,7 @@ const logicalOptions: ColumnOption[] = LOGICAL_OPERATORS.map((op) => ({ label: op.label, })) -const sortDirectionOptions: ColumnOption[] = SORT_DIRECTIONS.map((d) => ({ +const sortDirectionOptions: ColumnOption[] = SORT_DIRECTION_OPTIONS.map((d) => ({ value: d.value, label: d.label, })) diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts new file mode 100644 index 00000000000..f508cd56b57 --- /dev/null +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -0,0 +1,225 @@ +import { isRecordLike } from '@sim/utils/object' +import { getColumnId } from '@/lib/table/column-keys' +import { NAME_PATTERN } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' +import type { + ColumnDefinition, + ColumnType, + FilterOp, + Predicate, + PredicateNode, + SortSpec, + TablePredicate, +} from '@/lib/table/types' + +/** + * Schema-aware validation for the typed predicate/sort wire. The engine + * (`buildPredicateClause`) trusts its input, so this is the boundary gate that + * every caller-supplied filter/sort passes through — the same checks the old + * parser used to do inline, now grammar-agnostic and applied to the object + * directly (predicates are column-NAME-keyed at the boundary, translated to ids + * afterwards). + */ + +/** Equality/containment ops that are meaningless on a `json` column (they never match). */ +const CONTAINMENT_OPS = new Set(['eq', 'ne', 'in', 'nin']) + +/** Ops that legitimately carry no `value`. */ +const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty', 'isNull', 'isNotNull']) + +/** + * Cap on `in`/`nin` list length. Each element becomes its own containment clause, + * so an unbounded list is a cheap memory/CPU amplifier from a small request body. + */ +const MAX_IN_LIST_SIZE = 1000 + +/** + * Row-level system columns are filterable/sortable but are not in + * `schema.columns`. Must stay in sync with `SYSTEM_COLUMNS` in `lib/table/sql.ts` + * — a name here with no SQL dispatch there compiles to a `data->>` extraction + * that silently matches nothing. + */ +const SYSTEM_COLUMN_TYPES: ReadonlyArray<[string, ColumnType]> = [ + ['createdAt', 'date'], + ['updatedAt', 'date'], + ['id', 'string'], +] + +function buildTypeByName(columns: ColumnDefinition[]): Map { + const typeByName = new Map(columns.map((c) => [c.name, c.type])) + for (const [name, type] of SYSTEM_COLUMN_TYPES) typeByName.set(name, type) + return typeByName +} + +function validateFieldName(field: string): void { + if (!NAME_PATTERN.test(field)) { + throw new TableQueryValidationError( + `Invalid filter column "${field}". Use a column name (letters, digits, underscore).`, + 'INVALID_FILTER' + ) + } +} + +function validateLeaf(leaf: Predicate, typeByName: Map | null): void { + validateFieldName(leaf.field) + if (typeByName && !typeByName.has(leaf.field)) { + throw new TableQueryValidationError( + `Unknown filter column "${leaf.field}". It is not a column on this table.`, + 'INVALID_FILTER' + ) + } + if (typeByName?.get(leaf.field) === 'json' && CONTAINMENT_OPS.has(leaf.op)) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" is not supported on json column "${leaf.field}" — use like/ilike for text match, or isNull/isNotNull.`, + 'INVALID_FILTER' + ) + } + if (leaf.op === 'in' || leaf.op === 'nin') { + if (!Array.isArray(leaf.value) || leaf.value.length === 0) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" requires a non-empty array value.`, + 'INVALID_FILTER' + ) + } + if (leaf.value.length > MAX_IN_LIST_SIZE) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" accepts at most ${MAX_IN_LIST_SIZE} values, got ${leaf.value.length}.`, + 'INVALID_FILTER' + ) + } + return + } + // A value-taking op with no value, or a scalar op handed an array, compiles to a + // clause the legacy `$`-grammar silently discards — which WIDENS a bulk delete or + // update. Reject here so the copilot path (no Zod) fails the same way the HTTP + // boundary does. + if (!VALUELESS_OPS.has(leaf.op) && leaf.value === undefined) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" requires a value.`, + 'INVALID_FILTER' + ) + } + if (Array.isArray(leaf.value)) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" does not accept an array — use "in" to match any of several values.`, + 'INVALID_FILTER' + ) + } +} + +/** + * Structure-only validation: hybrid nodes, group shapes, leaf value rules — + * everything that does not require knowing the table's columns. Used by the + * dual-grammar boundaries where the predicate may be NAME- or ID-keyed, so a + * column-existence check against either keying would be wrong. + */ +export function validatePredicateShape(predicate: TablePredicate): void { + validateNode(predicate, null) +} + +function validateNode(node: PredicateNode, typeByName: Map | null): void { + // Guard before the `in` checks below: an untrusted caller (copilot args, a raw + // block value) can hand us a string/number/null, where `'all' in node` throws + // a raw TypeError. Fail with a clean, actionable message instead. + if (typeof node !== 'object' || node === null) { + throw new TableQueryValidationError( + 'Filter must be a predicate object ({ all | any: [...] }).', + 'INVALID_FILTER' + ) + } + // A node carrying BOTH a group key and `field` is ambiguous: the engine and this + // validator read it group-first while `predicateToFilter`/`predicateNamesToIds` + // read it leaf-first, so the gate would validate one predicate and the bulk-write + // path would execute a different one. Reject rather than pick a winner. + if (('all' in node || 'any' in node) && 'field' in node) { + throw new TableQueryValidationError( + 'A filter node must be either a group ({ all | any: [...] }) or a condition ({ field, op, value }), not both.', + 'INVALID_FILTER' + ) + } + // Same ambiguity with BOTH group keys: every group-first traversal + // (`predicateNamesToIds`, `predicateToFilter`, `buildPredicateNode`) reads + // `all` and silently DROPS `any`, so half the caller's conditions vanish — + // on a bulk delete/update that widens the matched set. Reject rather than + // pick a winner; nesting expresses the intent unambiguously. + if ('all' in node && 'any' in node) { + throw new TableQueryValidationError( + 'A filter group must use either "all" or "any", not both — nest one group inside the other instead.', + 'INVALID_FILTER' + ) + } + if ('all' in node || 'any' in node) { + const members = 'all' in node ? node.all : node.any + if (!Array.isArray(members)) { + throw new TableQueryValidationError( + 'A filter group ({ all | any }) must be an array of conditions.', + 'INVALID_FILTER' + ) + } + // Mirrors the Zod contract's .min(1). An empty group slips the strict union + // (falling to the non-empty-OBJECT legacy branch) and compiles to no WHERE + // clause — which on the run/cancel/delete scopes silently means EVERY row. + if (members.length === 0) { + throw new TableQueryValidationError( + 'A filter group must contain at least one condition.', + 'INVALID_FILTER' + ) + } + for (const child of members) validateNode(child, typeByName) + return + } + // Neither a group nor a leaf. Overwhelmingly this is the legacy `$`-grammar + // (`{ status: { $eq: 'x' } }`) — a shape `validateLeaf` would reject as + // `Unknown filter column "undefined"`, which tells an LLM caller nothing and + // sends it retrying column names forever. Name the actual mistake. + if (!('field' in node)) { + const keys = Object.keys(node) + const looksLegacy = keys.some( + (k) => k.startsWith('$') || isRecordLike((node as Record)[k]) + ) + throw new TableQueryValidationError( + looksLegacy + ? 'Filter uses the legacy operator-object grammar. Use a predicate tree instead: { all: [{ field, op, value }] } (or "any" for OR), with bare operators like eq/gte/contains/in.' + : 'A filter node must be a group ({ all | any: [...] }) or a condition ({ field, op, value }).', + 'INVALID_FILTER' + ) + } + validateLeaf(node as Predicate, typeByName) +} + +/** + * Validates a name-keyed predicate against the table schema: every leaf field + * exists, no equality/containment op targets a `json` column, `in`/`nin` carry a + * non-empty array. Throws {@link TableQueryValidationError} (`INVALID_FILTER`). + */ +export function validatePredicate(predicate: TablePredicate, columns: ColumnDefinition[]): void { + validateNode(predicate, buildTypeByName(columns)) +} + +/** Validates a name-keyed sort spec: every field is a real or system column. */ +export function validateSortSpec(spec: SortSpec, columns: ColumnDefinition[]): void { + const typeByName = buildTypeByName(columns) + for (const { field } of spec) { + validateFieldName(field) + if (!typeByName.has(field)) { + throw new TableQueryValidationError(`Unknown sort column "${field}"`, 'INVALID_ORDER') + } + } +} + +/** + * Validates a STORAGE-keyed predicate — leaf fields are column ids (plus the + * system columns, which keep their names). Runs AFTER wire translation, which + * makes it keying-correct for every caller: a session caller's ids are already + * storage keys, and a workflow tool's names have just been translated — so any + * field left unresolved here is a typo, and on the bulk write paths a typo must + * 400 rather than compile to a filter that silently matches nothing. + */ +export function validateStoragePredicate( + predicate: TablePredicate, + columns: ColumnDefinition[] +): void { + const typeById = new Map(columns.map((c) => [getColumnId(c), c.type])) + for (const [name, type] of SYSTEM_COLUMN_TYPES) typeById.set(name, type) + validateNode(predicate, typeById) +} diff --git a/apps/sim/lib/table/rows/__tests__/cursor.test.ts b/apps/sim/lib/table/rows/__tests__/cursor.test.ts new file mode 100644 index 00000000000..baf9a3434f6 --- /dev/null +++ b/apps/sim/lib/table/rows/__tests__/cursor.test.ts @@ -0,0 +1,96 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { decodeCursor, encodeCursor } from '@/lib/table/rows/cursor' + +describe('cursor codec', () => { + it('encodes a keyset cursor when keyset is valid and round-trips it', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: 'a0' }, + keysetValid: true, + nextOffset: 100, + }) + expect(typeof token).toBe('string') + expect(decodeCursor(token)).toEqual({ after: { orderKey: 'a0', id: 'row-9' } }) + }) + + it('falls back to an offset cursor when keyset is not valid (custom sort / flag off)', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: 'a0' }, + keysetValid: false, + nextOffset: 100, + }) + expect(decodeCursor(token)).toEqual({ offset: 100 }) + }) + + it('emits a compound cursor when the last row is unkeyed but an anchor is known', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: undefined }, + keysetValid: true, + nextOffset: 50, + seekBase: { anchor: { orderKey: 'a5', id: 'row-5' }, offsetFromAnchor: 4 }, + }) + expect(decodeCursor(token)).toEqual({ after: { orderKey: 'a5', id: 'row-5' }, offset: 4 }) + }) + + it('falls back to a whole-view offset when the row has no order key and no anchor', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: undefined }, + keysetValid: true, + nextOffset: 50, + }) + expect(decodeCursor(token)).toEqual({ offset: 50 }) + }) + + it('produces opaque base64url with no raw orderKey/offset leaking', () => { + const token = encodeCursor({ + lastRow: { id: 'r', orderKey: 'zz' }, + keysetValid: true, + nextOffset: 0, + }) + expect(token).not.toContain('orderKey') + expect(token).not.toContain('{') + expect(token).toMatch(/^[A-Za-z0-9_-]+$/) + }) + + it('stamps a version field and rejects any other version', () => { + const token = encodeCursor({ + lastRow: { id: 'r', orderKey: 'a' }, + keysetValid: true, + nextOffset: 0, + }) + expect(JSON.parse(Buffer.from(token, 'base64url').toString('utf8')).v).toBe(1) + // A future/unknown version fails cleanly instead of being misread. + expect(() => decodeCursor(Buffer.from('{"v":2,"o":0}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + // A pre-version token (no `v`) is no longer accepted. + expect(() => decodeCursor(Buffer.from('{"o":0}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) + + it('throws on a malformed cursor', () => { + expect(() => decodeCursor('not-base64-$$$')).toThrow('Invalid cursor') + // Valid base64url but wrong shape. + expect(() => decodeCursor(Buffer.from('{"v":1,"x":1}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) + + it('throws on valid JSON that is not an object (no raw TypeError leak)', () => { + for (const body of ['42', 'null', '"hi"', 'true', '[1,2]']) { + expect(() => decodeCursor(Buffer.from(body).toString('base64url'))).toThrow('Invalid cursor') + } + }) + + it('rejects negative and non-integer offsets', () => { + expect(() => decodeCursor(Buffer.from('{"v":1,"o":-1}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + expect(() => decodeCursor(Buffer.from('{"v":1,"o":1.5}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) +}) diff --git a/apps/sim/lib/table/rows/cursor.test.ts b/apps/sim/lib/table/rows/cursor.test.ts new file mode 100644 index 00000000000..bae39a35c66 --- /dev/null +++ b/apps/sim/lib/table/rows/cursor.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + * + * Opaque cursor encode/decode and the cursor↔sort binding. A cursor encodes a + * position in one specific ordering; replaying it under any other ordering + * silently pages the wrong sequence, so binding violations must throw + * CURSOR_SORT_CONFLICT rather than return wrong rows. + */ +import { describe, expect, it } from 'vitest' +import { TableQueryValidationError } from '@/lib/table/errors' +import { + assertCursorSortBinding, + canonicalSortKey, + decodeCursor, + encodeCursor, +} from '@/lib/table/rows/cursor' + +const ROW = { id: 'row_1', orderKey: 'a1' } + +describe('cursor↔sort binding (bugbot round 2)', () => { + it('stamps an offset cursor with the sort it was minted under', () => { + const token = encodeCursor({ + lastRow: { id: 'row_1', orderKey: null }, + keysetValid: false, + nextOffset: 100, + sort: { col_a: 'desc' }, + }) + const decoded = decodeCursor(token) + expect(decoded.offset).toBe(100) + expect(decoded.sortKey).toBe(canonicalSortKey({ col_a: 'desc' })) + }) + + it('accepts replay under the identical sort', () => { + const decoded = { offset: 100, sortKey: canonicalSortKey({ col_a: 'desc' }) } + expect(() => assertCursorSortBinding(decoded, { col_a: 'desc' })).not.toThrow() + }) + + it('rejects replay under a DIFFERENT sort', () => { + const decoded = { offset: 100, sortKey: canonicalSortKey({ col_a: 'desc' }) } + for (const sort of [{ col_a: 'asc' as const }, { col_b: 'desc' as const }, undefined]) { + expect(() => assertCursorSortBinding(decoded, sort)).toThrow(TableQueryValidationError) + try { + assertCursorSortBinding(decoded, sort) + } catch (e) { + expect((e as TableQueryValidationError).code).toBe('CURSOR_SORT_CONFLICT') + } + } + }) + + it('rejects adding a sort to an unsorted offset cursor', () => { + const token = encodeCursor({ + lastRow: { id: 'row_1', orderKey: null }, + keysetValid: false, + nextOffset: 50, + }) + const decoded = decodeCursor(token) + expect(decoded.sortKey).toBeUndefined() + expect(() => assertCursorSortBinding(decoded, { col_a: 'asc' })).toThrow( + /different sort|sorted query/ + ) + expect(() => assertCursorSortBinding(decoded, undefined)).not.toThrow() + }) + + it('keyset cursors stay default-order only and never carry a sort stamp', () => { + const token = encodeCursor({ lastRow: ROW, keysetValid: true, nextOffset: 10 }) + const decoded = decodeCursor(token) + expect(decoded.after).toEqual({ orderKey: 'a1', id: 'row_1' }) + expect(decoded.sortKey).toBeUndefined() + expect(() => assertCursorSortBinding(decoded, { col_a: 'asc' })).toThrow(/sorted query/) + expect(() => assertCursorSortBinding(decoded, undefined)).not.toThrow() + }) + + it('sort key order is significant (priority is part of the identity)', () => { + expect(canonicalSortKey({ a: 'asc', b: 'desc' })).not.toBe( + canonicalSortKey({ b: 'desc', a: 'asc' }) + ) + }) +}) diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts new file mode 100644 index 00000000000..0d2ab4ab206 --- /dev/null +++ b/apps/sim/lib/table/rows/cursor.ts @@ -0,0 +1,171 @@ +/** + * Opaque pagination cursor for the v2 table-query surface. + * + * The token is a base64url-encoded JSON payload that hides whether paging is + * keyset- or offset-based, so callers (the v2 tools, the agent) only ever echo + * an opaque `cursor` and never juggle `orderKey`/`id`/`offset` themselves. + * + * - Default order → keyset on `(order_key, id)` (`{ k, i }`), an index seek. + * - Sorted views → whole-view offset (`{ o }`), because `(order_key, id)` + * keyset can't seek a data-column ordering. + * - Keyset page whose last row lacks an `orderKey` (rows predating the backfill, + * or forked rows that inherited a NULL key) → compound (`{ k, i, o }`): seek to + * the last keyed anchor, then OFFSET past the unkeyed rows consumed after it. + * This only resolves correctly because the seek admits `order_key IS NULL` + * rows; a bare `(order_key, id) > (…)` excludes them and strands the tail. + */ + +import { TableQueryValidationError } from '@/lib/table/errors' +import type { Sort, TableRow, TableRowsCursor } from '@/lib/table/types' + +/** + * Cursor payload version. Every encoded token carries `v`; decode rejects any + * other value so a future shape change (new `v`) fails cleanly instead of being + * misread against the current field set. + */ +const CURSOR_VERSION = 1 + +type CursorBody = { k: string; i: string } | { o: number } | { k: string; i: string; o: number } +type SortBinding = { s?: string } +type CursorPayload = CursorBody & SortBinding & { v: number } + +/** + * Canonical fingerprint of a sort for cursor binding. Entry order is the sort + * priority (built from the ordered spec upstream), so stringifying entries is + * deterministic for equal sorts and distinct for different ones. + */ +export function canonicalSortKey(sort: Sort | null | undefined): string | undefined { + if (!sort) return undefined + const entries = Object.entries(sort) + return entries.length > 0 ? JSON.stringify(entries) : undefined +} + +/** + * A cursor is only valid for the exact query shape it was minted under: + * keyset/compound cursors encode a position in the DEFAULT `(order_key, id)` + * order, and an offset cursor from a sorted view encodes a position in THAT + * sort. Replaying either against a different ordering silently pages the wrong + * sequence — rows skipped or duplicated with no error. Throws + * `CURSOR_SORT_CONFLICT` so callers restart paging without the cursor. + */ +export function assertCursorSortBinding( + decoded: { after?: TableRowsCursor; offset?: number; sortKey?: string }, + sort: Sort | null | undefined +): void { + const requested = canonicalSortKey(sort) + if (decoded.after && requested !== undefined) { + throw new TableQueryValidationError( + 'Cursor is not valid for a sorted query. Restart paging without the cursor.', + 'CURSOR_SORT_CONFLICT' + ) + } + if ( + decoded.after === undefined && + decoded.offset !== undefined && + decoded.sortKey !== requested + ) { + throw new TableQueryValidationError( + 'Cursor was created under a different sort. Restart paging without the cursor.', + 'CURSOR_SORT_CONFLICT' + ) + } +} + +function invalidCursor(): never { + throw new TableQueryValidationError('Invalid cursor', 'INVALID_CURSOR') +} + +function toBase64Url(json: string): string { + return Buffer.from(json, 'utf8').toString('base64url') +} + +function fromBase64Url(token: string): string { + return Buffer.from(token, 'base64url').toString('utf8') +} + +/** + * Builds the cursor for the page *after* `lastRow`. + * + * Shape selection: + * 1. `keysetValid` and the row carries an `orderKey` → `{ k, i }`. + * 2. `keysetValid` with a known prior anchor (last row unkeyed) → `{ k, i, o }`. + * 3. Otherwise → `{ o: nextOffset }` (whole-view offset). + * + * `keysetValid` must only be true when the `(order_key, id)` index order is + * authoritative for the page: no custom sort AND fractional ordering enabled. + * Passing false forces the offset shape, which is correct under any ordering. + */ +export function encodeCursor(args: { + lastRow: Pick + keysetValid: boolean + nextOffset: number + seekBase?: { anchor: TableRowsCursor; offsetFromAnchor: number } + /** The sort the page was produced under — stamps offset cursors so they can't be replayed against a different ordering. */ + sort?: Sort | null +}): string { + let body: CursorBody + if (args.keysetValid && args.lastRow.orderKey) { + body = { k: args.lastRow.orderKey, i: args.lastRow.id } + } else if (args.seekBase) { + // An anchor is in effect (inbound seek or last keyed row) but a plain + // keyset can't stand alone — resume by seeking the anchor then offsetting + // past the rows consumed after it. Never valid under a custom sort, where + // callers must not pass a seekBase. + body = { + k: args.seekBase.anchor.orderKey, + i: args.seekBase.anchor.id, + o: args.seekBase.offsetFromAnchor, + } + } else { + body = { o: args.nextOffset } + } + const sortKey = canonicalSortKey(args.sort) + const payload: CursorPayload = { + ...body, + // Only the pure-offset shape can exist under a custom sort; keyset and + // compound shapes are default-order by construction and carry no binding. + ...('k' in body || sortKey === undefined ? {} : { s: sortKey }), + v: CURSOR_VERSION, + } + return toBase64Url(JSON.stringify(payload)) +} + +/** Decodes an opaque cursor into the `queryRows` paging inputs it stands for. */ +export function decodeCursor(token: string): { + after?: TableRowsCursor + offset?: number + /** Sort fingerprint an offset cursor was minted under; absent = default order. */ + sortKey?: string +} { + let payload: unknown + try { + payload = JSON.parse(fromBase64Url(token)) + } catch { + invalidCursor() + } + if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) { + invalidCursor() + } + + const record = payload as Record + if (record.v !== CURSOR_VERSION) invalidCursor() + const hasKeyset = typeof record.k === 'string' && typeof record.i === 'string' + const hasOffset = typeof record.o === 'number' && Number.isInteger(record.o) && record.o >= 0 + + if (hasKeyset && hasOffset) { + return { + after: { orderKey: record.k as string, id: record.i as string }, + offset: record.o as number, + } + } + if (hasKeyset) { + return { after: { orderKey: record.k as string, id: record.i as string } } + } + if (hasOffset) { + return { + offset: record.o as number, + ...(typeof record.s === 'string' ? { sortKey: record.s } : {}), + } + } + invalidCursor() +} diff --git a/apps/sim/lib/table/rows/paging.ts b/apps/sim/lib/table/rows/paging.ts deleted file mode 100644 index a96b98c041f..00000000000 --- a/apps/sim/lib/table/rows/paging.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Cuts a fetched page to a byte budget: keeps the longest prefix of rows whose - * serialized `data` fits within `maxBytes`, always keeping at least one row so - * pagination makes forward progress even when a single row exceeds the budget. - * - * The budget counts `data` only — the per-row envelope (`id`, `position`, - * `orderKey`, timestamps, executions) is not measured, so actual response - * payloads run slightly over `maxBytes`. Callers must leave headroom; the - * production SQL-side cut should account for the same overhead. - */ -export function trimRowsToByteBudget( - rows: T[], - maxBytes: number -): T[] { - let total = 0 - let kept = 0 - for (const row of rows) { - total += Buffer.byteLength(JSON.stringify(row.data)) - if (kept > 0 && total > maxBytes) break - kept++ - } - return kept === rows.length ? rows : rows.slice(0, kept) -} diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 41eedd066f8..dfcc3a4c282 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -25,6 +25,7 @@ import { } from '@/lib/table/billing' import { getColumnId } from '@/lib/table/column-keys' import { getMaxPageBytes, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' import { assertRowDelete, assertRowInsert, @@ -32,7 +33,13 @@ import { patchColumnIds, } from '@/lib/table/mutation-locks' import { nKeysBetween } from '@/lib/table/order-key' -import { type DbExecutor, type DbTransaction, withSeqscanOff } from '@/lib/table/planner' +import { + type DbExecutor, + type DbTransaction, + withReadGuards, + withSeqscanOff, +} from '@/lib/table/planner' +import { encodeCursor } from '@/lib/table/rows/cursor' import { applyExecutionsPatch, deriveExecClearsForDataPatch, @@ -49,8 +56,13 @@ import { resolveBatchInsertOrderKeys, resolveInsertOrderKey, } from '@/lib/table/rows/ordering' -import { trimRowsToByteBudget } from '@/lib/table/rows/paging' -import { buildFilterClause, buildSortClause, escapeLikePattern } from '@/lib/table/sql' +import { + buildFilterClause, + buildPredicateClause, + buildSortClause, + escapeLikePattern, + fieldPredicate, +} from '@/lib/table/sql' import { fireTableTrigger } from '@/lib/table/trigger' import { scaledStatementTimeoutMs, setTableTxTimeouts } from '@/lib/table/tx' import type { @@ -75,6 +87,7 @@ import type { TableDefinition, TableDeleteJobPayload, TableRow, + TableRowsCursor, UpdateRowData, UpsertResult, UpsertRowData, @@ -421,15 +434,19 @@ export async function replaceTableRowsWithTx( if (uniqueColumns.length > 0 && data.rows.length > 0) { const seen = new Map>() for (const col of uniqueColumns) { - seen.set(col.name, new Map()) + seen.set(getColumnId(col), new Map()) } for (let i = 0; i < data.rows.length; i++) { const row = data.rows[i] for (const col of uniqueColumns) { - const value = row[col.name] + // Coerced rows are keyed by column id, not display name — reading + // `row[col.name]` silently misses renamed columns and lets dupes through. + const colId = getColumnId(col) + const value = row[colId] if (value === null || value === undefined) continue - const normalized = typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value) - const map = seen.get(col.name)! + // Case-sensitive, consistent with the unique-constraint check leaf. + const normalized = typeof value === 'string' ? value : JSON.stringify(value) + const map = seen.get(colId)! if (map.has(normalized)) { throw new Error( `Row ${i + 1}: Column "${col.name}" must be unique. Value "${String(value)}" duplicates row ${map.get(normalized)! + 1} in batch` @@ -568,12 +585,21 @@ export async function upsertRow( throw new Error(`Upsert requires a value for the conflict target column "${targetColumnName}"`) } - // `data->` and `data->>` accept the JSON key as a parameterized text value; - // no need for `sql.raw` interpolation. - const matchFilter = - typeof targetValue === 'string' - ? sql`${userTableRows.data}->>${targetColumnKey}::text = ${String(targetValue)}` - : sql`(${userTableRows.data}->${targetColumnKey}::text)::jsonb = ${JSON.stringify(targetValue)}::jsonb` + // Build the conflict probe through the SAME leaf as the unique-constraint check + // (`fieldPredicate` → case-sensitive JSONB containment). This is what makes + // "find the row to update" and "is this value unique" agree: a value differing + // only in case can no longer slip past the probe and then trip the guard. + // `eq` always yields a clause for a non-null value (guaranteed above). + const matchFilter = fieldPredicate( + USER_TABLE_ROWS_SQL_NAME, + targetColumnKey, + 'eq', + targetValue, + table.schema.columns.find((c) => getColumnId(c) === targetColumnKey) + ) + if (!matchFilter) { + throw new Error('Failed to build upsert conflict predicate') + } // Resolve the plan limit BEFORE the tx (the lookup is a separate pool read; doing // it inside the tx would hold a connection + the row-order lock during it). The @@ -1005,8 +1031,11 @@ export async function queryRows( ): Promise { const { filter, + predicate, sort, - limit = TABLE_LIMITS.DEFAULT_QUERY_LIMIT, + // No default: an undefined limit returns every matching row, bounded only by + // the MAX_QUERY_RESULT_BYTES fail-fast guard below. + limit, offset = 0, after, includeTotal = true, @@ -1026,61 +1055,61 @@ export async function queryRows( deleteMask ) + // v2 predicate takes precedence over the legacy `$`-filter; both compile to a + // WHERE through the same `fieldPredicate` leaf. + const userClause = predicate + ? buildPredicateClause(predicate, tableName, columns) + : filter && Object.keys(filter).length > 0 + ? buildFilterClause(filter, tableName, columns) + : undefined + let whereClause = baseConditions - if (filter && Object.keys(filter).length > 0) { - const filterClause = buildFilterClause(filter, tableName, columns) - if (filterClause) { - whereClause = and(baseConditions, filterClause) - } + if (userClause) { + whereClause = and(baseConditions, userClause) } - // Keyset page: seek past the cursor on the default `(order_key, id)` order instead of paying - // OFFSET's scan-and-discard of every prior row (O(N²) across a deep scroll / full drain). Only - // valid without a custom sort — the contract rejects `after` + `sort` together. The count below - // deliberately excludes the cursor: totals cover the whole view, not the remaining pages. - const pageWhere = - after && !sort - ? and( - whereClause, - sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id})` - ) - : whereClause + // Keyset seeks are only authoritative when the page order IS the + // `(order_key, id)` index order: no custom sort. Order keys are NOT guaranteed + // present — unkeyed rows sort last and the seek admits them explicitly. + const keysetValid = !sort - const buildPageQuery = (executor: DbExecutor) => { - const query = executor - .select() - .from(userTableRows) - .where(pageWhere ?? baseConditions) - .orderBy(buildRowOrderBySql(sort, tableName, columns)) - return after ? query.limit(limit) : query.limit(limit).offset(offset) - } - - // Count and page fetch are independent reads — run them concurrently so the + // Count and page drain are independent reads — run them concurrently so the // `includeTotal` hot path doesn't pay two serial round-trips. Filtered counts // go through the tenant-bounded variant (see countRowsTenantBounded); the // unfiltered count already plans an index-only scan on the table_id prefix. - // Custom column sorts order by `data->>'col'` — unestimatable, so left alone - // the planner seq-scans and sorts the whole shared relation on every page - // (9.7s measured on a 1M-row table; 0.76s tenant-bounded). Default-order - // pages already stream the `(table_id, order_key, id)` index. - const hasFilter = Boolean(filter && Object.keys(filter).length > 0) - const rowsPromise = sort ? withSeqscanOff(async (trx) => buildPageQuery(trx)) : buildPageQuery(db) + // The count uses the full-view WHERE (no cursor seek): totals cover the whole + // view, not the remaining pages. + const hasFilter = Boolean(userClause) const countPromise = includeTotal ? hasFilter ? countRowsTenantBounded(whereClause) - : db - .select({ count: count() }) - .from(userTableRows) - .where(whereClause ?? baseConditions) - .then((r) => Number(r[0].count)) + : // Unfiltered count plans an index-only scan on the table_id prefix, but + // still runs under the read timeout so it can't pin a connection. + withReadGuards(async (trx) => { + const [r] = await trx + .select({ count: count() }) + .from(userTableRows) + .where(whereClause ?? baseConditions) + return Number(r.count) + }) : null - const [fetchedRows, totalCount] = await Promise.all([rowsPromise, countPromise]) + // The inbound seek is honored whenever there's no custom sort; `keysetValid` + // only gates re-anchoring and plain-keyset cursor emission. + const drainPromise = fetchRowsBounded({ + baseWhere: whereClause ?? baseConditions, + orderBy: buildRowOrderBySql(sort, tableName, columns), + sorted: Boolean(sort), + keysetValid, + seek: !sort ? after : undefined, + startOffset: offset, + limit, + budgetBytes: TABLE_LIMITS.MAX_QUERY_RESULT_BYTES, + pageCutBytes: getMaxPageBytes() ?? undefined, + }) - // Dev-preview byte cut (TABLE_MAX_PAGE_BYTES, off by default): clients terminate on - // empty page / totalCount, never page fullness, so a short page is safe to return. - const maxPageBytes = getMaxPageBytes() - const rows = maxPageBytes === null ? fetchedRows : trimRowsToByteBudget(fetchedRows, maxPageBytes) + const [fetched, totalCount] = await Promise.all([drainPromise, countPromise]) + const rows = fetched.rows const executionsByRow = withExecutions ? await loadExecutionsByRow( @@ -1090,23 +1119,218 @@ export async function queryRows( : null logger.info( - `[${requestId}] Queried ${rows.length} rows from table ${table.id} (total: ${totalCount})` + `[${requestId}] Queried ${rows.length} rows from table ${table.id} (total: ${totalCount}, bytes: ${fetched.bytes}, more: ${fetched.hasMore})` ) + const mappedRows = rows.map((r) => ({ + id: r.id, + data: r.data as RowData, + executions: executionsByRow?.get(r.id) ?? {}, + position: r.position, + orderKey: r.orderKey ?? undefined, + createdAt: r.createdAt, + updatedAt: r.updatedAt, + })) + + // Opaque next-page cursor: non-null whenever more matching rows exist beyond + // this page — whether the page was cut by `limit` or by the byte budget. The + // drain loop proves `hasMore` with a fetched-but-unreturned witness row. + const lastRow = mappedRows[mappedRows.length - 1] + const nextCursor = + fetched.hasMore && lastRow + ? encodeCursor({ + lastRow, + keysetValid, + nextOffset: offset + mappedRows.length, + seekBase: fetched.anchor + ? { anchor: fetched.anchor, offsetFromAnchor: fetched.anchorOffset } + : undefined, + sort, + }) + : null + return { - rows: rows.map((r) => ({ - id: r.id, - data: r.data as RowData, - executions: executionsByRow?.get(r.id) ?? {}, - position: r.position, - orderKey: r.orderKey ?? undefined, - createdAt: r.createdAt, - updatedAt: r.updatedAt, - })), + rows: mappedRows, rowCount: rows.length, totalCount, - limit, + limit: limit ?? mappedRows.length, offset, + nextCursor, + } +} + +interface BoundedFetchParams { + /** Tenant + delete-mask + user filter — WITHOUT any seek predicate. */ + baseWhere: SQL | undefined + orderBy: SQL + /** Custom sort present → per-batch withSeqscanOff (JSONB order is unestimatable). */ + sorted: boolean + /** `(order_key, id)` order is authoritative → keyset re-anchoring + seeks. */ + keysetValid: boolean + /** Inbound seek anchor (decoded keyset/compound cursor), if any. */ + seek?: TableRowsCursor + /** Inbound offset — the whole-view offset, or the past-anchor offset of a compound cursor. */ + startOffset: number + limit?: number + /** Drain ceiling: sizes batches, and the fail-fast bound for an unbounded query. */ + budgetBytes: number + /** + * Opt-in byte cut for a **bounded** page (`TABLE_MAX_PAGE_BYTES`); `undefined` + * disables it, so a bounded page always returns its full `limit`. + */ + pageCutBytes?: number +} + +interface BoundedFetchResult { + rows: Array + bytes: number + /** Proven by a fetched-but-unreturned witness row — never inferred from page fullness. */ + hasMore: boolean + /** Final keyset anchor, for compound-cursor emission when the last row is unkeyed. */ + anchor?: TableRowsCursor + /** Rows consumed past `anchor` (0 when the anchor is the last returned row). */ + anchorOffset: number +} + +/** Belt-and-braces bound on drain iterations; unreachable in practice. */ +const MAX_QUERY_BATCHES = 1000 + +/** + * Drains rows in adaptively-sized bounded batches until the caller's `limit` + * or the byte ceiling ends the page. Never issues an unbounded SELECT: the + * first batch is capped so its worst-case bytes stay within ~4× the budget at + * the max row size, and later batches are sized from the observed average. + * + * Byte ceiling: an **unbounded** query (no `limit`) always fails fast at + * `budgetBytes` — returning part of a result that promised everything would be + * silent truncation. A **bounded** page cuts short only when `pageCutBytes` is + * set (`TABLE_MAX_PAGE_BYTES`), because a short page is only safe for clients + * that terminate on `nextCursor === null` rather than on page fullness. + * + * Advance strategy: when `keysetValid`, the loop re-anchors on each consumed + * keyed row and seeks `(order_key, id) > (anchor)` — delete-tolerant and an + * index seek. Otherwise (custom sort, flag off) it advances by OFFSET from the + * inbound position; rows deleted mid-drain can skip/duplicate exactly as + * cross-request offset paging already does. + * + * Always returns at least one row when any match exists, even if that row + * alone exceeds the budget. + */ +async function fetchRowsBounded(params: BoundedFetchParams): Promise { + const { baseWhere, orderBy, sorted, keysetValid, limit, budgetBytes, pageCutBytes } = params + + const firstBatchCap = Math.max(1, Math.floor((4 * budgetBytes) / TABLE_LIMITS.MAX_ROW_SIZE_BYTES)) + + // The byte ceiling that ends the drain: an unbounded query fails fast at the + // budget; a bounded page cuts only when the operator opted in. + const cutBytes = limit === undefined ? budgetBytes : pageCutBytes + + const rows: Array = [] + let bytes = 0 + let maxRowBytes = 0 + let hasMore = false + let anchor = params.seek + let anchorOffset = params.startOffset + let consumedSinceAnchor = 0 + + const nextBatchRows = (): number => { + if (rows.length === 0) return Math.min(limit ?? firstBatchCap, firstBatchCap) + const avg = Math.max(1, bytes / rows.length) + // Bytes we may still fetch this batch. When a cut is active it's the + // remainder of that cut; otherwise each batch gets a fresh budget's worth, + // so a large bounded page keeps draining in real steps instead of degrading + // to one row per query once cumulative bytes pass the budget. + const remaining = Math.max(1, cutBytes === undefined ? budgetBytes : cutBytes - bytes) + const byAverage = Math.ceil(remaining / avg) + 1 + const varianceCap = Math.ceil((8 * remaining) / Math.max(maxRowBytes, 1)) + return Math.max(1, Math.min(byAverage, TABLE_LIMITS.QUERY_BATCH_MAX_ROWS, varianceCap)) + } + + const runBatch = (batchSeek: TableRowsCursor | undefined, batchOffset: number, ask: number) => { + const buildQuery = (executor: DbExecutor) => { + // `order_key` is nullable (rows predating the backfill, and forked rows that + // inherit a NULL key). A bare row-constructor comparison evaluates to NULL for + // those rows, so they are dropped by WHERE — and because NULLs sort LAST under + // `ORDER BY order_key, id`, the entire unkeyed tail becomes unreachable and the + // drain terminates early reporting `hasMore: false`. Admitting NULLs keeps the + // seek set exactly "the tail after the anchor", which is also what the compound + // `{k,i,o}` cursor's `offsetFromAnchor` accounting assumes. + const seekWhere = batchSeek + ? and( + baseWhere, + sql`(${userTableRows.orderKey} IS NULL OR (${userTableRows.orderKey}, ${userTableRows.id}) > (${batchSeek.orderKey}, ${batchSeek.id}))` + ) + : baseWhere + const query = executor + .select() + .from(userTableRows) + .where(seekWhere) + .orderBy(orderBy) + .limit(ask) + return batchOffset > 0 ? query.offset(batchOffset) : query + } + // One tx per batch (SET LOCAL dies with it; holding a tx across JS + // accounting between batches would pin a pooled connection). Custom sorts + // order by `data->>'col'` — unestimatable — so they also penalize seq scans + // (9.7s→0.76s on a 1M-row table); default-order pages stream the index and + // just need the read timeout. Either way the batch runs under a statement + // timeout so a pathological filter can't scan unbounded. + return withReadGuards(async (trx) => buildQuery(trx), { seqscanOff: sorted }) + } + + for (let iteration = 0; iteration < MAX_QUERY_BATCHES; iteration++) { + const limitRemaining = limit === undefined ? Number.POSITIVE_INFINITY : limit - rows.length + const target = Math.min(nextBatchRows(), limitRemaining) + const ask = target + 1 // +1 = witness row proving more data exists past a cut + const batch = await runBatch(anchor, anchorOffset + consumedSinceAnchor, ask) + if (batch.length === 0) break + + let cut = false + for (const row of batch) { + const rowBytes = Buffer.byteLength(JSON.stringify(row.data)) + if (cutBytes !== undefined && rows.length > 0 && bytes + rowBytes > cutBytes) { + // Unbounded queries promise the ENTIRE result — a partial page would be + // silent truncation, so fail fast instead (the drain has only fetched + // ~budget bytes at this point, never the whole table). + if (limit === undefined) { + throw new TableQueryValidationError( + `Query result exceeds the ${Math.floor(cutBytes / (1024 * 1024))}MB limit. Add a filter or a limit to narrow the result.`, + 'TABLE_QUERY_RESULT_TOO_LARGE' + ) + } + // Bounded page, byte cut opted in: `row` is the witness. Requires a + // non-empty page so a single over-budget row is still returned alone. + hasMore = true + cut = true + break + } + // Limit cut: `row` is the +1 peek witness. + if (rows.length === limit) { + hasMore = true + cut = true + break + } + rows.push(row) + bytes += rowBytes + consumedSinceAnchor++ + if (rowBytes > maxRowBytes) maxRowBytes = rowBytes + if (keysetValid && row.orderKey) { + anchor = { orderKey: row.orderKey, id: row.id } + anchorOffset = 0 + consumedSinceAnchor = 0 + } + } + if (cut) break + // Short batch = the source is exhausted; hasMore stays false. + if (batch.length < ask) break + } + + return { + rows, + bytes, + hasMore, + anchor, + anchorOffset: anchorOffset + consumedSinceAnchor, } } @@ -1419,17 +1643,22 @@ export async function updateRowsByFilter( eq(userTableRows.workspaceId, table.workspaceId) ) + // A limit selects a SUBSET, so impose the default `(order_key, id)` order — + // without it Postgres returns planner-arbitrary rows and "update the first N" + // is nondeterministic. Sort is irrelevant (and skipped) when every match is updated. // Tenant-bounded: the jsonb filter is unestimatable and otherwise sends the planner to a // whole-shared-relation seq scan (14.4s measured on a 1M-row table). const matchingRows = await withSeqscanOff(async (trx) => { - let query = trx + const base = trx .select({ id: userTableRows.id, data: userTableRows.data }) .from(userTableRows) .where(and(baseConditions, filterClause)) if (data.limit) { - query = query.limit(data.limit) as typeof query + return base + .orderBy(buildRowOrderBySql(undefined, tableName, table.schema.columns)) + .limit(data.limit) } - return query + return base }) if (matchingRows.length === 0) { @@ -1785,16 +2014,20 @@ export async function deleteRowsByFilter( eq(userTableRows.workspaceId, table.workspaceId) ) + // A limit deletes a SUBSET, so order deterministically by `(order_key, id)` — + // see updateRowsByFilter. Unbounded deletes affect every match, so order is moot. // Tenant-bounded for the same reason as updateRowsByFilter — see withSeqscanOff. const matchingRows = await withSeqscanOff(async (trx) => { - let query = trx + const base = trx .select({ id: userTableRows.id, position: userTableRows.position }) .from(userTableRows) .where(and(baseConditions, filterClause)) if (data.limit) { - query = query.limit(data.limit) as typeof query + return base + .orderBy(buildRowOrderBySql(undefined, tableName, table.schema.columns)) + .limit(data.limit) } - return query + return base }) if (matchingRows.length === 0) { diff --git a/apps/sim/lib/table/select-values.test.ts b/apps/sim/lib/table/select-values.test.ts index 7c4d4b456da..041b2f61242 100644 --- a/apps/sim/lib/table/select-values.test.ts +++ b/apps/sim/lib/table/select-values.test.ts @@ -2,7 +2,11 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { resolveFilterSelectValues, selectValueToNames } from '@/lib/table/select-values' +import { + resolveFilterSelectValues, + resolvePredicateSelectValues, + selectValueToNames, +} from '@/lib/table/select-values' import type { ColumnDefinition } from '@/lib/table/types' const status: ColumnDefinition = { @@ -100,3 +104,49 @@ describe('resolveFilterSelectValues', () => { ).toEqual({ $or: [{ col_status: 'opt_open' }, { col_title: 'x' }] }) }) }) + +/** + * The block builder serializes without schema access, so an option NAME that + * looks numeric or boolean arrives scalar-coerced ("123" → 123). Resolution + * must still find the option, or a correctly-authored builder filter compares + * a number against the stored id string and matches nothing. + */ +describe('resolvePredicateSelectValues — scalar-coerced option names', () => { + const numericStatus: ColumnDefinition = { + id: 'col_code', + name: 'code', + type: 'select', + options: [ + { id: 'opt_123', name: '123' }, + { id: 'opt_true', name: 'true' }, + ], + } + const columns = [numericStatus] + + it('resolves a coerced numeric name to its option id', () => { + expect( + resolvePredicateSelectValues({ all: [{ field: 'col_code', op: 'eq', value: 123 }] }, columns) + ).toEqual({ all: [{ field: 'col_code', op: 'eq', value: 'opt_123' }] }) + }) + + it('resolves a coerced boolean name, including inside in/contains', () => { + expect( + resolvePredicateSelectValues( + { any: [{ field: 'col_code', op: 'in', value: [true, 123] }] }, + columns + ) + ).toEqual({ any: [{ field: 'col_code', op: 'in', value: ['opt_true', 'opt_123'] }] }) + expect( + resolvePredicateSelectValues( + { all: [{ field: 'col_code', op: 'contains', value: 123 }] }, + columns + ) + ).toEqual({ all: [{ field: 'col_code', op: 'contains', value: 'opt_123' }] }) + }) + + it('leaves a scalar with no matching option name as-is', () => { + expect( + resolvePredicateSelectValues({ all: [{ field: 'col_code', op: 'eq', value: 999 }] }, columns) + ).toEqual({ all: [{ field: 'col_code', op: 'eq', value: 999 }] }) + }) +}) diff --git a/apps/sim/lib/table/select-values.ts b/apps/sim/lib/table/select-values.ts index 8addc65140d..c120a989ed7 100644 --- a/apps/sim/lib/table/select-values.ts +++ b/apps/sim/lib/table/select-values.ts @@ -5,11 +5,22 @@ * * Row-level id→name translation lives in `cell-format.ts`, which fuses it with * the column key translation. What remains here is the reverse direction: a - * filter operand typed as an option name resolving back to the stored id. + * filter operand typed as an option name resolving back to the stored id — in + * both the legacy `$` grammar and the v2 predicate tree. */ -import { getColumnId } from '@/lib/table/column-keys' -import type { ColumnDefinition, ConditionOperators, Filter, JsonValue } from '@/lib/table/types' +import { buildIdByName, getColumnId, predicateNamesToIds } from '@/lib/table/column-keys' +import type { + ColumnDefinition, + ConditionOperators, + Filter, + FilterOp, + JsonValue, + Predicate, + PredicateNode, + TablePredicate, + TableSchema, +} from '@/lib/table/types' import { resolveSelectOptionId } from '@/lib/table/validation' /** @@ -96,3 +107,67 @@ export function resolveFilterSelectValues(filter: Filter, columns: ColumnDefinit } return walk(filter) } + +/** + * Predicate-grammar sibling of {@link resolveFilterSelectValues}: returns a copy + * of an id-keyed predicate tree with `select` leaf values resolved from option + * name → id. + * + * Without it a v2 filter written against a select column (`{field:'status', + * op:'eq', value:'Open'}`) compares the option NAME against the stored option + * ID and silently matches nothing. + * + * Mirrors the `$`-grammar set exactly: equality/membership on a single select + * (`eq`/`ne`/`in`/`nin`) AND `contains`/`ncontains`, which on a MULTI-select are + * not pattern matches at all — the cell is an array of option ids, so those ops + * express membership and their operand is an option, not a substring. Leaving + * them out is what made a correctly-formed multi-select filter match nothing. + * The remaining pattern ops (`like`, `startsWith`, …) are rejected on select + * columns by `fieldPredicate`'s allowlist, so they never reach here. The + * valueless ops carry nothing to resolve. + */ +export function resolvePredicateSelectValues( + predicate: TablePredicate, + columns: ColumnDefinition[] +): TablePredicate { + const selectById = new Map( + columns.filter((c) => c.type === 'select').map((c) => [getColumnId(c), c]) + ) + if (selectById.size === 0) return predicate + + const RESOLVED_OPS = new Set(['eq', 'ne', 'in', 'nin', 'contains', 'ncontains']) + + const walk = (node: PredicateNode): PredicateNode => { + if ('all' in node) return { all: node.all.map(walk) } + if ('any' in node) return { any: node.any.map(walk) } + + const leaf = node as Predicate + const column = selectById.get(leaf.field) + if (!column || leaf.value === undefined || !RESOLVED_OPS.has(leaf.op)) return leaf + + const options = column.options + const value = Array.isArray(leaf.value) + ? leaf.value.map((v) => resolveOperand(v as JsonValue, options)) + : resolveOperand(leaf.value as JsonValue, options) + return { ...leaf, value: value as Predicate['value'] } + } + + return walk(predicate) as TablePredicate +} + +/** + * The complete name-keyed → storage-keyed translation for a v2 predicate: + * column names become column ids AND select operands become option ids. + * + * Both halves are required and neither is useful alone, but they lived as two + * separate calls that every boundary had to remember to pair — and three of them + * did not, so a filter on a select column compared an option NAME against a + * stored option ID and returned zero rows while reporting success. Call this + * instead of `predicateNamesToIds` so the pair cannot be split again. + */ +export function predicateToStorage(predicate: TablePredicate, schema: TableSchema): TablePredicate { + return resolvePredicateSelectValues( + predicateNamesToIds(predicate, buildIdByName(schema)), + schema.columns + ) +} diff --git a/apps/sim/lib/table/snapshot-cache.ts b/apps/sim/lib/table/snapshot-cache.ts index 0c24eccfb39..e99869b4adf 100644 --- a/apps/sim/lib/table/snapshot-cache.ts +++ b/apps/sim/lib/table/snapshot-cache.ts @@ -103,7 +103,9 @@ async function materialize(table: TableDefinition, key: string): Promise bytes += Buffer.byteLength(header) await handle.write(header) - let after: { orderKey: string; id: string } | null = null + // `order_key` is nullable (rows predating the backfill), and the page query + // seeks NULLs explicitly — so the cursor has to carry a null too. + let after: { orderKey: string | null; id: string } | null = null while (true) { const page = await selectExportRowPage(table, after, SNAPSHOT_BATCH_SIZE) if (page.length === 0) break diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index 3b254ad3b17..7ea5a6de3f0 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -10,25 +10,19 @@ import type { SQL } from 'drizzle-orm' import { sql } from 'drizzle-orm' import { getColumnId } from '@/lib/table/column-keys' import { NAME_PATTERN } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' import type { ColumnDefinition, ConditionOperators, Filter, + FilterOp, JsonValue, + Predicate, + PredicateNode, Sort, + TablePredicate, } from '@/lib/table/types' -/** - * Error thrown when caller-supplied filter or sort input is malformed. - * Routes should map this to HTTP 400 with the message preserved. - */ -export class TableQueryValidationError extends Error { - constructor(message: string) { - super(message) - this.name = 'TableQueryValidationError' - } -} - type ColumnType = ColumnDefinition['type'] type ColumnMap = ReadonlyMap @@ -43,6 +37,33 @@ type ColumnMap = ReadonlyMap export const SINGLE_SELECT_OPERATORS = new Set(['$eq', '$ne', '$in', '$nin', '$empty']) export const MULTI_SELECT_OPERATORS = new Set(['$contains', '$ncontains', '$empty']) +/** + * The same allowlists in the v2 bare-operator grammar, applied inside + * `fieldPredicate` so both wire formats gate identically. Not derived from the + * `$` sets above by string surgery because the mapping is not 1:1 — `$empty` + * splits into `isEmpty`/`isNotEmpty`. `isNull`/`isNotNull` have no `$` + * equivalent and are allowed on both: a strict null check is meaningful on any + * column, select included. + */ +const SINGLE_SELECT_OPS = new Set([ + 'eq', + 'ne', + 'in', + 'nin', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +]) +const MULTI_SELECT_OPS = new Set([ + 'contains', + 'ncontains', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +]) + /** * Returns the Postgres cast needed to compare a JSONB text value of the given * column type, or `null` when text comparison is correct. Single source of @@ -87,7 +108,13 @@ const ALLOWED_OPERATORS = new Set([ '$ncontains', '$startsWith', '$endsWith', + '$like', + '$ilike', + '$nlike', + '$nilike', '$empty', + '$isNull', + '$isNotNull', ]) /** @@ -169,6 +196,18 @@ function buildFilterClauseInternal( } // Skip arrays for regular fields - arrays are only valid for $or and $and. + // A v2 predicate tree (`{ all | any: [...] }`) that reaches this legacy + // compiler is a VERSION MISMATCH — a caller speaking the newer grammar + // against an older server. Skipping it as "an array on a regular field" + // compiles to no WHERE clause at all, which on a bulk delete means every + // row rather than none. Fail fast and name the mismatch instead. + if ((field === 'all' || field === 'any') && Array.isArray(condition)) { + throw new TableQueryValidationError( + `Filter looks like a v2 predicate tree ("${field}" group) but reached the legacy filter compiler. ` + + 'This usually means a client is sending the predicate grammar to a server that predates it.' + ) + } + // If we encounter an array here, it's likely malformed input (e.g., { name: [filter1, filter2] }) // which doesn't have a clear semantic meaning, so we skip it. if (Array.isArray(condition)) { @@ -191,6 +230,48 @@ function buildFilterClauseInternal( return sql.join(conditions, sql.raw(' AND ')) } +/** + * Builds a WHERE clause from a v2 `TablePredicate` (nestable `all`/`any` groups + * of `{ field, op, value }` leaves). Sibling of `buildFilterClause`: same engine, + * same `fieldPredicate` leaf — only the grammar differs. Returns `undefined` when + * the tree contributes no conditions (empty groups, all-no-op leaves). + * + * @throws {TableQueryValidationError} if a field name is invalid or an operator is not allowed + */ +export function buildPredicateClause( + predicate: TablePredicate, + tableName: string, + columns: ColumnDefinition[] +): SQL | undefined { + return buildPredicateNode(predicate, tableName, buildColumnMap(columns)) +} + +function isPredicateGroup(node: PredicateNode): node is TablePredicate { + return 'all' in node || 'any' in node +} + +function buildPredicateNode( + node: PredicateNode, + tableName: string, + columnMap: ColumnMap +): SQL | undefined { + if (isPredicateGroup(node)) { + const isAll = 'all' in node + const members = isAll ? node.all : node.any + const clauses: SQL[] = [] + for (const member of members) { + const clause = buildPredicateNode(member, tableName, columnMap) + if (clause) clauses.push(clause) + } + if (clauses.length === 0) return undefined + if (clauses.length === 1) return clauses[0] + return sql`(${sql.join(clauses, sql.raw(isAll ? ' AND ' : ' OR '))})` + } + + const leaf = node as Predicate + return fieldPredicate(tableName, leaf.field, leaf.op, leaf.value, columnMap.get(leaf.field)) +} + /** * Builds an ORDER BY clause from a sort object. * @@ -328,7 +409,8 @@ function buildFieldCondition( if (isRecordLike(condition)) { for (const [op, value] of Object.entries(condition)) { - // Validate operator to ensure only allowed operators are used + // Validate against the legacy `$`-whitelist, then normalize onto the shared + // `FilterOp` so v1 and v2 emit byte-identical leaf SQL. validateOperator(op) // Select values are opaque option ids — range/pattern operators are meaningless. if (isSelect) { @@ -340,120 +422,170 @@ function buildFieldCondition( } } - switch (op) { - case '$eq': - conditions.push(buildContainmentClause(tableName, field, value as JsonValue)) - break + if (op === '$empty') { + // `$empty: true/false` maps onto the valueless v2 ops. + const filterOp: FilterOp = coerceEmptyFlag(field, value) ? 'isEmpty' : 'isNotEmpty' + const clause = fieldPredicate(tableName, field, filterOp, undefined, column) + if (clause) conditions.push(clause) + continue + } - case '$ne': - conditions.push( - sql`NOT (${buildContainmentClause(tableName, field, value as JsonValue)})` - ) - break + // Every other `$op` is `op` minus the leading `$` (e.g. `$gte` → `gte`). + const clause = fieldPredicate(tableName, field, op.slice(1) as FilterOp, value, column) + if (clause) conditions.push(clause) + } + } else { + // Simple value (primitive or null) - shorthand for equality. + // Example: { name: 'John' } is equivalent to { name: { $eq: 'John' } } + // isRecordLike's negation can't structurally exclude ConditionOperators (no index + // signature), so the JsonValue-only shape of this branch is asserted, not inferred. + // Routes through the unified `fieldPredicate` leaf like every other matcher, + // so equality semantics stay defined in exactly one place. + // + // On a multi-select the shorthand reads as "holds this option" — the cell is + // an array of option ids, so scalar equality can never be true. It maps to + // `contains` (membership). An EXPLICIT `$eq` on a multi-select still errors + // via the select allowlist: writing it out is a mistake worth naming, while + // the shorthand has an unambiguous intent. + const shorthandOp: FilterOp = column?.type === 'select' && column.multiple ? 'contains' : 'eq' + const clause = fieldPredicate(tableName, field, shorthandOp, condition as JsonValue, column) + if (clause) conditions.push(clause) + } - case '$gt': - conditions.push( - buildComparisonClause(tableName, field, '>', value as number | string, columnType) - ) - break + return conditions +} - case '$gte': - conditions.push( - buildComparisonClause(tableName, field, '>=', value as number | string, columnType) - ) - break +/** + * The single leaf primitive: compiles one `field op value` into SQL. Every + * matcher routes through here — both filter compilers (`buildFilterClause` for + * the legacy `$`-grammar, `buildPredicateClause` for the v2 grammar), the upsert + * conflict probe, and the unique-constraint checks. Centralizing the leaf means + * equality/case/null/cast semantics are defined exactly once, so "find the row" + * and "is this value unique" can never disagree. + * + * Returns `undefined` when the predicate is a no-op (empty `in`/`nin` array), + * matching the legacy behavior of emitting no clause. + * + * Equality (`eq`/`ne`/`in`/`nin`) uses case-sensitive JSONB containment (GIN + * indexed). Text matches (`contains`/`ncontains`/`startsWith`/`endsWith`) are + * ILIKE (case-insensitive). Ranges cast per column type. + */ +export function fieldPredicate( + tableName: string, + field: string, + op: FilterOp, + value: JsonValue | undefined, + column: ColumnDefinition | undefined +): SQL | undefined { + validateFieldName(field) - case '$lt': - conditions.push( - buildComparisonClause(tableName, field, '<', value as number | string, columnType) - ) - break + // System columns (`createdAt`/`updatedAt`/`id`) are real row columns, not + // JSONB keys — dispatch before the `data->>` builders below, which would + // silently match nothing (the key never exists in `data`). + if (isSystemColumn(field)) { + return buildSystemColumnClause(tableName, field, op, value) + } - case '$lte': - conditions.push( - buildComparisonClause(tableName, field, '<=', value as number | string, columnType) - ) - break - - case '$in': - if (Array.isArray(value) && value.length > 0) { - if (value.length === 1) { - // Single value then use containment clause - conditions.push(buildContainmentClause(tableName, field, value[0])) - } else { - // Multiple values then use OR clause - const inConditions = value.map((v) => buildContainmentClause(tableName, field, v)) - conditions.push(sql`(${sql.join(inConditions, sql.raw(' OR '))})`) - } - } - break - - case '$nin': - if (Array.isArray(value) && value.length > 0) { - const ninConditions = value.map( - (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})` - ) - conditions.push(sql`(${sql.join(ninConditions, sql.raw(' AND '))})`) - } - break - - case '$contains': - conditions.push( - isMultiSelect - ? buildArrayMembershipClause(tableName, field, value as JsonValue) - : buildLikeClause(tableName, field, value as string, 'contains') - ) - break - - case '$ncontains': - conditions.push( - isMultiSelect - ? // Mirror the ILIKE negation: rows with no selection at all count - // as "does not contain", so an empty cell isn't silently excluded. - sql`NOT (${buildArrayMembershipClause(tableName, field, value as JsonValue)})` - : buildLikeClause(tableName, field, value as string, 'contains', { negate: true }) - ) - break + const columnType = column?.type + const isSelect = columnType === 'select' + // A multi-select cell holds an ARRAY of option ids, so equality against a + // scalar can never be true; the question is membership. Gating and clause + // choice both live here rather than in `buildFieldCondition` so the v2 + // predicate grammar gets the identical treatment. + const isMultiSelect = isSelect && column?.multiple === true - case '$startsWith': - conditions.push(buildLikeClause(tableName, field, value as string, 'startsWith')) - break + if (isSelect) { + const allowed = isMultiSelect ? MULTI_SELECT_OPS : SINGLE_SELECT_OPS + if (!allowed.has(op)) { + throw new TableQueryValidationError( + `Operator "${op}" is not supported on ${isMultiSelect ? 'multi-select' : 'select'} column "${field}". Allowed: ${Array.from(allowed).join(', ')}` + ) + } + } - case '$endsWith': - conditions.push(buildLikeClause(tableName, field, value as string, 'endsWith')) - break + if (isMultiSelect) { + switch (op) { + case 'contains': + return buildArrayMembershipClause(tableName, field, value as JsonValue) + case 'ncontains': + return sql`NOT (${buildArrayMembershipClause(tableName, field, value as JsonValue)})` + case 'isEmpty': + return buildEmptyClause(tableName, field, true, true) + case 'isNotEmpty': + return buildEmptyClause(tableName, field, false, true) + default: + break + } + } - case '$empty': - conditions.push( - buildEmptyClause(tableName, field, coerceEmptyFlag(field, value), isMultiSelect) - ) - break + switch (op) { + case 'eq': + return buildContainmentClause(tableName, field, value as JsonValue) + + case 'ne': + return sql`NOT (${buildContainmentClause(tableName, field, value as JsonValue)})` + + case 'gt': + return buildComparisonClause(tableName, field, '>', value as number | string, columnType) + case 'gte': + return buildComparisonClause(tableName, field, '>=', value as number | string, columnType) + case 'lt': + return buildComparisonClause(tableName, field, '<', value as number | string, columnType) + case 'lte': + return buildComparisonClause(tableName, field, '<=', value as number | string, columnType) + + case 'in': { + if (!Array.isArray(value) || value.length === 0) return undefined + if (value.length === 1) return buildContainmentClause(tableName, field, value[0]) + const inConditions = value.map((v) => buildContainmentClause(tableName, field, v)) + return sql`(${sql.join(inConditions, sql.raw(' OR '))})` + } - default: - // This should never happen due to validateOperator, but added for completeness. - // Throw a plain Error (→ 500) since reaching this default means the switch - // and ALLOWED_OPERATORS have drifted — that's a programmer error, not a caller error. - throw new Error(`Unsupported operator: ${op}`) - } + case 'nin': { + if (!Array.isArray(value) || value.length === 0) return undefined + const ninConditions = value.map( + (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})` + ) + return sql`(${sql.join(ninConditions, sql.raw(' AND '))})` } - } else { - // Simple value (primitive or null) - shorthand for equality. - // Example: { name: 'John' } is equivalent to { name: { $eq: 'John' } } - // isRecordLike's negation can't structurally exclude ConditionOperators (no index - // signature), unlike the prior typeof-based narrowing, so the JsonValue-only shape - // of this branch is asserted rather than inferred. - // - // On a multi-select the shorthand reads as "holds this option" — scalar - // equality against an array cell can never be true, so treat it as - // membership rather than compiling a filter that silently matches nothing. - conditions.push( - isMultiSelect - ? buildArrayMembershipClause(tableName, field, condition as JsonValue) - : buildContainmentClause(tableName, field, condition as JsonValue) - ) - } - return conditions + case 'contains': + return buildLikeClause(tableName, field, value as string, 'contains') + case 'ncontains': + return buildLikeClause(tableName, field, value as string, 'contains', { negate: true }) + case 'startsWith': + return buildLikeClause(tableName, field, value as string, 'startsWith') + case 'endsWith': + return buildLikeClause(tableName, field, value as string, 'endsWith') + + case 'like': + return buildPatternClause(tableName, field, value as string, { caseInsensitive: false }) + case 'ilike': + return buildPatternClause(tableName, field, value as string, { caseInsensitive: true }) + case 'nlike': + return buildPatternClause(tableName, field, value as string, { + caseInsensitive: false, + negate: true, + }) + case 'nilike': + return buildPatternClause(tableName, field, value as string, { + caseInsensitive: true, + negate: true, + }) + + case 'isEmpty': + return buildEmptyClause(tableName, field, true) + case 'isNotEmpty': + return buildEmptyClause(tableName, field, false) + + case 'isNull': + return buildNullClause(tableName, field, true) + case 'isNotNull': + return buildNullClause(tableName, field, false) + + default: + throw new TableQueryValidationError(`Invalid operator "${op}"`) + } } /** @@ -497,6 +629,124 @@ function buildLogicalClause( return sql`(${sql.join(clauses, sql.raw(` ${operator} `))})` } +/** + * Row columns that are addressable in filters/sorts but live on the row itself + * rather than inside the JSONB `data` blob. The docs advertise all three as + * filterable and sortable; without this dispatch they compile to a `data->>'…'` + * extraction of a key that never exists, so they silently match nothing. + */ +const SYSTEM_COLUMNS: Readonly> = { + createdAt: { column: 'created_at', kind: 'timestamp' }, + updatedAt: { column: 'updated_at', kind: 'timestamp' }, + id: { column: 'id', kind: 'text' }, +} + +function isSystemColumn(field: string): boolean { + return Object.hasOwn(SYSTEM_COLUMNS, field) +} + +/** + * Builds a predicate against a system column. Timestamp columns bind ISO strings + * normalized to UTC wall clock; the text column (`id`) binds as text and also + * accepts the pattern ops. Anything else is rejected with an actionable error. + */ +function buildSystemColumnClause( + tableName: string, + field: string, + op: FilterOp, + value: JsonValue | undefined +): SQL | undefined { + const spec = SYSTEM_COLUMNS[field] + const col = sql.raw(`${tableName}.${spec.column}`) + // `created_at`/`updated_at` are `timestamp WITHOUT time zone` holding UTC wall + // clock. A bare `::timestamptz` comparison promotes the column using the session + // `TimeZone` GUC, so identical queries return different rows per environment and + // day-boundary ranges land off by the offset. Normalizing the bound to UTC wall + // clock is session-independent and still honors an explicit offset in the input. + const ts = (v: JsonValue | undefined) => sql`${String(v)}::timestamptz AT TIME ZONE 'UTC'` + const bind = spec.kind === 'timestamp' ? ts : (v: JsonValue | undefined) => sql`${String(v)}` + /** + * Mirrors the JSONB pattern builders: `*` is the caller's only wildcard, an + * empty pattern is rejected (it would collapse to `%` and match every row), + * and the negated forms keep NULL cells so "does not contain X" retains them. + */ + const like = ( + v: JsonValue | undefined, + pattern: (escaped: string) => string, + ci: boolean, + negate = false + ) => { + const text = String(v ?? '') + if (text.length === 0) { + throw new TableQueryValidationError( + `Operator "${op}" on column "${field}" requires a non-empty value` + ) + } + const p = pattern(escapeLikePattern(text)) + const match = ci ? sql`${col} ILIKE ${p}` : sql`${col} LIKE ${p}` + return negate ? sql`NOT (${match})` : match + } + + // The text system column (`id`) additionally supports the pattern ops; + // timestamps fall through to the unsupported-operator error below. + if (spec.kind === 'text') { + const star = (e: string) => e.replace(/\*/g, '%') + switch (op) { + case 'like': + return like(value, star, false) + case 'ilike': + return like(value, star, true) + case 'nlike': + return like(value, star, false, true) + case 'nilike': + return like(value, star, true, true) + case 'contains': + return like(value, (e) => `%${e}%`, true) + case 'ncontains': + return like(value, (e) => `%${e}%`, true, true) + case 'startsWith': + return like(value, (e) => `${e}%`, true) + case 'endsWith': + return like(value, (e) => `%${e}`, true) + default: + break + } + } + + switch (op) { + case 'eq': + return sql`${col} = ${bind(value)}` + case 'ne': + return sql`${col} <> ${bind(value)}` + case 'gt': + return sql`${col} > ${bind(value)}` + case 'gte': + return sql`${col} >= ${bind(value)}` + case 'lt': + return sql`${col} < ${bind(value)}` + case 'lte': + return sql`${col} <= ${bind(value)}` + case 'in': { + if (!Array.isArray(value) || value.length === 0) return undefined + return sql`${col} IN (${sql.join(value.map(bind), sql.raw(', '))})` + } + case 'nin': { + if (!Array.isArray(value) || value.length === 0) return undefined + return sql`${col} NOT IN (${sql.join(value.map(bind), sql.raw(', '))})` + } + case 'isNull': + case 'isEmpty': + return sql`${col} IS NULL` + case 'isNotNull': + case 'isNotEmpty': + return sql`${col} IS NOT NULL` + default: + throw new TableQueryValidationError( + `Operator "${op}" is not supported on the built-in column "${field}" — use eq, ne, gt, gte, lt, lte, in, nin, isNull, isNotNull.` + ) + } +} + /** Builds JSONB containment clause: `data @> '{"field": value}'::jsonb` (uses GIN index) */ function buildContainmentClause(tableName: string, field: string, value: JsonValue): SQL { const jsonObj = JSON.stringify({ [field]: value }) @@ -523,11 +773,12 @@ function buildArrayMembershipClause(tableName: string, field: string, value: Jso * to `timestamptz` so date strings compare chronologically and timezone offsets * in ISO strings (e.g. `2024-01-01T00:00:00Z`) are preserved rather than * silently stripped (which would make results depend on the server's TimeZone - * setting). Unknown/other types - * fall back to `numeric` (legacy default — preserves behavior for ad-hoc fields - * with no schema entry). The right-hand value is cast explicitly because - * drizzle parameterizes it as `text`; without the cast, Postgres would compare - * `text text` and silently produce lexicographic results. + * setting). `string` columns compare lexicographically as text. `boolean`/`json` + * columns have no meaningful ordering and are rejected. Columns with no schema + * entry fall back to `numeric` (legacy default — preserves behavior for ad-hoc + * fields). The right-hand value is cast explicitly because drizzle parameterizes + * it as `text`; without the cast, Postgres would compare `text text` and + * silently produce lexicographic results. * * Cannot use the GIN index — falls back to a sequential scan over the table's * rows (bounded by the btree prefix on `table_id`). @@ -540,6 +791,18 @@ function buildComparisonClause( columnType: ColumnType | undefined ): SQL { const escapedField = field.replace(/'/g, "''") + + if (columnType === 'boolean' || columnType === 'json') { + throw new TableQueryValidationError( + `Range operator on column "${field}" (${columnType}) is not supported — ${columnType} values have no ordering.` + ) + } + + if (columnType === 'string') { + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + return sql`${cell} ${sql.raw(operator)} ${String(value)}` + } + const cast = jsonbCastForType(columnType) ?? 'numeric' validateComparisonValue(field, columnType, cast, value) const cell = sql.raw(`(${tableName}.data->>'${escapedField}')::${cast}`) @@ -553,6 +816,36 @@ export function escapeLikePattern(value: string): string { return value.replace(/[\\%_]/g, '\\$&') } +/** + * General LIKE/ILIKE pattern match (the `like`/`ilike` ops). The caller's `*` + * is the only wildcard — it maps to SQL `%`; any literal `%`/`_`/`\` in the + * value is escaped so it matches itself. Empty/exact patterns are allowed (an + * empty pattern matches only the empty string, not every row, so it's not the + * footgun the positional `buildLikeClause` guards against). `negate` inverts + * the match and keeps null cells — "does not match" retains empty rows, + * mirroring `buildLikeClause`'s ncontains semantics. Cannot use the GIN index; + * sequential scan bounded by the `table_id` btree prefix. + */ +function buildPatternClause( + tableName: string, + field: string, + value: string, + options: { caseInsensitive: boolean; negate?: boolean } +): SQL { + const escapedField = field.replace(/'/g, "''") + const pattern = String(value) + .replace(/[\\%_]/g, '\\$&') + .replace(/\*/g, '%') + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + const match = options.caseInsensitive + ? sql`${cell} ILIKE ${pattern}` + : sql`${cell} LIKE ${pattern}` + if (!options.negate) return match + return options.caseInsensitive + ? sql`(${cell} IS NULL OR ${cell} NOT ILIKE ${pattern})` + : sql`(${cell} IS NULL OR ${cell} NOT LIKE ${pattern})` +} + /** * Builds a case-insensitive pattern match against a JSONB cell using ILIKE. * `position` controls wildcard placement: `contains` → `%value%`, `startsWith` @@ -635,6 +928,18 @@ function buildEmptyClause( : sql`(${cell} IS NOT NULL AND ${cell} <> '')` } +/** + * Strict null check on a JSONB cell — distinct from `isEmpty`/`isNotEmpty`, + * which also treat the empty string as empty. `isNull` matches an absent key or + * JSON null (both surfaced as SQL NULL by `->>`); the negation requires the cell + * to be present (an empty string counts as not-null here). + */ +function buildNullClause(tableName: string, field: string, isNull: boolean): SQL { + const escapedField = field.replace(/'/g, "''") + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + return isNull ? sql`${cell} IS NULL` : sql`${cell} IS NOT NULL` +} + /** * Builds a single ORDER BY clause for a field. * Timestamp fields use direct column access, others use JSONB text extraction. @@ -654,8 +959,8 @@ function buildSortFieldClause( const escapedField = field.replace(/'/g, "''") const directionSql = direction.toUpperCase() - if (field === 'createdAt' || field === 'updatedAt') { - return sql.raw(`${tableName}.${escapedField} ${directionSql}`) + if (isSystemColumn(field)) { + return sql.raw(`${tableName}.${SYSTEM_COLUMNS[field].column} ${directionSql}`) } const jsonbExtract = `${tableName}.data->>'${escapedField}'` diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 6f38357552f..1c6b5e39913 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -2,7 +2,7 @@ * Type definitions for user-defined tables. */ -import type { COLUMN_TYPES } from '@/lib/table/constants' +import type { COLUMN_TYPES, FILTER_OPS } from '@/lib/table/constants' export type ColumnValue = string | number | boolean | null | Date export type JsonValue = ColumnValue | JsonValue[] | { [key: string]: JsonValue } @@ -63,6 +63,9 @@ export interface ColumnDefinition { multiple?: boolean } +/** The column `type` discriminator, named so callers don't index into the interface. */ +export type ColumnType = ColumnDefinition['type'] + /** One group output → one plain column. */ export interface WorkflowGroupOutput { /** Source block id within the configured workflow. `''` for enrichment groups. */ @@ -283,8 +286,8 @@ export interface TableMetadata { * user resizes, reorders, pins, or hides columns. */ export interface TableViewConfig extends TableMetadata { - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } /** Async background-job lifecycle state for a table. NULL/undefined = idle (no job). */ @@ -500,6 +503,39 @@ export interface Filter { [key: string]: ColumnValue | ConditionOperators | Filter[] | undefined } +/** + * v2 filter operators (bare, no `$`). Equality and `in`/`nin` are case-sensitive + * (JSONB containment, GIN-indexed); the text ops `contains`/`ncontains`/ + * `startsWith`/`endsWith` are ILIKE (case-insensitive). `isEmpty`/`isNotEmpty` + * match null OR empty string; + * `isNull`/`isNotNull` are strict null checks. The four `is*` ops are valueless. + * This is the canonical operator set the shared `fieldPredicate` leaf + * understands; the legacy `$`-operators normalize onto it. + */ +export type FilterOp = (typeof FILTER_OPS)[number] + +/** A single v2 leaf predicate: `field op value`. `value` is omitted for `isEmpty`/`isNotEmpty`. */ +export interface Predicate { + field: string + op: FilterOp + value?: JsonValue +} + +/** + * v2 nestable filter tree. A group is either `{ all: [...] }` (AND) or + * `{ any: [...] }` (OR); members are leaves or nested groups. Replaces the + * MongoDB-style `Filter` on the v2 surface — same engine, legible grammar. + * + * @example + * { all: [{ field: 'slack_user_id', op: 'in', value: ['U1','U2'] }, { field: 'wins', op: 'gte', value: 10 }] } + * { any: [{ field: 'status', op: 'eq', value: 'active' }, { field: 'status', op: 'eq', value: 'pending' }] } + */ +export type PredicateNode = Predicate | TablePredicate +export type TablePredicate = { all: PredicateNode[] } | { any: PredicateNode[] } + +/** v2 sort specification: an ordered list of `{ field, direction }`. */ +export type SortSpec = Array<{ field: string; direction: SortDirection }> + export interface ValidationResult { valid: boolean errors: string[] @@ -531,11 +567,21 @@ export interface SortRule { export interface QueryOptions { filter?: Filter + /** + * v2 nestable predicate. When set it takes precedence over `filter` — the two + * compile through the same `fieldPredicate` leaf, so callers pick one grammar. + */ + predicate?: TablePredicate sort?: Sort + /** Page row cap. Omitted = return the ENTIRE matching result, failing fast if + * it exceeds the response byte budget (`MAX_QUERY_RESULT_BYTES`). A bounded + * page may byte-cut early with `nextCursor` set. Never an unbounded fetch — + * the drain stops at the budget either way. */ limit?: number offset?: number /** Keyset cursor for the default `(order_key, id)` order — see {@link TableRowsCursor}. - * Mutually exclusive with `sort` and `offset`; takes precedence over `offset` when set. */ + * Mutually exclusive with `sort`. May be combined with `offset` (a compound + * cursor seeks the anchor, then offsets past unkeyed rows consumed after it). */ after?: TableRowsCursor /** * When true (default), runs a `COUNT(*)` and returns `totalCount` as a number. @@ -557,6 +603,13 @@ export interface QueryResult { totalCount: number | null limit: number offset: number + /** + * Opaque cursor for the next page — non-null whenever more matching rows + * exist beyond this page, whether the page was cut by `limit` or by the + * response byte budget. Callers echo it back as `cursor` and never construct + * keyset/offset state themselves. See `rows/cursor.ts`. + */ + nextCursor: string | null } export interface BulkOperationResult { diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index 8b6d58ff0e2..14275ea74eb 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -13,9 +13,11 @@ import { MAX_SELECT_OPTIONS, NAME_PATTERN, TABLE_LIMITS, + USER_TABLE_ROWS_SQL_NAME, } from '@/lib/table/constants' import { normalizeDateCellValue } from '@/lib/table/dates' import { withSeqscanOff } from '@/lib/table/planner' +import { fieldPredicate } from '@/lib/table/sql' import type { ColumnDefinition, JsonValue, @@ -297,12 +299,21 @@ function optionIds(column: ColumnDefinition): Set { * actually fit the target option set. */ export function resolveSelectOptionId(value: JsonValue, options: SelectOption[]): string | null { - if (typeof value !== 'string') return null - const byId = options.find((o) => o.id === value) + // The block builder serializes without schema access, so an option NAME that + // looks numeric or boolean ("123", "true") arrives scalar-coerced. Stringify + // scalars so the name still resolves; arrays/objects stay unresolvable. + const text = + typeof value === 'string' + ? value + : typeof value === 'number' || typeof value === 'boolean' + ? String(value) + : null + if (text === null) return null + const byId = options.find((o) => o.id === text) if (byId) return byId.id const byName = - options.find((o) => o.name === value) ?? - options.find((o) => o.name.toLowerCase() === value.toLowerCase()) + options.find((o) => o.name === text) ?? + options.find((o) => o.name.toLowerCase() === text.toLowerCase()) return byName ? byName.id : null } @@ -469,12 +480,8 @@ export function validateUniqueConstraints( const duplicate = existingRows.find((row) => { if (excludeRowId && row.id === excludeRowId) return false - - const existingValue = row.data[key] - if (typeof value === 'string' && typeof existingValue === 'string') { - return value.toLowerCase() === existingValue.toLowerCase() - } - return value === existingValue + // Case-sensitive, matching the DB unique-check leaf (`fieldPredicate` eq). + return value === row.data[key] }) if (duplicate) { @@ -517,27 +524,13 @@ export async function checkUniqueConstraintsDb( for (const column of uniqueColumns) { const key = getColumnId(column) - if (!NAME_PATTERN.test(key)) { - throw new Error(`Invalid column id: ${key}`) - } - const value = data[key] if (value === null || value === undefined) continue - if (typeof value === 'string') { - conditions.push({ - column, - value, - sql: sql`lower(${userTableRows.data}->>${sql.raw(`'${key}'`)}) = ${value.toLowerCase()}`, - }) - } else { - // For other types, use direct JSONB comparison - conditions.push({ - column, - value, - sql: sql`(${userTableRows.data}->${sql.raw(`'${key}'`)})::jsonb = ${JSON.stringify(value)}::jsonb`, - }) - } + // Same leaf as the upsert conflict probe → case-sensitive JSONB containment + // (GIN-indexed). `eq` always yields a clause for a non-null value. + const clause = fieldPredicate(USER_TABLE_ROWS_SQL_NAME, key, 'eq', value, column) + if (clause) conditions.push({ column, value, sql: clause }) } if (conditions.length === 0) { @@ -545,9 +538,9 @@ export async function checkUniqueConstraintsDb( } // Query for each unique column separately to provide specific error messages. - // Tenant-bounded: `lower(data->>'col') = ...` is unestimatable, so the planner - // otherwise seq-scans the whole shared relation per check — 3.5s on every - // insert/edit when the value is unique (no early exit). With an external + // The predicate is now case-sensitive JSONB containment (`data @> {...}`), + // which can use the GIN index. We still pin `enable_seqscan = off` (tenant- + // bounded) defensively for the small-table / cold-stats case. With an external // transaction the flag is set on it directly — opening our own transaction // inside the caller's would be the nested pool checkout the migration- // hardening work eliminated (self-deadlock under pool exhaustion). @@ -635,8 +628,7 @@ export async function checkBatchUniqueConstraintsDb( const value = rowData[key] if (value === null || value === undefined) continue - const normalizedValue = - typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value) + const normalizedValue = typeof value === 'string' ? value : JSON.stringify(value) // Check for duplicate within batch const columnValueMap = batchValueMap.get(key)! @@ -672,14 +664,22 @@ export async function checkBatchUniqueConstraintsDb( const valueArray = Array.from(values) const valueConditions = valueArray.map((normalizedValue) => { - // Check if the original values are strings (normalized values for strings are lowercase) - // We need to determine the type from the column definition or the first row that has this value - const isStringColumn = column.type === 'string' - - if (isStringColumn) { - return sql`lower(${userTableRows.data}->>${sql.raw(`'${columnId}'`)}) = ${normalizedValue}` + // Reconstruct the original typed value from its normalized key: string + // columns store the raw string; others store JSON.stringify(value). + const originalValue: JsonValue = + column.type === 'string' ? normalizedValue : JSON.parse(normalizedValue) + // Same case-sensitive containment leaf as every other matcher. + const clause = fieldPredicate( + USER_TABLE_ROWS_SQL_NAME, + columnId, + 'eq', + originalValue, + column + ) + if (!clause) { + throw new Error(`Failed to build unique-constraint predicate for column "${column.name}"`) } - return sql`(${userTableRows.data}->${sql.raw(`'${columnId}'`)})::jsonb = ${normalizedValue}::jsonb` + return clause }) const conflictingRows = await ex @@ -697,9 +697,7 @@ export async function checkBatchUniqueConstraintsDb( const conflictData = conflict.data as RowData const conflictValue = conflictData[columnId] const normalizedConflictValue = - typeof conflictValue === 'string' - ? conflictValue.toLowerCase() - : JSON.stringify(conflictValue) + typeof conflictValue === 'string' ? conflictValue : JSON.stringify(conflictValue) // Find which batch rows have this conflicting value for (let i = 0; i < rows.length; i++) { @@ -707,7 +705,7 @@ export async function checkBatchUniqueConstraintsDb( if (rowValue === null || rowValue === undefined) continue const normalizedRowValue = - typeof rowValue === 'string' ? rowValue.toLowerCase() : JSON.stringify(rowValue) + typeof rowValue === 'string' ? rowValue : JSON.stringify(rowValue) if (normalizedRowValue === normalizedConflictValue) { // Check if this row already has errors for this column diff --git a/apps/sim/lib/table/views/service.test.ts b/apps/sim/lib/table/views/service.test.ts index 5b048954313..3405f73bd90 100644 --- a/apps/sim/lib/table/views/service.test.ts +++ b/apps/sim/lib/table/views/service.test.ts @@ -3,7 +3,7 @@ */ import { describe, expect, it } from 'vitest' import type { ColumnDefinition, TableViewConfig } from '@/lib/table/types' -import { pruneViewConfig } from '@/lib/table/views/service' +import { normalizeStoredViewConfig, pruneViewConfig } from '@/lib/table/views/service' const columns: ColumnDefinition[] = [ { id: 'col_a', name: 'Name', type: 'text' }, @@ -28,14 +28,18 @@ describe('pruneViewConfig', () => { }) it('drops a sort on a deleted column and collapses to null when none remain', () => { - expect(pruneViewConfig({ sort: { col_gone: 'asc' } }, columns).sort).toBeNull() - expect(pruneViewConfig({ sort: { col_a: 'desc' } }, columns).sort).toEqual({ col_a: 'desc' }) + expect( + pruneViewConfig({ sort: [{ field: 'col_gone', direction: 'asc' }] }, columns).sort + ).toBeNull() + expect( + pruneViewConfig({ sort: [{ field: 'col_a', direction: 'desc' }] }, columns).sort + ).toEqual([{ field: 'col_a', direction: 'desc' }]) }) it('leaves the filter untouched even when it references a deleted column', () => { // Pruning a predicate would silently widen the view's row set — surfacing a // stale condition the user can see and remove is the safer failure. - const filter = { col_gone: { $eq: 'x' } } + const filter = { all: [{ field: 'col_gone', op: 'eq' as const, value: 'x' }] } expect(pruneViewConfig({ filter }, columns).filter).toEqual(filter) }) @@ -50,3 +54,33 @@ describe('pruneViewConfig', () => { ]) }) }) + +/** + * Reads written before the grammar switch: the feature never released, so + * legacy-shaped configs exist only from pre-refactor testing — but they must + * come back as v2, not render broken. + */ +describe('normalizeStoredViewConfig', () => { + it('converts a legacy $-object filter to a predicate tree', () => { + const out = normalizeStoredViewConfig({ filter: { col_a: { $eq: 'x' } } }) + expect(out.filter).toEqual({ all: [{ field: 'col_a', op: 'eq', value: 'x' }] }) + }) + + it('converts a legacy {col: dir} sort record to an ordered spec', () => { + const out = normalizeStoredViewConfig({ sort: { col_a: 'desc' } }) + expect(out.sort).toEqual([{ field: 'col_a', direction: 'desc' }]) + }) + + it('passes v2-shaped configs through untouched', () => { + const config = { + filter: { all: [{ field: 'col_a', op: 'eq', value: 'x' }] }, + sort: [{ field: 'col_a', direction: 'asc' }], + } + expect(normalizeStoredViewConfig(config)).toEqual(config) + }) + + it('drops an unconvertible legacy filter rather than surfacing it broken', () => { + const out = normalizeStoredViewConfig({ filter: { $bogus: [{ nested: true }] } }) + expect(out.filter).toBeNull() + }) +}) diff --git a/apps/sim/lib/table/views/service.ts b/apps/sim/lib/table/views/service.ts index 1b575bb618a..1c8684a414c 100644 --- a/apps/sim/lib/table/views/service.ts +++ b/apps/sim/lib/table/views/service.ts @@ -15,7 +15,15 @@ import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { and, asc, eq, ne, sql } from 'drizzle-orm' import { getColumnId } from '@/lib/table/column-keys' -import type { ColumnDefinition, TableViewConfig } from '@/lib/table/types' +import { NAME_PATTERN } from '@/lib/table/constants' +import { filterRulesToPredicate, filterToRules } from '@/lib/table/query-builder/converters' +import type { + ColumnDefinition, + Filter, + Predicate, + PredicateNode, + TableViewConfig, +} from '@/lib/table/types' const logger = createLogger('TableViewsService') @@ -69,22 +77,58 @@ export function pruneViewConfig( pruned.columnWidths = widths } if (config.sort) { - const sort: Record = {} - for (const [id, direction] of Object.entries(config.sort)) { - if (live.has(id)) sort[id] = direction - } - pruned.sort = Object.keys(sort).length > 0 ? sort : null + const sort = config.sort.filter((s) => live.has(s.field)) + pruned.sort = sort.length > 0 ? sort : null } return pruned } +/** + * Migrates a config stored before the grammar switch. The feature never + * released, so legacy-shaped rows exist only from pre-refactor testing: a + * `$`-object filter converts through the builder-rule round-trip (its exact + * authoring domain), and a `{col: dir}` sort record becomes an ordered spec. + * Anything unconvertible is dropped rather than surfaced broken. + */ + +/** Every leaf field in the tree is a plausible column id. */ +function predicateFieldsAreValid(node: PredicateNode): boolean { + if ('all' in node) return node.all.every(predicateFieldsAreValid) + if ('any' in node) return node.any.every(predicateFieldsAreValid) + return NAME_PATTERN.test((node as Predicate).field) +} + +export function normalizeStoredViewConfig(raw: Record): TableViewConfig { + const config = { ...raw } as TableViewConfig + const filter = raw.filter as Record | null | undefined + if (filter && !('all' in filter) && !('any' in filter)) { + try { + const converted = filterRulesToPredicate(filterToRules(filter as Filter)) + // The rule converters don't reject garbage — an unknown `$op` becomes a + // rule on a column literally named `$op`. A converted leaf whose field + // fails the column-name pattern proves the input wasn't builder-authored. + config.filter = converted && predicateFieldsAreValid(converted) ? converted : null + } catch { + config.filter = null + } + } + const sort = raw.sort as Record | unknown[] | null | undefined + if (sort && !Array.isArray(sort)) { + config.sort = Object.entries(sort).map(([field, direction]) => ({ field, direction })) + } + return config +} + function toTableView(row: typeof tableViews.$inferSelect, columns: ColumnDefinition[]): TableView { return { id: row.id, tableId: row.tableId, name: row.name, - config: pruneViewConfig((row.config ?? {}) as TableViewConfig, columns), + config: pruneViewConfig( + normalizeStoredViewConfig((row.config ?? {}) as Record), + columns + ), isDefault: row.isDefault, createdBy: row.createdBy, createdAt: row.createdAt, diff --git a/apps/sim/lib/workspaces/utils.ts b/apps/sim/lib/workspaces/utils.ts index a8fccb2b79c..05dd7a62466 100644 --- a/apps/sim/lib/workspaces/utils.ts +++ b/apps/sim/lib/workspaces/utils.ts @@ -48,6 +48,21 @@ export async function getWorkspaceBilledAccountUserId(workspaceId: string): Prom return settings?.billedAccountUserId ?? null } +/** + * The organization that owns a workspace (null for personal workspaces). Used to + * gate features by org cohort — the flag model allowlists org ids, and API routes + * only carry a `workspaceId`, so this resolves the one from the other. + */ +export async function getWorkspaceOrganizationId(workspaceId: string): Promise { + if (!workspaceId) return null + const rows = await db + .select({ organizationId: workspaceTable.organizationId }) + .from(workspaceTable) + .where(eq(workspaceTable.id, workspaceId)) + .limit(1) + return rows[0]?.organizationId ?? null +} + /** * Workspaces the user administers purely through organization owner/admin role, * with no explicit permission row required. Empty when the user is not an org diff --git a/apps/sim/tools/normalize.test.ts b/apps/sim/tools/normalize.test.ts new file mode 100644 index 00000000000..3f2f845f5ea --- /dev/null +++ b/apps/sim/tools/normalize.test.ts @@ -0,0 +1,40 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { normalizeToolId } from '@/tools/normalize' + +describe('normalizeToolId', () => { + it('strips a resource-id suffix', () => { + expect(normalizeToolId('table_query_rows_tbl_1a2c1741')).toBe('table_query_rows') + expect(normalizeToolId('knowledge_search_5cc56998-3e1d-4d91')).toBe('knowledge_search') + expect(normalizeToolId('workflow_executor_f3d81b32')).toBe('workflow_executor') + expect(normalizeToolId('deployed_block_executor_custom_block_9')).toBe( + 'deployed_block_executor' + ) + }) + + it('leaves a bare tool id alone', () => { + expect(normalizeToolId('table_query_rows')).toBe('table_query_rows') + expect(normalizeToolId('gmail_send')).toBe('gmail_send') + }) + + /** + * Regression: `table_query_rows_v2` starts with `table_query_rows_`, so the + * resource-suffix strip turned it into the v1 tool. The executor then logged + * the v2 id while issuing v1's `GET /rows?filter=` — which reached + * the legacy filter compiler and 400'd on a correctly configured v2 block. + */ + it('does not mistake a version suffix for a resource id', () => { + expect(normalizeToolId('table_query_rows_v2')).toBe('table_query_rows_v2') + }) + + it('still strips a resource id from a VERSIONED op', () => { + expect(normalizeToolId('table_query_rows_v2_tbl_1a2c1741')).toBe('table_query_rows_v2') + }) + + it('is not fooled by a table id that merely starts with v', () => { + expect(normalizeToolId('table_query_rows_v2x')).toBe('table_query_rows') + expect(normalizeToolId('table_query_rows_version')).toBe('table_query_rows') + }) +}) diff --git a/apps/sim/tools/normalize.ts b/apps/sim/tools/normalize.ts index 177c11aa140..c01dceaf33b 100644 --- a/apps/sim/tools/normalize.ts +++ b/apps/sim/tools/normalize.ts @@ -6,6 +6,29 @@ * * Pure string utility — no server dependencies, safe to import in client components. */ + +/** + * A trailing `_v2`-style segment is a VERSION marker, not a resource id, so it + * must not be stripped: `table_query_rows_v2` is its own registered tool, and + * normalizing it to `table_query_rows` silently executes the v1 tool's request + * shape under the v2 tool's name. + */ +const VERSION_SUFFIX = /^v\d+$/ + +/** + * Longest id first, so a versioned op claims its own suffixed ids before the + * unversioned prefix can match them (`table_query_rows_v2_` must + * normalize to `table_query_rows_v2`, not `table_query_rows`). + */ +function stripResourceSuffix(toolId: string, ops: string[]): string | null { + for (const op of [...ops].sort((a, b) => b.length - a.length)) { + if (!toolId.startsWith(`${op}_`) || toolId.length <= op.length + 1) continue + if (VERSION_SUFFIX.test(toolId.slice(op.length + 1))) continue + return op + } + return null +} + export function normalizeToolId(toolId: string): string { // Custom (deploy-as-block) tools: 'deployed_block_executor_custom_block_' -> // 'deployed_block_executor'. Note the id deliberately does NOT start with @@ -23,14 +46,12 @@ export function normalizeToolId(toolId: string): string { } const knowledgeOps = ['knowledge_search', 'knowledge_upload_chunk', 'knowledge_create_document'] - for (const op of knowledgeOps) { - if (toolId.startsWith(`${op}_`) && toolId.length > op.length + 1) { - return op - } - } + const knowledge = stripResourceSuffix(toolId, knowledgeOps) + if (knowledge) return knowledge const tableOps = [ 'table_query_rows', + 'table_query_rows_v2', 'table_insert_row', 'table_batch_insert_rows', 'table_update_row', @@ -41,11 +62,8 @@ export function normalizeToolId(toolId: string): string { 'table_delete_row', 'table_get_schema', ] - for (const op of tableOps) { - if (toolId.startsWith(`${op}_`) && toolId.length > op.length + 1) { - return op - } - } + const table = stripResourceSuffix(toolId, tableOps) + if (table) return table return toolId } diff --git a/apps/sim/tools/registry.minimal.ts b/apps/sim/tools/registry.minimal.ts index bacfae7e566..899124191ce 100644 --- a/apps/sim/tools/registry.minimal.ts +++ b/apps/sim/tools/registry.minimal.ts @@ -71,6 +71,21 @@ import { slackUpdateMessageTool, slackUpdateViewTool, } from '@/tools/slack' +import { + tableBatchInsertRowsTool, + tableCreateTool, + tableDeleteRowsByFilterTool, + tableDeleteRowTool, + tableGetRowTool, + tableGetSchemaTool, + tableInsertRowTool, + tableListTool, + tableQueryRowsTool, + tableQueryRowsV2Tool, + tableUpdateRowsByFilterTool, + tableUpdateRowTool, + tableUpsertRowTool, +} from '@/tools/table' import type { ToolConfig } from '@/tools/types' import { customBlockExecutorTool, workflowExecutorTool } from '@/tools/workflow' @@ -87,6 +102,20 @@ import { customBlockExecutorTool, workflowExecutorTool } from '@/tools/workflow' export const tools: Record = { http_request: httpRequestTool, function_execute: functionExecuteTool, + // Table block operations (v1 + v2 Table blocks are both runnable in minimal mode). + table_query_rows: tableQueryRowsTool, + table_insert_row: tableInsertRowTool, + table_batch_insert_rows: tableBatchInsertRowsTool, + table_upsert_row: tableUpsertRowTool, + table_update_row: tableUpdateRowTool, + table_update_rows_by_filter: tableUpdateRowsByFilterTool, + table_delete_row: tableDeleteRowTool, + table_delete_rows_by_filter: tableDeleteRowsByFilterTool, + table_query_rows_v2: tableQueryRowsV2Tool, + table_get_row: tableGetRowTool, + table_get_schema: tableGetSchemaTool, + table_create: tableCreateTool, + table_list: tableListTool, guardrails_validate: guardrailsValidateTool, // Needed so workflow-as-tool and custom (deploy-as-block) tools resolve their // config in minimal-registry dev mode (both route through `workflow_executor`). diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 1e2facaa404..06fdf3eb441 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -4153,6 +4153,7 @@ import { tableInsertRowTool, tableListTool, tableQueryRowsTool, + tableQueryRowsV2Tool, tableUpdateRowsByFilterTool, tableUpdateRowTool, tableUpsertRowTool, @@ -8796,6 +8797,7 @@ export const tools: Record = { table_delete_row: tableDeleteRowTool, table_delete_rows_by_filter: tableDeleteRowsByFilterTool, table_query_rows: tableQueryRowsTool, + table_query_rows_v2: tableQueryRowsV2Tool, table_get_row: tableGetRowTool, table_get_schema: tableGetSchemaTool, mailchimp_get_audiences: mailchimpGetAudiencesTool, diff --git a/apps/sim/tools/table/index.ts b/apps/sim/tools/table/index.ts index 1d4b84596c0..7b99fecedce 100644 --- a/apps/sim/tools/table/index.ts +++ b/apps/sim/tools/table/index.ts @@ -7,6 +7,7 @@ export * from './get_schema' export * from './insert_row' export * from './list' export * from './query_rows' +export * from './query_rows_v2' export * from './types' export * from './update_row' export * from './update_rows_by_filter' diff --git a/apps/sim/tools/table/query_rows_v2.ts b/apps/sim/tools/table/query_rows_v2.ts new file mode 100644 index 00000000000..6f589de7450 --- /dev/null +++ b/apps/sim/tools/table/query_rows_v2.ts @@ -0,0 +1,106 @@ +import type { TableQueryV2Response, TableRowQueryV2Params } from '@/tools/table/types' +import type { ToolConfig } from '@/tools/types' + +/** + * v2 row query: typed predicate grammar + opaque cursor pagination (no offset). + * Hits POST `/api/table/[tableId]/query`. + */ +export const tableQueryRowsV2Tool: ToolConfig = { + id: 'table_query_rows_v2', + name: 'Query Rows', + description: + 'Query rows with a typed predicate filter and cursor pagination. ' + + 'Filter is a predicate tree: `{"all":[{"field":"wins","op":"gte","value":10},{"field":"status","op":"in","value":["active","pending"]}]}` ' + + '(`all` = AND, `any` = OR; groups nest). Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, ' + + 'nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. ' + + 'Order is a sort spec, e.g. `[{"field":"wins","direction":"desc"}]`. Omit limit to return the entire result — ' + + 'the query fails if it exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, ' + + 'a page can end early at the byte budget: a non-null nextCursor means more rows exist — pass it back ' + + 'as cursor to continue; never infer completion from page size.', + version: '1.0.0', + + params: { + tableId: { + type: 'string', + required: true, + description: 'Table ID', + visibility: 'user-only', + }, + filter: { + type: 'json', + required: false, + description: + 'Predicate tree, e.g. `{"all":[{"field":"wins","op":"gte","value":10}]}`. Omit to match all rows.', + visibility: 'user-or-llm', + }, + order: { + type: 'json', + required: false, + description: 'Sort spec, e.g. `[{"field":"wins","direction":"desc"}]`.', + visibility: 'user-or-llm', + }, + limit: { + type: 'number', + required: false, + description: + 'Maximum rows per page. Omit to return the entire matching result — fails if it exceeds the 5MB budget. With a limit, pages may byte-cut early and set nextCursor when more remain.', + visibility: 'user-or-llm', + }, + cursor: { + type: 'string', + required: false, + description: 'Opaque pagination cursor returned by a prior query. Omit for the first page.', + visibility: 'user-or-llm', + }, + }, + + request: { + url: (params: TableRowQueryV2Params) => `/api/table/${params.tableId}/query`, + method: 'POST', + headers: () => ({ 'Content-Type': 'application/json' }), + body: (params: TableRowQueryV2Params) => { + const workspaceId = params._context?.workspaceId + if (!workspaceId) { + throw new Error('Workspace ID is required in execution context') + } + return { + workspaceId, + ...(params.filter ? { predicate: params.filter } : {}), + ...(params.order ? { sort: params.order } : {}), + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.cursor ? { cursor: params.cursor } : {}), + } + }, + }, + + transformResponse: async (response): Promise => { + const result = await response.json() + const data = result.data || result + + return { + success: true, + output: { + rows: data.rows, + rowCount: data.rowCount, + totalCount: data.totalCount, + limit: data.limit, + nextCursor: data.nextCursor, + }, + } + }, + + outputs: { + success: { type: 'boolean', description: 'Whether the query succeeded' }, + rows: { type: 'array', description: 'Query result rows' }, + rowCount: { type: 'number', description: 'Number of rows returned' }, + totalCount: { + type: 'number', + description: 'Total rows matching the predicate (computed on the first page only)', + }, + limit: { type: 'number', description: 'Limit used in the query' }, + nextCursor: { + type: 'string', + description: 'Cursor to fetch the next page, or null on the last page', + }, + }, +} diff --git a/apps/sim/tools/table/types.ts b/apps/sim/tools/table/types.ts index ee3e347a590..f5f2633a2e3 100644 --- a/apps/sim/tools/table/types.ts +++ b/apps/sim/tools/table/types.ts @@ -3,7 +3,9 @@ import type { Filter, RowData, Sort, + SortSpec, TableDefinition, + TablePredicate, TableRow, TableSchema, } from '@/lib/table/types' @@ -56,6 +58,26 @@ export interface TableRowGetParams { _context?: WorkflowToolExecutionContext } +/** v2 query params: typed predicate/sort objects + opaque cursor (no offset). */ +export interface TableRowQueryV2Params { + tableId: string + filter?: TablePredicate + order?: SortSpec + limit?: number + cursor?: string + _context?: WorkflowToolExecutionContext +} + +export interface TableQueryV2Response extends ToolResponse { + output: { + rows: TableRow[] + rowCount: number + totalCount: number | null + limit: number + nextCursor: string | null + } +} + export interface TableCreateResponse extends ToolResponse { output: { table: TableDefinition diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 4e7f5f6ca23..8df1ad2a511 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 994, - zodRoutes: 994, + totalRoutes: 997, + zodRoutes: 997, nonZodRoutes: 0, } as const