diff --git a/.server-changes/sentry-request-attribution.md b/.server-changes/sentry-request-attribution.md new file mode 100644 index 00000000000..d42abf78d09 --- /dev/null +++ b/.server-changes/sentry-request-attribution.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fixed error reports being attributed to the wrong request when several requests were in flight at once. diff --git a/.server-changes/tsql-query-error-log-levels.md b/.server-changes/tsql-query-error-log-levels.md new file mode 100644 index 00000000000..1aaedba3b76 --- /dev/null +++ b/.server-changes/tsql-query-error-log-levels.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Invalid queries sent to the query API are no longer treated as internal errors, and a query that does fail is now recorded together with the query text that produced it. diff --git a/apps/webapp/app/components/dashboard-agent/AgentChart.tsx b/apps/webapp/app/components/dashboard-agent/AgentChart.tsx index 6e082231eb6..dbd1297c58f 100644 --- a/apps/webapp/app/components/dashboard-agent/AgentChart.tsx +++ b/apps/webapp/app/components/dashboard-agent/AgentChart.tsx @@ -67,6 +67,7 @@ export function AgentChart({ block }: { block: ChartBlock }) { period: block.period ?? null, from: null, to: null, + userAuthoredQuery: true, }), signal: controller.signal, }) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx index 848546a518f..c16037f5e10 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx @@ -183,6 +183,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const queryResult = await executeQuery({ name: "query-page", query, + userAuthoredQuery: true, scope, organizationId: project.organizationId, projectId: project.id, diff --git a/apps/webapp/app/routes/api.v1.query.ts b/apps/webapp/app/routes/api.v1.query.ts index b3459fa51c8..2f4011455ba 100644 --- a/apps/webapp/app/routes/api.v1.query.ts +++ b/apps/webapp/app/routes/api.v1.query.ts @@ -54,6 +54,7 @@ const { action, loader } = createActionApiRoute( const queryResult = await executeQuery({ name: "api-query", query, + userAuthoredQuery: true, scope: scope as QueryScope, organizationId: env.organization.id, projectId: env.project.id, diff --git a/apps/webapp/app/routes/resources.metric.tsx b/apps/webapp/app/routes/resources.metric.tsx index 5bf0ed693ad..1808f6107e4 100644 --- a/apps/webapp/app/routes/resources.metric.tsx +++ b/apps/webapp/app/routes/resources.metric.tsx @@ -52,6 +52,7 @@ const MetricWidgetQuery = z.object({ tags: z.array(z.string()).optional(), // Opt into server-side gap fill (carry-forward for gauges, zero-fill for counters). fillGaps: z.boolean().optional(), + userAuthoredQuery: z.boolean().optional(), }); export const action = async ({ request }: ActionFunctionArgs) => { @@ -88,6 +89,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { providers, tags: _tags, fillGaps, + userAuthoredQuery, } = submission.data; // Check they should be able to access it @@ -126,6 +128,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { operations, providers, fillGaps, + userAuthoredQuery, // Set higher concurrency if many widgets are on screen at once customOrgConcurrencyLimit: env.METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT, }); diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 3a34f82cf1d..bcceca25bbb 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -100,6 +100,13 @@ export type ExecuteQueryOptions = Omit< }; /** Custom per-org concurrency limit (overrides default) */ customOrgConcurrencyLimit?: number; + /** + * Set when the caller wrote `query` themselves, as on the public query API and + * the query editor. ClickHouse rejecting their SQL is then their mistake, so it + * is logged as a warning instead of raising an alert. Leave unset for TRQL we + * generate, where the same rejection is a bug worth alerting on. + */ + userAuthoredQuery?: boolean; }; /** diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index 3b924ff8a19..87e0877091f 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -1,6 +1,7 @@ import { type Attributes, type Context, + context as otelContext, createContextKey, DiagConsoleLogger, DiagLogLevel, @@ -14,6 +15,7 @@ import { metrics, type Meter, } from "@opentelemetry/api"; +import sentryRemix from "@sentry/remix"; import { logs, SeverityNumber } from "@opentelemetry/api-logs"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http"; @@ -209,10 +211,32 @@ function getResource() { return baseResource.merge(detectedResource); } +/** + * Sentry's `withIsolationScope` only marks the OTel context; the fork itself is + * done by Sentry's context manager. We pass `skipOpenTelemetrySetup: true` to + * `Sentry.init` because we run our own OTel pipeline, which also skips the + * `setGlobalContextManager(new SentryContextManager())` that Sentry would + * otherwise do. Registering it here is what keeps per-request scopes (and so + * the request attributed to each Sentry event) from leaking between concurrent + * requests. It extends `AsyncLocalStorageContextManager`, so OTel behaviour is + * unchanged. + * + * Reached through the default export because `@sentry/remix` is CommonJS and + * Node's ESM loader does not detect this transitively re-exported name, so a + * named import resolves at build time and then fails when the server boots. + */ +function createContextManager() { + return new sentryRemix.SentryContextManager(); +} + function setupTelemetry() { if (env.INTERNAL_OTEL_TRACE_DISABLED === "1") { console.log(`🔦 Tracer disabled, returning a noop tracer`); + const contextManager = createContextManager(); + contextManager.enable(); + otelContext.setGlobalContextManager(contextManager); + return { tracer: trace.getTracer("trigger.dev", "3.3.12"), logger: logs.getLogger("trigger.dev", "3.3.12"), @@ -300,7 +324,7 @@ function setupTelemetry() { ); } - provider.register(); + provider.register({ contextManager: createContextManager() }); let instrumentations: Instrumentation[] = [ new AwsSdkInstrumentation({ diff --git a/apps/webapp/test/sentryRequestIsolation.test.ts b/apps/webapp/test/sentryRequestIsolation.test.ts new file mode 100644 index 00000000000..ebcb0af6929 --- /dev/null +++ b/apps/webapp/test/sentryRequestIsolation.test.ts @@ -0,0 +1,51 @@ +import { context } from "@opentelemetry/api"; +import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; +import * as Sentry from "@sentry/remix"; +import sentryRemix from "@sentry/remix"; +import { afterEach, beforeAll, describe, expect, it } from "vitest"; + +/** + * Two overlapping requests, each tagging its own isolation scope, mirroring what + * `SentryHttpInstrumentation` does per incoming request. Returns what each one + * reads back after the other has started. + */ +async function raceTwoRequests(): Promise> { + const observed: Record = {}; + + const handleRequest = (name: string, holdMs: number) => + Sentry.withIsolationScope(async () => { + Sentry.getIsolationScope().setTag("request", name); + await new Promise((resolve) => setTimeout(resolve, holdMs)); + observed[name] = Sentry.getIsolationScope().getScopeData().tags.request; + }); + + await Promise.all([handleRequest("slow", 30), handleRequest("fast", 5)]); + + return observed; +} + +describe("Sentry request isolation", () => { + beforeAll(() => { + Sentry.init({ dsn: undefined, defaultIntegrations: false, skipOpenTelemetrySetup: true }); + }); + + afterEach(() => { + context.disable(); + }); + + it("leaks the isolation scope between concurrent requests without SentryContextManager", async () => { + new NodeTracerProvider().register(); + + const observed = await raceTwoRequests(); + + expect(observed).toEqual({ slow: "fast", fast: "fast" }); + }); + + it("keeps each request's isolation scope separate with SentryContextManager", async () => { + new NodeTracerProvider().register({ contextManager: new sentryRemix.SentryContextManager() }); + + const observed = await raceTwoRequests(); + + expect(observed).toEqual({ slow: "slow", fast: "fast" }); + }); +}); diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index d91f86b428f..ef18883745b 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -171,13 +171,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + }; + + this.logger.error("Error querying clickhouse", errorLogFields); recordClickhouseError(span, clickhouseError); @@ -260,6 +262,16 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { * These will be merged with the default settings. */ settings?: ClickHouseSettings; + /** + * Extra fields to attach to the error log if the query fails. Use this to + * record what produced the SQL, e.g. the TSQL a caller actually wrote. + */ + logFields?: Record; + /** + * Set when the SQL originates from whoever made the request rather than + * from us. Invalid-SQL rejections are then their mistake, not a bug. + */ + userAuthoredQuery?: boolean; }): ClickhouseQueryWithStatsFunction, z.output> { return async (params, options) => { const queryId = randomUUID(); @@ -320,13 +332,25 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { + ...req.logFields, name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + }; + + switch (classifyClickhouseError(clickhouseError, req.userAuthoredQuery)) { + case "quota": + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + break; + case "invalid-sql": + this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); + break; + default: + this.logger.error("Error querying clickhouse", errorLogFields); + } recordClickhouseError(span, clickhouseError); @@ -453,13 +477,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + }; + + this.logger.error("Error querying clickhouse", errorLogFields); recordClickhouseError(span, clickhouseError); @@ -599,13 +625,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { span.setAttributes({ "clickhouse.rows": rowCount }); } catch (error) { - self.logger.error("Error streaming clickhouse", { + const errorLogFields = { name: req.name, error, query: req.query, params, queryId, - }); + }; + + self.logger.error("Error streaming clickhouse", errorLogFields); if (error instanceof Error) { recordClickhouseError(span, error); @@ -1001,6 +1029,65 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { } } +/** + * ClickHouse error types raised by a query that is valid but asks for more than + * it is allowed to spend. Only downgraded for SQL the caller wrote: a runaway + * query we generated is our bug and still has to alert. + */ +const CLICKHOUSE_QUOTA_ERROR_TYPES = new Set([ + "MEMORY_LIMIT_EXCEEDED", + "TIMEOUT_EXCEEDED", + "TOO_SLOW", + "TOO_MANY_ROWS", + "TOO_MANY_BYTES", + "TOO_MANY_ROWS_OR_BYTES", +]); + +/** + * ClickHouse error types that mean the SQL itself is wrong. Only treated as the + * caller's fault when the query was written by the caller — the same error on a + * query we generated is our bug and has to keep alerting. + */ +const CLICKHOUSE_INVALID_SQL_ERROR_TYPES = new Set([ + "NOT_AN_AGGREGATE", + "ILLEGAL_AGGREGATION", + "UNKNOWN_IDENTIFIER", + "UNKNOWN_FUNCTION", + "UNKNOWN_TABLE", + "AMBIGUOUS_COLUMN_NAME", + "MULTIPLE_EXPRESSIONS_FOR_ALIAS", + "SYNTAX_ERROR", + "BAD_ARGUMENTS", + "TYPE_MISMATCH", + "NO_COMMON_TYPE", + "ILLEGAL_TYPE_OF_ARGUMENT", + "ILLEGAL_COLUMN", + "CANNOT_CONVERT_TYPE", + "CANNOT_PARSE_TEXT", + "CANNOT_PARSE_NUMBER", + "CANNOT_PARSE_DATE", + "CANNOT_PARSE_DATETIME", + "CANNOT_PARSE_INPUT_ASSERTION_FAILED", +]); + +type ClickhouseErrorCategory = "quota" | "invalid-sql" | "fault"; + +function classifyClickhouseError( + error: Error, + userAuthoredQuery: boolean | undefined +): ClickhouseErrorCategory { + if (!userAuthoredQuery || !(error instanceof ClickHouseError) || error.type === undefined) { + return "fault"; + } + if (CLICKHOUSE_QUOTA_ERROR_TYPES.has(error.type)) { + return "quota"; + } + if (CLICKHOUSE_INVALID_SQL_ERROR_TYPES.has(error.type)) { + return "invalid-sql"; + } + return "fault"; +} + function recordClickhouseError(span: Span, error: Error): void { if (error instanceof ClickHouseError) { span.setAttributes({ diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index ddf1d059b97..2cf586c36f1 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -8,6 +8,7 @@ import type { ClickHouseSettings } from "@clickhouse/client"; import { compileTSQL, + ExposedTSQLError, type OutputColumnMetadata, sanitizeErrorMessage, transformResults, @@ -113,6 +114,11 @@ export interface ExecuteTSQLOptions { * (counters zero-fill, gauges carry forward). Off by default. */ fillGaps?: boolean; + /** + * Set when `query` was written by whoever made the request rather than by us. + * A rejection of their SQL is then their mistake, not a bug on our side. + */ + userAuthoredQuery?: boolean; } /** @@ -213,6 +219,8 @@ export async function executeTSQL( // EXPLAIN returns rows with an 'explain' column schema: isExplain ? z.object({ explain: z.string() }) : options.schema, settings: options.clickhouseSettings, + logFields: { tsql: options.query }, + userAuthoredQuery: options.userAuthoredQuery, }); const [error, result] = await queryFn(params); @@ -246,6 +254,8 @@ export async function executeTSQL( params: z.record(z.any()), schema: z.object({ explain: z.string() }), settings: options.clickhouseSettings, + logFields: { tsql: options.query }, + userAuthoredQuery: options.userAuthoredQuery, }); const [additionalError, additionalResult] = await additionalQueryFn(params); @@ -303,14 +313,21 @@ export async function executeTSQL( } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error"; - // Log TSQL compilation or unexpected errors (with original message for debugging) - logger.error("[TSQL] Query error", { + const logFields = { name: options.name, error: errorMessage, tsql: options.query, generatedSql: generatedSql ?? "(compilation failed)", generatedParams: generatedParams ?? {}, - }); + }; + + const callerWroteABadQuery = options.userAuthoredQuery && error instanceof ExposedTSQLError; + + if (callerWroteABadQuery) { + logger.warn("[TSQL] Invalid query", logFields); + } else { + logger.error("[TSQL] Query error", logFields); + } // Sanitize error message to show TSQL names instead of ClickHouse internals const sanitizedMessage = sanitizeErrorMessage(errorMessage, options.tableSchema); diff --git a/internal-packages/clickhouse/src/client/types.ts b/internal-packages/clickhouse/src/client/types.ts index d785ab34ebb..4bfa6dc4662 100644 --- a/internal-packages/clickhouse/src/client/types.ts +++ b/internal-packages/clickhouse/src/client/types.ts @@ -135,6 +135,16 @@ export interface ClickhouseReader { * These will be merged with the default settings. */ settings?: ClickHouseSettings; + /** + * Extra fields to attach to the error log if the query fails. Use this to + * record what produced the SQL, e.g. the TSQL a caller actually wrote. + */ + logFields?: Record; + /** + * Set when the SQL originates from whoever made the request rather than + * from us. Invalid-SQL rejections are then their mistake, not a bug. + */ + userAuthoredQuery?: boolean; }): ClickhouseQueryWithStatsFunction, z.output>; queryFast, TParams extends Record>(req: { diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index b0a93bfe704..ff16508fdc7 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -1,4 +1,5 @@ import { clickhouseTest } from "@internal/testcontainers"; +import type { MockInstance } from "vitest"; import { z } from "zod"; import { ClickhouseClient } from "./client/client.js"; import { executeTSQL, createTSQLExecutor, type TableSchema } from "./client/tsql.js"; @@ -1609,3 +1610,222 @@ describe("Field Mapping Tests", () => { expect(result?.rows?.map((r) => r.run_id).sort()).toEqual(["run_fm_in1", "run_fm_in2"]); }); }); + +describe("TSQL Error Log Levels", () => { + let warnSpy: MockInstance; + let errorSpy: MockInstance; + + beforeEach(() => { + warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + function logged(spy: MockInstance): string { + return spy.mock.calls.map(([line]) => String(line)).join("\n"); + } + + clickhouseTest("logs an unknown column as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-unknown-column", + query: "SELECT nope FROM task_runs", + schema: z.object({ nope: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("[TSQL] Invalid query"); + expect(logged(errorSpy)).not.toContain("[TSQL] Query error"); + }); + + clickhouseTest("logs a syntax error as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-syntax-error", + query: "SELECT FROM WHERE", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("[TSQL] Invalid query"); + expect(logged(errorSpy)).not.toContain("[TSQL] Query error"); + }); + + clickhouseTest( + "logs a query ClickHouse rejects at execution as an error, with the TSQL that produced it", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-execution-error", + query: "SELECT toDateTime(tags) AS bad FROM task_runs", + schema: z.object({ bad: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(errorSpy)).toContain("SELECT toDateTime(tags) AS bad FROM task_runs"); + } + ); + + clickhouseTest( + "keeps a compile failure on TRQL we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-internal-compile-error", + query: "SELECT nope FROM task_runs", + schema: z.object({ nope: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("[TSQL] Query error"); + expect(logged(warnSpy)).not.toContain("[TSQL] Invalid query"); + } + ); + + clickhouseTest( + "logs invalid caller-written SQL as a warning", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-user-authored-invalid", + query: "SELECT status, sum(is_test) AS n FROM task_runs", + schema: z.object({ status: z.string(), n: z.number() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("ClickHouse rejected an invalid query"); + expect(logged(errorSpy)).not.toContain("Error querying clickhouse"); + } + ); + + clickhouseTest( + "keeps invalid SQL we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-internal-invalid", + query: "SELECT status, sum(is_test) AS n FROM task_runs", + schema: z.object({ status: z.string(), n: z.number() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(warnSpy)).not.toContain("ClickHouse rejected an invalid query"); + } + ); + + clickhouseTest("logs a ClickHouse limit breach as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + await insertTaskRuns(client, { async_insert: 0 })([ + createTaskRun({ run_id: "run_limit1" }), + createTaskRun({ run_id: "run_limit2" }), + createTaskRun({ run_id: "run_limit3" }), + ]); + + const [error] = await executeTSQL(client, { + name: "test-resource-limit", + query: "SELECT run_id FROM task_runs", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + clickhouseSettings: { max_rows_to_read: "1" }, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("Query exceeded a ClickHouse limit"); + expect(logged(errorSpy)).not.toContain("Error querying clickhouse"); + }); + + clickhouseTest( + "keeps a limit breach on a query we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + await insertTaskRuns(client, { async_insert: 0 })([ + createTaskRun({ run_id: "run_intlimit1" }), + createTaskRun({ run_id: "run_intlimit2" }), + createTaskRun({ run_id: "run_intlimit3" }), + ]); + + const [error] = await executeTSQL(client, { + name: "test-internal-resource-limit", + query: "SELECT run_id FROM task_runs", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + clickhouseSettings: { max_rows_to_read: "1" }, + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(warnSpy)).not.toContain("Query exceeded a ClickHouse limit"); + } + ); +});