From dd2975cad09246365e0de0353a8ca413c7955620 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 24 Jul 2026 22:03:08 +0100 Subject: [PATCH 1/7] fix(webapp,clickhouse): stop invalid customer queries alerting, and isolate Sentry scope per request Three fixes to how query failures are reported. Invalid TSQL is a caller mistake, not ours: executeTSQL now logs ExposedTSQLError at warn and reserves error for InternalTSQLError and unanticipated exceptions, so a bad column name no longer raises an alert. The route above it already returned 400 and logged at warn; the layer below was overriding that decision. ClickHouse rejections that come from a query asking for too much (memory ceiling, timeout, row/byte caps) drop to warn as well. Those are decided in the client, which is the only place holding the parsed ClickHouseError type, and queryWithStats gained a logFields option so a failing query is recorded with the TSQL that generated it rather than the generated SQL alone. Sentry.init runs with skipOpenTelemetrySetup because we register our own OTel pipeline, which also skipped installing SentryContextManager. withIsolationScope only marks the context and relies on that manager to fork, so without it every request shared one global isolation scope and events were attributed to whichever request wrote last. The tracer now registers it, including on the path where tracing is disabled and register() was never called. --- .server-changes/sentry-request-attribution.md | 6 + .../tsql-query-error-log-levels.md | 6 + apps/webapp/app/v3/tracer.server.ts | 22 +++- .../test/sentryRequestIsolation.test.ts | 51 ++++++++ .../clickhouse/src/client/client.ts | 59 ++++++++- .../clickhouse/src/client/tsql.ts | 15 ++- .../clickhouse/src/client/types.ts | 5 + internal-packages/clickhouse/src/tsql.test.ts | 113 ++++++++++++++++++ 8 files changed, 267 insertions(+), 10 deletions(-) create mode 100644 .server-changes/sentry-request-attribution.md create mode 100644 .server-changes/tsql-query-error-log-levels.md create mode 100644 apps/webapp/test/sentryRequestIsolation.test.ts diff --git a/.server-changes/sentry-request-attribution.md b/.server-changes/sentry-request-attribution.md new file mode 100644 index 00000000000..d42abf78d09 --- /dev/null +++ b/.server-changes/sentry-request-attribution.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fixed error reports being attributed to the wrong request when several requests were in flight at once. diff --git a/.server-changes/tsql-query-error-log-levels.md b/.server-changes/tsql-query-error-log-levels.md new file mode 100644 index 00000000000..1aaedba3b76 --- /dev/null +++ b/.server-changes/tsql-query-error-log-levels.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Invalid queries sent to the query API are no longer treated as internal errors, and a query that does fail is now recorded together with the query text that produced it. diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index 3b924ff8a19..a55a9381083 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -1,6 +1,7 @@ import { type Attributes, type Context, + context as otelContext, createContextKey, DiagConsoleLogger, DiagLogLevel, @@ -14,6 +15,7 @@ import { metrics, type Meter, } from "@opentelemetry/api"; +import { SentryContextManager } from "@sentry/remix"; import { logs, SeverityNumber } from "@opentelemetry/api-logs"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http"; @@ -209,10 +211,28 @@ function getResource() { return baseResource.merge(detectedResource); } +/** + * Sentry's `withIsolationScope` only marks the OTel context; the fork itself is + * done by Sentry's context manager. We pass `skipOpenTelemetrySetup: true` to + * `Sentry.init` because we run our own OTel pipeline, which also skips the + * `setGlobalContextManager(new SentryContextManager())` that Sentry would + * otherwise do. Registering it here is what keeps per-request scopes (and so + * the request attributed to each Sentry event) from leaking between concurrent + * requests. It extends `AsyncLocalStorageContextManager`, so OTel behaviour is + * unchanged. + */ +function createContextManager() { + return new SentryContextManager(); +} + function setupTelemetry() { if (env.INTERNAL_OTEL_TRACE_DISABLED === "1") { console.log(`🔦 Tracer disabled, returning a noop tracer`); + const contextManager = createContextManager(); + contextManager.enable(); + otelContext.setGlobalContextManager(contextManager); + return { tracer: trace.getTracer("trigger.dev", "3.3.12"), logger: logs.getLogger("trigger.dev", "3.3.12"), @@ -300,7 +320,7 @@ function setupTelemetry() { ); } - provider.register(); + provider.register({ contextManager: createContextManager() }); let instrumentations: Instrumentation[] = [ new AwsSdkInstrumentation({ diff --git a/apps/webapp/test/sentryRequestIsolation.test.ts b/apps/webapp/test/sentryRequestIsolation.test.ts new file mode 100644 index 00000000000..1205981d513 --- /dev/null +++ b/apps/webapp/test/sentryRequestIsolation.test.ts @@ -0,0 +1,51 @@ +import { context } from "@opentelemetry/api"; +import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; +import * as Sentry from "@sentry/remix"; +import { SentryContextManager } from "@sentry/remix"; +import { afterEach, beforeAll, describe, expect, it } from "vitest"; + +/** + * Two overlapping requests, each tagging its own isolation scope, mirroring what + * `SentryHttpInstrumentation` does per incoming request. Returns what each one + * reads back after the other has started. + */ +async function raceTwoRequests(): Promise> { + const observed: Record = {}; + + const handleRequest = (name: string, holdMs: number) => + Sentry.withIsolationScope(async () => { + Sentry.getIsolationScope().setTag("request", name); + await new Promise((resolve) => setTimeout(resolve, holdMs)); + observed[name] = Sentry.getIsolationScope().getScopeData().tags.request; + }); + + await Promise.all([handleRequest("slow", 30), handleRequest("fast", 5)]); + + return observed; +} + +describe("Sentry request isolation", () => { + beforeAll(() => { + Sentry.init({ dsn: undefined, defaultIntegrations: false, skipOpenTelemetrySetup: true }); + }); + + afterEach(() => { + context.disable(); + }); + + it("leaks the isolation scope between concurrent requests without SentryContextManager", async () => { + new NodeTracerProvider().register(); + + const observed = await raceTwoRequests(); + + expect(observed).toEqual({ slow: "fast", fast: "fast" }); + }); + + it("keeps each request's isolation scope separate with SentryContextManager", async () => { + new NodeTracerProvider().register({ contextManager: new SentryContextManager() }); + + const observed = await raceTwoRequests(); + + expect(observed).toEqual({ slow: "slow", fast: "fast" }); + }); +}); diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index d91f86b428f..067ba6c4216 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -171,13 +171,19 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + }; + + if (isClickhouseQuotaError(clickhouseError)) { + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + } else { + this.logger.error("Error querying clickhouse", errorLogFields); + } recordClickhouseError(span, clickhouseError); @@ -260,6 +266,11 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { * These will be merged with the default settings. */ settings?: ClickHouseSettings; + /** + * Extra fields to attach to the error log if the query fails. Use this to + * record what produced the SQL, e.g. the TSQL a caller actually wrote. + */ + logFields?: Record; }): ClickhouseQueryWithStatsFunction, z.output> { return async (params, options) => { const queryId = randomUUID(); @@ -320,13 +331,20 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + ...req.logFields, + }; + + if (isClickhouseQuotaError(clickhouseError)) { + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + } else { + this.logger.error("Error querying clickhouse", errorLogFields); + } recordClickhouseError(span, clickhouseError); @@ -453,13 +471,19 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + }; + + if (isClickhouseQuotaError(clickhouseError)) { + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + } else { + this.logger.error("Error querying clickhouse", errorLogFields); + } recordClickhouseError(span, clickhouseError); @@ -1001,6 +1025,29 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { } } +/** + * ClickHouse error types raised by a query that is valid but asks for more than + * the caller is allowed to spend. The caller gets a 4xx and there is nothing on + * our side to fix, so these are logged at warn rather than error. + */ +const CLICKHOUSE_QUOTA_ERROR_TYPES = new Set([ + "MEMORY_LIMIT_EXCEEDED", + "TIMEOUT_EXCEEDED", + "TOO_SLOW", + "TOO_MANY_ROWS", + "TOO_MANY_BYTES", + "TOO_MANY_ROWS_OR_BYTES", + "QUERY_WAS_CANCELLED", +]); + +function isClickhouseQuotaError(error: Error): boolean { + return ( + error instanceof ClickHouseError && + error.type !== undefined && + CLICKHOUSE_QUOTA_ERROR_TYPES.has(error.type) + ); +} + function recordClickhouseError(span: Span, error: Error): void { if (error instanceof ClickHouseError) { span.setAttributes({ diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index ddf1d059b97..244a65c565f 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -8,6 +8,7 @@ import type { ClickHouseSettings } from "@clickhouse/client"; import { compileTSQL, + ExposedTSQLError, type OutputColumnMetadata, sanitizeErrorMessage, transformResults, @@ -213,6 +214,7 @@ export async function executeTSQL( // EXPLAIN returns rows with an 'explain' column schema: isExplain ? z.object({ explain: z.string() }) : options.schema, settings: options.clickhouseSettings, + logFields: { tsql: options.query }, }); const [error, result] = await queryFn(params); @@ -303,14 +305,21 @@ export async function executeTSQL( } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error"; - // Log TSQL compilation or unexpected errors (with original message for debugging) - logger.error("[TSQL] Query error", { + const logFields = { name: options.name, error: errorMessage, tsql: options.query, generatedSql: generatedSql ?? "(compilation failed)", generatedParams: generatedParams ?? {}, - }); + }; + + const callerWroteABadQuery = error instanceof ExposedTSQLError; + + if (callerWroteABadQuery) { + logger.warn("[TSQL] Invalid query", logFields); + } else { + logger.error("[TSQL] Query error", logFields); + } // Sanitize error message to show TSQL names instead of ClickHouse internals const sanitizedMessage = sanitizeErrorMessage(errorMessage, options.tableSchema); diff --git a/internal-packages/clickhouse/src/client/types.ts b/internal-packages/clickhouse/src/client/types.ts index d785ab34ebb..5c674776370 100644 --- a/internal-packages/clickhouse/src/client/types.ts +++ b/internal-packages/clickhouse/src/client/types.ts @@ -135,6 +135,11 @@ export interface ClickhouseReader { * These will be merged with the default settings. */ settings?: ClickHouseSettings; + /** + * Extra fields to attach to the error log if the query fails. Use this to + * record what produced the SQL, e.g. the TSQL a caller actually wrote. + */ + logFields?: Record; }): ClickhouseQueryWithStatsFunction, z.output>; queryFast, TParams extends Record>(req: { diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index b0a93bfe704..937b2a2dcae 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -1,4 +1,5 @@ import { clickhouseTest } from "@internal/testcontainers"; +import type { MockInstance } from "vitest"; import { z } from "zod"; import { ClickhouseClient } from "./client/client.js"; import { executeTSQL, createTSQLExecutor, type TableSchema } from "./client/tsql.js"; @@ -1609,3 +1610,115 @@ describe("Field Mapping Tests", () => { expect(result?.rows?.map((r) => r.run_id).sort()).toEqual(["run_fm_in1", "run_fm_in2"]); }); }); + +describe("TSQL Error Log Levels", () => { + let warnSpy: MockInstance; + let errorSpy: MockInstance; + + beforeEach(() => { + warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + function logged(spy: MockInstance): string { + return spy.mock.calls.map(([line]) => String(line)).join("\n"); + } + + clickhouseTest("logs an unknown column as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-unknown-column", + query: "SELECT nope FROM task_runs", + schema: z.object({ nope: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("[TSQL] Invalid query"); + expect(logged(errorSpy)).not.toContain("[TSQL] Query error"); + }); + + clickhouseTest("logs a syntax error as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-syntax-error", + query: "SELECT FROM WHERE", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("[TSQL] Invalid query"); + expect(logged(errorSpy)).not.toContain("[TSQL] Query error"); + }); + + clickhouseTest( + "logs a query ClickHouse rejects at execution as an error, with the TSQL that produced it", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-execution-error", + query: "SELECT toDateTime(tags) AS bad FROM task_runs", + schema: z.object({ bad: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(errorSpy)).toContain("SELECT toDateTime(tags) AS bad FROM task_runs"); + } + ); + + clickhouseTest("logs a ClickHouse limit breach as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + await insertTaskRuns(client, { async_insert: 0 })([ + createTaskRun({ run_id: "run_limit1" }), + createTaskRun({ run_id: "run_limit2" }), + createTaskRun({ run_id: "run_limit3" }), + ]); + + const [error] = await executeTSQL(client, { + name: "test-resource-limit", + query: "SELECT run_id FROM task_runs", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + clickhouseSettings: { max_rows_to_read: "1" }, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("Query exceeded a ClickHouse limit"); + expect(logged(errorSpy)).not.toContain("Error querying clickhouse"); + }); +}); From 03f8355e10c1e3f9873538a76d7d01044e034d99 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 24 Jul 2026 22:10:28 +0100 Subject: [PATCH 2/7] fix(clickhouse): classify streamed limit failures and keep canonical log fields Spread caller-supplied logFields before the canonical ones so a caller cannot overwrite error, query, params, or queryId in a failure log. queryFastStream classifies quota failures the same way the buffered query paths do; a limit hit partway through a stream is still the caller asking for too much. The supplemental EXPLAIN queries carry the originating TSQL too. --- internal-packages/clickhouse/src/client/client.ts | 12 +++++++++--- internal-packages/clickhouse/src/client/tsql.ts | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index 067ba6c4216..49c594c296e 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -332,12 +332,12 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { if (clickhouseError) { const errorLogFields = { + ...req.logFields, name: req.name, error: clickhouseError, query: req.query, params, queryId, - ...req.logFields, }; if (isClickhouseQuotaError(clickhouseError)) { @@ -623,13 +623,19 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { span.setAttributes({ "clickhouse.rows": rowCount }); } catch (error) { - self.logger.error("Error streaming clickhouse", { + const errorLogFields = { name: req.name, error, query: req.query, params, queryId, - }); + }; + + if (error instanceof Error && isClickhouseQuotaError(error)) { + self.logger.warn("Streamed query exceeded a ClickHouse limit", errorLogFields); + } else { + self.logger.error("Error streaming clickhouse", errorLogFields); + } if (error instanceof Error) { recordClickhouseError(span, error); diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index 244a65c565f..c39794bf282 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -248,6 +248,7 @@ export async function executeTSQL( params: z.record(z.any()), schema: z.object({ explain: z.string() }), settings: options.clickhouseSettings, + logFields: { tsql: options.query }, }); const [additionalError, additionalResult] = await additionalQueryFn(params); From 5e6956f5b00c368b62f305fa292d3279297830a9 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 24 Jul 2026 22:24:57 +0100 Subject: [PATCH 3/7] fix(webapp): reach SentryContextManager through the default export The webapp server bundle is ESM and @sentry/remix is CommonJS. Node's loader derives named exports by static analysis, and it does not see SentryContextManager because that name is re-exported transitively from @sentry/node-core. A named import type-checks and bundles, then throws SyntaxError when the server boots. The property is reachable on the default export, which for a CommonJS module is module.exports. Vitest resolves the named import fine, so this only shows up when the built server actually starts. --- apps/webapp/app/v3/tracer.server.ts | 8 ++++++-- apps/webapp/test/sentryRequestIsolation.test.ts | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index a55a9381083..87e0877091f 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -15,7 +15,7 @@ import { metrics, type Meter, } from "@opentelemetry/api"; -import { SentryContextManager } from "@sentry/remix"; +import sentryRemix from "@sentry/remix"; import { logs, SeverityNumber } from "@opentelemetry/api-logs"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http"; @@ -220,9 +220,13 @@ function getResource() { * the request attributed to each Sentry event) from leaking between concurrent * requests. It extends `AsyncLocalStorageContextManager`, so OTel behaviour is * unchanged. + * + * Reached through the default export because `@sentry/remix` is CommonJS and + * Node's ESM loader does not detect this transitively re-exported name, so a + * named import resolves at build time and then fails when the server boots. */ function createContextManager() { - return new SentryContextManager(); + return new sentryRemix.SentryContextManager(); } function setupTelemetry() { diff --git a/apps/webapp/test/sentryRequestIsolation.test.ts b/apps/webapp/test/sentryRequestIsolation.test.ts index 1205981d513..ebcb0af6929 100644 --- a/apps/webapp/test/sentryRequestIsolation.test.ts +++ b/apps/webapp/test/sentryRequestIsolation.test.ts @@ -1,7 +1,7 @@ import { context } from "@opentelemetry/api"; import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; import * as Sentry from "@sentry/remix"; -import { SentryContextManager } from "@sentry/remix"; +import sentryRemix from "@sentry/remix"; import { afterEach, beforeAll, describe, expect, it } from "vitest"; /** @@ -42,7 +42,7 @@ describe("Sentry request isolation", () => { }); it("keeps each request's isolation scope separate with SentryContextManager", async () => { - new NodeTracerProvider().register({ contextManager: new SentryContextManager() }); + new NodeTracerProvider().register({ contextManager: new sentryRemix.SentryContextManager() }); const observed = await raceTwoRequests(); From c6a52918712790a7d452ce93daf01ca8df8b566a Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 29 Jul 2026 16:47:48 +0100 Subject: [PATCH 4/7] fix(webapp,clickhouse): treat SQL the caller wrote as their mistake, not ours The quota list was built on the wrong axis. It classified by resource limit, but production shows the volume is in plain SQL mistakes: one error group alone, a missing GROUP BY on the public query API, accounts for over a million events across hundreds of users, and it stayed at error level because its code is not a limit. ClickHouse rejections that mean the SQL itself is wrong now log at warn, but only when the caller wrote that SQL. The same rejection on TRQL we generated is our bug and keeps alerting, so the decision is gated on a userAuthoredQuery flag set by the public query API and the query editor. Generated dashboard tiles deliberately do not set it. Drops QUERY_WAS_CANCELLED from the limit list. It was there on the reasoning that client disconnects would be noisy; it has not fired once in the last fourteen days, so it was speculation rather than evidence. --- .../route.tsx | 1 + apps/webapp/app/routes/api.v1.query.ts | 1 + .../app/services/queryService.server.ts | 7 ++ .../clickhouse/src/client/client.ts | 96 +++++++++++++++---- .../clickhouse/src/client/tsql.ts | 7 ++ .../clickhouse/src/client/types.ts | 5 + internal-packages/clickhouse/src/tsql.test.ts | 49 ++++++++++ 7 files changed, 146 insertions(+), 20 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx index 848546a518f..c16037f5e10 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx @@ -183,6 +183,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const queryResult = await executeQuery({ name: "query-page", query, + userAuthoredQuery: true, scope, organizationId: project.organizationId, projectId: project.id, diff --git a/apps/webapp/app/routes/api.v1.query.ts b/apps/webapp/app/routes/api.v1.query.ts index b3459fa51c8..2f4011455ba 100644 --- a/apps/webapp/app/routes/api.v1.query.ts +++ b/apps/webapp/app/routes/api.v1.query.ts @@ -54,6 +54,7 @@ const { action, loader } = createActionApiRoute( const queryResult = await executeQuery({ name: "api-query", query, + userAuthoredQuery: true, scope: scope as QueryScope, organizationId: env.organization.id, projectId: env.project.id, diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 3a34f82cf1d..bcceca25bbb 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -100,6 +100,13 @@ export type ExecuteQueryOptions = Omit< }; /** Custom per-org concurrency limit (overrides default) */ customOrgConcurrencyLimit?: number; + /** + * Set when the caller wrote `query` themselves, as on the public query API and + * the query editor. ClickHouse rejecting their SQL is then their mistake, so it + * is logged as a warning instead of raising an alert. Leave unset for TRQL we + * generate, where the same rejection is a bug worth alerting on. + */ + userAuthoredQuery?: boolean; }; /** diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index 49c594c296e..f35f83d214c 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -179,10 +179,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { queryId, }; - if (isClickhouseQuotaError(clickhouseError)) { - this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); - } else { - this.logger.error("Error querying clickhouse", errorLogFields); + switch (classifyClickhouseError(clickhouseError, false)) { + case "quota": + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + break; + case "invalid-sql": + this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); + break; + default: + this.logger.error("Error querying clickhouse", errorLogFields); } recordClickhouseError(span, clickhouseError); @@ -271,6 +276,11 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { * record what produced the SQL, e.g. the TSQL a caller actually wrote. */ logFields?: Record; + /** + * Set when the SQL originates from whoever made the request rather than + * from us. Invalid-SQL rejections are then their mistake, not a bug. + */ + userAuthoredQuery?: boolean; }): ClickhouseQueryWithStatsFunction, z.output> { return async (params, options) => { const queryId = randomUUID(); @@ -340,10 +350,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { queryId, }; - if (isClickhouseQuotaError(clickhouseError)) { - this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); - } else { - this.logger.error("Error querying clickhouse", errorLogFields); + switch (classifyClickhouseError(clickhouseError, req.userAuthoredQuery)) { + case "quota": + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + break; + case "invalid-sql": + this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); + break; + default: + this.logger.error("Error querying clickhouse", errorLogFields); } recordClickhouseError(span, clickhouseError); @@ -479,10 +494,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { queryId, }; - if (isClickhouseQuotaError(clickhouseError)) { - this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); - } else { - this.logger.error("Error querying clickhouse", errorLogFields); + switch (classifyClickhouseError(clickhouseError, false)) { + case "quota": + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + break; + case "invalid-sql": + this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); + break; + default: + this.logger.error("Error querying clickhouse", errorLogFields); } recordClickhouseError(span, clickhouseError); @@ -631,7 +651,7 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { queryId, }; - if (error instanceof Error && isClickhouseQuotaError(error)) { + if (error instanceof Error && classifyClickhouseError(error, false) === "quota") { self.logger.warn("Streamed query exceeded a ClickHouse limit", errorLogFields); } else { self.logger.error("Error streaming clickhouse", errorLogFields); @@ -1043,15 +1063,51 @@ const CLICKHOUSE_QUOTA_ERROR_TYPES = new Set([ "TOO_MANY_ROWS", "TOO_MANY_BYTES", "TOO_MANY_ROWS_OR_BYTES", - "QUERY_WAS_CANCELLED", ]); -function isClickhouseQuotaError(error: Error): boolean { - return ( - error instanceof ClickHouseError && - error.type !== undefined && - CLICKHOUSE_QUOTA_ERROR_TYPES.has(error.type) - ); +/** + * ClickHouse error types that mean the SQL itself is wrong. Only treated as the + * caller's fault when the query was written by the caller — the same error on a + * query we generated is our bug and has to keep alerting. + */ +const CLICKHOUSE_INVALID_SQL_ERROR_TYPES = new Set([ + "NOT_AN_AGGREGATE", + "ILLEGAL_AGGREGATION", + "UNKNOWN_IDENTIFIER", + "UNKNOWN_FUNCTION", + "UNKNOWN_TABLE", + "AMBIGUOUS_COLUMN_NAME", + "MULTIPLE_EXPRESSIONS_FOR_ALIAS", + "SYNTAX_ERROR", + "BAD_ARGUMENTS", + "TYPE_MISMATCH", + "NO_COMMON_TYPE", + "ILLEGAL_TYPE_OF_ARGUMENT", + "ILLEGAL_COLUMN", + "CANNOT_CONVERT_TYPE", + "CANNOT_PARSE_TEXT", + "CANNOT_PARSE_NUMBER", + "CANNOT_PARSE_DATE", + "CANNOT_PARSE_DATETIME", + "CANNOT_PARSE_INPUT_ASSERTION_FAILED", +]); + +type ClickhouseErrorCategory = "quota" | "invalid-sql" | "fault"; + +function classifyClickhouseError( + error: Error, + userAuthoredQuery: boolean | undefined +): ClickhouseErrorCategory { + if (!(error instanceof ClickHouseError) || error.type === undefined) { + return "fault"; + } + if (CLICKHOUSE_QUOTA_ERROR_TYPES.has(error.type)) { + return "quota"; + } + if (userAuthoredQuery && CLICKHOUSE_INVALID_SQL_ERROR_TYPES.has(error.type)) { + return "invalid-sql"; + } + return "fault"; } function recordClickhouseError(span: Span, error: Error): void { diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index c39794bf282..3fefd72066d 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -114,6 +114,11 @@ export interface ExecuteTSQLOptions { * (counters zero-fill, gauges carry forward). Off by default. */ fillGaps?: boolean; + /** + * Set when `query` was written by whoever made the request rather than by us. + * A rejection of their SQL is then their mistake, not a bug on our side. + */ + userAuthoredQuery?: boolean; } /** @@ -215,6 +220,7 @@ export async function executeTSQL( schema: isExplain ? z.object({ explain: z.string() }) : options.schema, settings: options.clickhouseSettings, logFields: { tsql: options.query }, + userAuthoredQuery: options.userAuthoredQuery, }); const [error, result] = await queryFn(params); @@ -249,6 +255,7 @@ export async function executeTSQL( schema: z.object({ explain: z.string() }), settings: options.clickhouseSettings, logFields: { tsql: options.query }, + userAuthoredQuery: options.userAuthoredQuery, }); const [additionalError, additionalResult] = await additionalQueryFn(params); diff --git a/internal-packages/clickhouse/src/client/types.ts b/internal-packages/clickhouse/src/client/types.ts index 5c674776370..4bfa6dc4662 100644 --- a/internal-packages/clickhouse/src/client/types.ts +++ b/internal-packages/clickhouse/src/client/types.ts @@ -140,6 +140,11 @@ export interface ClickhouseReader { * record what produced the SQL, e.g. the TSQL a caller actually wrote. */ logFields?: Record; + /** + * Set when the SQL originates from whoever made the request rather than + * from us. Invalid-SQL rejections are then their mistake, not a bug. + */ + userAuthoredQuery?: boolean; }): ClickhouseQueryWithStatsFunction, z.output>; queryFast, TParams extends Record>(req: { diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index 937b2a2dcae..7092966b931 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -1694,6 +1694,55 @@ describe("TSQL Error Log Levels", () => { } ); + clickhouseTest( + "logs invalid caller-written SQL as a warning", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-user-authored-invalid", + query: "SELECT status, sum(is_test) AS n FROM task_runs", + schema: z.object({ status: z.string(), n: z.number() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("ClickHouse rejected an invalid query"); + expect(logged(errorSpy)).not.toContain("Error querying clickhouse"); + } + ); + + clickhouseTest( + "keeps invalid SQL we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-internal-invalid", + query: "SELECT status, sum(is_test) AS n FROM task_runs", + schema: z.object({ status: z.string(), n: z.number() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(warnSpy)).not.toContain("ClickHouse rejected an invalid query"); + } + ); + clickhouseTest("logs a ClickHouse limit breach as a warning", async ({ clickhouseContainer }) => { const client = new ClickhouseClient({ name: "test", From ac5a75802473019694067357a8df40669a940369 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 29 Jul 2026 17:41:22 +0100 Subject: [PATCH 5/7] fix(clickhouse): keep limit breaches on our own queries at error level Downgrading quota errors regardless of who wrote the SQL was wrong. An internal analytics query hitting the memory ceiling or a timeout is a runaway query of ours, and it was losing the alert it used to raise. Both downgrades are now gated on the same condition: ClickHouse rejected SQL that the caller wrote. Anything on SQL we generated stays at error, which also makes query, queryFast and the streaming path plain error logs again, since none of them can carry caller-written SQL. --- .../clickhouse/src/client/client.ts | 36 ++++--------------- internal-packages/clickhouse/src/tsql.test.ts | 32 +++++++++++++++++ 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index f35f83d214c..ef18883745b 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -179,16 +179,7 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { queryId, }; - switch (classifyClickhouseError(clickhouseError, false)) { - case "quota": - this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); - break; - case "invalid-sql": - this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); - break; - default: - this.logger.error("Error querying clickhouse", errorLogFields); - } + this.logger.error("Error querying clickhouse", errorLogFields); recordClickhouseError(span, clickhouseError); @@ -494,16 +485,7 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { queryId, }; - switch (classifyClickhouseError(clickhouseError, false)) { - case "quota": - this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); - break; - case "invalid-sql": - this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); - break; - default: - this.logger.error("Error querying clickhouse", errorLogFields); - } + this.logger.error("Error querying clickhouse", errorLogFields); recordClickhouseError(span, clickhouseError); @@ -651,11 +633,7 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { queryId, }; - if (error instanceof Error && classifyClickhouseError(error, false) === "quota") { - self.logger.warn("Streamed query exceeded a ClickHouse limit", errorLogFields); - } else { - self.logger.error("Error streaming clickhouse", errorLogFields); - } + self.logger.error("Error streaming clickhouse", errorLogFields); if (error instanceof Error) { recordClickhouseError(span, error); @@ -1053,8 +1031,8 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { /** * ClickHouse error types raised by a query that is valid but asks for more than - * the caller is allowed to spend. The caller gets a 4xx and there is nothing on - * our side to fix, so these are logged at warn rather than error. + * it is allowed to spend. Only downgraded for SQL the caller wrote: a runaway + * query we generated is our bug and still has to alert. */ const CLICKHOUSE_QUOTA_ERROR_TYPES = new Set([ "MEMORY_LIMIT_EXCEEDED", @@ -1098,13 +1076,13 @@ function classifyClickhouseError( error: Error, userAuthoredQuery: boolean | undefined ): ClickhouseErrorCategory { - if (!(error instanceof ClickHouseError) || error.type === undefined) { + if (!userAuthoredQuery || !(error instanceof ClickHouseError) || error.type === undefined) { return "fault"; } if (CLICKHOUSE_QUOTA_ERROR_TYPES.has(error.type)) { return "quota"; } - if (userAuthoredQuery && CLICKHOUSE_INVALID_SQL_ERROR_TYPES.has(error.type)) { + if (CLICKHOUSE_INVALID_SQL_ERROR_TYPES.has(error.type)) { return "invalid-sql"; } return "fault"; diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index 7092966b931..37b25e241ea 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -1763,6 +1763,7 @@ describe("TSQL Error Log Levels", () => { organization_id: { op: "eq", value: "org_tenant1" }, }, tableSchema: [taskRunsSchema], + userAuthoredQuery: true, clickhouseSettings: { max_rows_to_read: "1" }, }); @@ -1770,4 +1771,35 @@ describe("TSQL Error Log Levels", () => { expect(logged(warnSpy)).toContain("Query exceeded a ClickHouse limit"); expect(logged(errorSpy)).not.toContain("Error querying clickhouse"); }); + + clickhouseTest( + "keeps a limit breach on a query we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + await insertTaskRuns(client, { async_insert: 0 })([ + createTaskRun({ run_id: "run_intlimit1" }), + createTaskRun({ run_id: "run_intlimit2" }), + createTaskRun({ run_id: "run_intlimit3" }), + ]); + + const [error] = await executeTSQL(client, { + name: "test-internal-resource-limit", + query: "SELECT run_id FROM task_runs", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + clickhouseSettings: { max_rows_to_read: "1" }, + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(warnSpy)).not.toContain("Query exceeded a ClickHouse limit"); + } + ); }); From 80fd75de59382be83fe087b0e553f4849d734652 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 29 Jul 2026 17:56:24 +0100 Subject: [PATCH 6/7] fix(clickhouse): gate the compile-time downgrade on who wrote the query The execution layer only downgrades when the caller wrote the SQL, but the compile-time catch downgraded every ExposedTSQLError regardless. A compile failure on TRQL we generated is our bug, and it was logging at warn. Whether an error is safe to show someone is a separate question from whose mistake it is, so the same gate now applies at both layers. --- .../clickhouse/src/client/tsql.ts | 2 +- internal-packages/clickhouse/src/tsql.test.ts | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index 3fefd72066d..2cf586c36f1 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -321,7 +321,7 @@ export async function executeTSQL( generatedParams: generatedParams ?? {}, }; - const callerWroteABadQuery = error instanceof ExposedTSQLError; + const callerWroteABadQuery = options.userAuthoredQuery && error instanceof ExposedTSQLError; if (callerWroteABadQuery) { logger.warn("[TSQL] Invalid query", logFields); diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index 37b25e241ea..ff16508fdc7 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -1642,6 +1642,7 @@ describe("TSQL Error Log Levels", () => { organization_id: { op: "eq", value: "org_tenant1" }, }, tableSchema: [taskRunsSchema], + userAuthoredQuery: true, }); expect(error).not.toBeNull(); @@ -1663,6 +1664,7 @@ describe("TSQL Error Log Levels", () => { organization_id: { op: "eq", value: "org_tenant1" }, }, tableSchema: [taskRunsSchema], + userAuthoredQuery: true, }); expect(error).not.toBeNull(); @@ -1694,6 +1696,30 @@ describe("TSQL Error Log Levels", () => { } ); + clickhouseTest( + "keeps a compile failure on TRQL we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-internal-compile-error", + query: "SELECT nope FROM task_runs", + schema: z.object({ nope: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("[TSQL] Query error"); + expect(logged(warnSpy)).not.toContain("[TSQL] Invalid query"); + } + ); + clickhouseTest( "logs invalid caller-written SQL as a warning", async ({ clickhouseContainer }) => { From 5463a4089ea2f831fece4cfd840ac4ab972b370c Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 29 Jul 2026 18:08:52 +0100 Subject: [PATCH 7/7] fix(webapp): treat agent-written TRQL as caller-authored The agent's charts post TRQL an LLM wrote to the metric endpoint. A query it gets wrong is not a bug in our code, so those rejections should not alert either. The endpoint also serves built-in dashboard tiles, whose TRQL we do write, so the flag is set by the caller rather than the route: only the agent's charts opt in. Built-in tiles and the queue metric cards stay at error level. --- apps/webapp/app/components/dashboard-agent/AgentChart.tsx | 1 + apps/webapp/app/routes/resources.metric.tsx | 3 +++ 2 files changed, 4 insertions(+) diff --git a/apps/webapp/app/components/dashboard-agent/AgentChart.tsx b/apps/webapp/app/components/dashboard-agent/AgentChart.tsx index 6e082231eb6..dbd1297c58f 100644 --- a/apps/webapp/app/components/dashboard-agent/AgentChart.tsx +++ b/apps/webapp/app/components/dashboard-agent/AgentChart.tsx @@ -67,6 +67,7 @@ export function AgentChart({ block }: { block: ChartBlock }) { period: block.period ?? null, from: null, to: null, + userAuthoredQuery: true, }), signal: controller.signal, }) diff --git a/apps/webapp/app/routes/resources.metric.tsx b/apps/webapp/app/routes/resources.metric.tsx index 5bf0ed693ad..1808f6107e4 100644 --- a/apps/webapp/app/routes/resources.metric.tsx +++ b/apps/webapp/app/routes/resources.metric.tsx @@ -52,6 +52,7 @@ const MetricWidgetQuery = z.object({ tags: z.array(z.string()).optional(), // Opt into server-side gap fill (carry-forward for gauges, zero-fill for counters). fillGaps: z.boolean().optional(), + userAuthoredQuery: z.boolean().optional(), }); export const action = async ({ request }: ActionFunctionArgs) => { @@ -88,6 +89,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { providers, tags: _tags, fillGaps, + userAuthoredQuery, } = submission.data; // Check they should be able to access it @@ -126,6 +128,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { operations, providers, fillGaps, + userAuthoredQuery, // Set higher concurrency if many widgets are on screen at once customOrgConcurrencyLimit: env.METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT, });