From 381ae3c7ee36a4b0531cbbaad27f9c25f6030b7c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 12:38:30 -0700 Subject: [PATCH 1/2] fix(realtime): noindex the socket server's 404 responses The sockets.* hostnames are served by this server and return a plain JSON 404, which Google Search Console reports as crawl errors. Mark unmatched routes noindex so crawlers drop the hostnames instead of retrying them. --- apps/realtime/src/routes/http.test.ts | 44 +++++++++++++++++++++++++++ apps/realtime/src/routes/http.ts | 5 ++- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 apps/realtime/src/routes/http.test.ts diff --git a/apps/realtime/src/routes/http.test.ts b/apps/realtime/src/routes/http.test.ts new file mode 100644 index 00000000000..057bfe46642 --- /dev/null +++ b/apps/realtime/src/routes/http.test.ts @@ -0,0 +1,44 @@ +import type { IncomingMessage, ServerResponse } from 'http' +import { describe, expect, it, vi } from 'vitest' +import type { IRoomManager } from '@/rooms' +import { createHttpHandler } from '@/routes/http' + +function createMocks(req: Partial) { + const writeHead = vi.fn() + const end = vi.fn() + const logger = { info: vi.fn(), error: vi.fn(), debug: vi.fn(), warn: vi.fn() } + const roomManager = { + getTotalActiveConnections: vi.fn().mockResolvedValue(0), + isReady: vi.fn().mockReturnValue(true), + } as unknown as IRoomManager + + return { + handler: createHttpHandler(roomManager, logger), + req: { headers: {}, ...req } as IncomingMessage, + res: { writeHead, end } as unknown as ServerResponse, + writeHead, + end, + } +} + +describe('createHttpHandler', () => { + it('marks unmatched routes noindex so crawlers drop the socket hostnames', async () => { + const { handler, req, res, writeHead, end } = createMocks({ method: 'GET', url: '/' }) + + await handler(req, res) + + expect(writeHead).toHaveBeenCalledWith(404, { + 'Content-Type': 'application/json', + 'X-Robots-Tag': 'noindex, nofollow', + }) + expect(end).toHaveBeenCalledWith(JSON.stringify({ error: 'Not found' })) + }) + + it('does not mark the health check noindex', async () => { + const { handler, req, res, writeHead } = createMocks({ method: 'GET', url: '/health' }) + + await handler(req, res) + + expect(writeHead).toHaveBeenCalledWith(200, { 'Content-Type': 'application/json' }) + }) +}) diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 0f8ed73cc52..c22cf81388a 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -150,7 +150,10 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } - res.writeHead(404, { 'Content-Type': 'application/json' }) + res.writeHead(404, { + 'Content-Type': 'application/json', + 'X-Robots-Tag': 'noindex, nofollow', + }) res.end(JSON.stringify({ error: 'Not found' })) } } From 88e87472ea60e2f691b4c37f257b785652dfcb24 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 31 Jul 2026 12:45:48 -0700 Subject: [PATCH 2/2] fix(realtime): noindex every response, not just the 404 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting the header only on the 404 fallback covered the one response that crawlers already drop on status code alone, while /health — the sole route returning 200 with a body, and so the only indexable surface on the socket hostnames — stayed uncovered, with a test pinning it that way. Set it once on the handler instead. Node merges setHeader values into writeHead and no branch sets X-Robots-Tag, so it reaches every response. --- apps/realtime/src/routes/http.test.ts | 31 +++++++++++++++++++++------ apps/realtime/src/routes/http.ts | 7 +++--- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/apps/realtime/src/routes/http.test.ts b/apps/realtime/src/routes/http.test.ts index 057bfe46642..725341deac9 100644 --- a/apps/realtime/src/routes/http.test.ts +++ b/apps/realtime/src/routes/http.test.ts @@ -4,6 +4,7 @@ import type { IRoomManager } from '@/rooms' import { createHttpHandler } from '@/routes/http' function createMocks(req: Partial) { + const setHeader = vi.fn() const writeHead = vi.fn() const end = vi.fn() const logger = { info: vi.fn(), error: vi.fn(), debug: vi.fn(), warn: vi.fn() } @@ -15,26 +16,42 @@ function createMocks(req: Partial) { return { handler: createHttpHandler(roomManager, logger), req: { headers: {}, ...req } as IncomingMessage, - res: { writeHead, end } as unknown as ServerResponse, + res: { setHeader, writeHead, end } as unknown as ServerResponse, + setHeader, writeHead, end, } } describe('createHttpHandler', () => { - it('marks unmatched routes noindex so crawlers drop the socket hostnames', async () => { + /** + * `/health` is the only route on this server that returns 200 with a body, so + * it is the only genuinely indexable surface on the `sockets.*` hostnames. + * Node merges `setHeader` values into `writeHead`, and no branch here sets + * `X-Robots-Tag`, so the handler-level call reaches every response. + */ + it.each([ + ['health check', { method: 'GET', url: '/health' }], + ['unmatched route', { method: 'GET', url: '/' }], + ['unauthenticated internal API call', { method: 'POST', url: '/api/workflow-deleted' }], + ])('marks the %s noindex', async (_label, req) => { + const { handler, req: request, res, setHeader } = createMocks(req) + + await handler(request, res) + + expect(setHeader).toHaveBeenCalledWith('X-Robots-Tag', 'noindex, nofollow') + }) + + it('still serves the unmatched-route 404 unchanged', async () => { const { handler, req, res, writeHead, end } = createMocks({ method: 'GET', url: '/' }) await handler(req, res) - expect(writeHead).toHaveBeenCalledWith(404, { - 'Content-Type': 'application/json', - 'X-Robots-Tag': 'noindex, nofollow', - }) + expect(writeHead).toHaveBeenCalledWith(404, { 'Content-Type': 'application/json' }) expect(end).toHaveBeenCalledWith(JSON.stringify({ error: 'Not found' })) }) - it('does not mark the health check noindex', async () => { + it('still serves the health check as 200', async () => { const { handler, req, res, writeHead } = createMocks({ method: 'GET', url: '/health' }) await handler(req, res) diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index c22cf81388a..1e7b6faae99 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -59,6 +59,8 @@ function sendError(res: ServerResponse, message: string, status = 500): void { */ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return async (req: IncomingMessage, res: ServerResponse) => { + res.setHeader('X-Robots-Tag', 'noindex, nofollow') + // Health check doesn't require auth if (req.method === 'GET' && req.url === '/health') { try { @@ -150,10 +152,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } - res.writeHead(404, { - 'Content-Type': 'application/json', - 'X-Robots-Tag': 'noindex, nofollow', - }) + res.writeHead(404, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ error: 'Not found' })) } }